<commit_before>package ioutils

import (
	"fmt"
	"strings"
	"testing"
)

// Implement io.Reader
type errorReader struct{}

func (r *errorReader) Read(p []byte) (int, error) {
	return 0, fmt.Errorf("Error reader always fail.")
}

func TestReadCloserWrapperClose(t *testing.T) {
	reader := strings.NewReader("A string reader")
	wrapper := NewReadCloserWrapper(reader, func() error {
		return fmt.Errorf("This will be called when closing")
	})
	err := wrapper.Close()
	if err == nil || !strings.Contains(err.Error(), "This will be called when closing") {
		t.Fatalf("readCloserWrapper should have call the anonymous func and thus, fail.")
	}
}

func TestReaderErrWrapperReadOnError(t *testing.T) {
	called := false
	reader := &errorReader{}
	wrapper := NewReaderErrWrapper(reader, func() {
		called = true
	})
	_, err := wrapper.Read([]byte{})
	if err == nil || !strings.Contains(err.Error(), "Error reader always fail.") {
		t.Fatalf("readErrWrapper should returned an error")
	}
	if !called {
		t.Fatalf("readErrWrapper should have call the anonymous function on failure")
	}
}

func TestReaderErrWrapperRead(t *testing.T) {
	reader := strings.NewReader("a string reader.")
	wrapper := NewReaderErrWrapper(reader, func() {
		t.Fatalf("readErrWrapper should not have called the anonymous function")
	})
	// Read 20 byte (should be ok with the string above)
	num, err := wrapper.Read(make([]byte, 20))
	if err != nil {
		t.Fatal(err)
	}
	if num != 16 {
		t.Fatalf("readerErrWrapper should have read 16 byte, but read %d", num)
	}
}

func TestHashData(t *testing.T) {
	reader := strings.NewReader("hash-me")
	actual, err := HashData(reader)
	if err != nil {
		t.Fatal(err)
	}
	expected := "sha256:4d11186aed035cc624d553e10db358492c84a7cd6b9670d92123c144930450aa"
	if actual != expected {
		t.Fatalf("Expecting %s, got %s", expected, actual)
	}
}

type perpetualReader struct{}

func (p *perpetualReader) Read(buf []byte) (n int, err error) {
	for i := 0; i != len(buf); i++ {
		buf[i] = 'a'
	}
	return len(buf), nil
}
<commit_msg>Fix a lint error<commit_after>package ioutils

import (
	"fmt"
	"strings"
	"testing"
)

// Implement io.Reader
type errorReader struct{}

func (r *errorReader) Read(p []byte) (int, error) {
	return 0, fmt.Errorf("error reader always fail")
}

func TestReadCloserWrapperClose(t *testing.T) {
	reader := strings.NewReader("A string reader")
	wrapper := NewReadCloserWrapper(reader, func() error {
		return fmt.Errorf("This will be called when closing")
	})
	err := wrapper.Close()
	if err == nil || !strings.Contains(err.Error(), "This will be called when closing") {
		t.Fatalf("readCloserWrapper should have call the anonymous func and thus, fail.")
	}
}

func TestReaderErrWrapperReadOnError(t *testing.T) {
	called := false
	reader := &errorReader{}
	wrapper := NewReaderErrWrapper(reader, func() {
		called = true
	})
	_, err := wrapper.Read([]byte{})
	if err == nil || !strings.Contains(err.Error(), "error reader always fail") {
		t.Fatalf("readErrWrapper should returned an error")
	}
	if !called {
		t.Fatalf("readErrWrapper should have call the anonymous function on failure")
	}
}

func TestReaderErrWrapperRead(t *testing.T) {
	reader := strings.NewReader("a string reader.")
	wrapper := NewReaderErrWrapper(reader, func() {
		t.Fatalf("readErrWrapper should not have called the anonymous function")
	})
	// Read 20 byte (should be ok with the string above)
	num, err := wrapper.Read(make([]byte, 20))
	if err != nil {
		t.Fatal(err)
	}
	if num != 16 {
		t.Fatalf("readerErrWrapper should have read 16 byte, but read %d", num)
	}
}

func TestHashData(t *testing.T) {
	reader := strings.NewReader("hash-me")
	actual, err := HashData(reader)
	if err != nil {
		t.Fatal(err)
	}
	expected := "sha256:4d11186aed035cc624d553e10db358492c84a7cd6b9670d92123c144930450aa"
	if actual != expected {
		t.Fatalf("Expecting %s, got %s", expected, actual)
	}
}

type perpetualReader struct{}

func (p *perpetualReader) Read(buf []byte) (n int, err error) {
	for i := 0; i != len(buf); i++ {
		buf[i] = 'a'
	}
	return len(buf), nil
}
<|endoftext|>


<commit_before>// Copyright 2018-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sockmap

import (
	"fmt"
	"net"
	"unsafe"

	"github.com/cilium/cilium/common/types"
	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"
)

// SockmapKey is the 5-tuple used to lookup a socket
// +k8s:deepcopy-gen=true
// +k8s:deepcopy-gen:interfaces=github.com/cilium/cilium/pkg/bpf.MapKey
type SockmapKey struct {
	DIP    types.IPv6 `align:"$union0"`
	SIP    types.IPv6 `align:"$union1"`
	Family uint8      `align:"family"`
	Pad7   uint8      `align:"pad7"`
	Pad8   uint16     `align:"pad8"`
	SPort  uint32     `align:"sport"`
	DPort  uint32     `align:"dport"`
}

// SockmapValue is the fd of a socket
// +k8s:deepcopy-gen=true
// +k8s:deepcopy-gen:interfaces=github.com/cilium/cilium/pkg/bpf.MapValue
type SockmapValue struct {
	fd uint32
}

// String pretty print the 5-tuple as sip:sport->dip:dport
func (v SockmapKey) String() string {
	return fmt.Sprintf("%s:%d->%s:%d", v.SIP.String(), v.SPort, v.DIP.String(), v.DPort)
}

// String pretty print the file descriptor value, note this is local to agent.
func (v SockmapValue) String() string {
	return fmt.Sprintf("%d", v.fd)
}

// GetValuePtr returns the unsafe pointer to the BPF value.
func (v *SockmapValue) GetValuePtr() unsafe.Pointer { return unsafe.Pointer(v) }

// GetKeyPtr returns the unsafe pointer to the BPF key
func (k *SockmapKey) GetKeyPtr() unsafe.Pointer { return unsafe.Pointer(k) }

// NewValue returns a new empty instance of the structure representing the BPF
// map value
func (k SockmapKey) NewValue() bpf.MapValue { return &SockmapValue{} }

// NewSockmapKey returns a new key using 5-tuple input.
func NewSockmapKey(dip, sip net.IP, sport, dport uint32) SockmapKey {
	result := SockmapKey{}

	if sip4 := sip.To4(); sip4 != nil {
		result.Family = bpf.EndpointKeyIPv4
		copy(result.SIP[:], sip4)
	} else {
		result.Family = bpf.EndpointKeyIPv6
		copy(result.SIP[:], sip)
	}

	if dip4 := dip.To4(); dip4 != nil {
		result.Family = bpf.EndpointKeyIPv4
		copy(result.SIP[:], dip4)
	} else {
		result.Family = bpf.EndpointKeyIPv6
		copy(result.DIP[:], dip)
	}

	result.DPort = dport
	result.SPort = sport
	return result
}

var log = logging.DefaultLogger.WithField(logfields.LogSubsys, "sockmap")

const (
	mapName = "cilium_sock_ops"

	// MaxEntries represents the maximum number of endpoints in the map
	MaxEntries = 65535
)

var (
	// SockMap represents the BPF map for sockets
	SockMap = bpf.NewMap(mapName,
		bpf.MapTypeSockHash,
		&SockmapKey{},
		int(unsafe.Sizeof(SockmapKey{})),
		&SockmapValue{},
		4,
		MaxEntries,
		0, 0,
		bpf.ConvertKeyValue,
	)
)

// SockmapCreate will create sockmap map
func SockmapCreate() {
	SockMap.OpenOrCreate()
}
<commit_msg>sockmap: Factor out for better testing<commit_after>// Copyright 2018-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sockmap

import (
	"fmt"
	"net"
	"sync"
	"unsafe"

	"github.com/cilium/cilium/common/types"
	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"
)

// SockmapKey is the 5-tuple used to lookup a socket
// +k8s:deepcopy-gen=true
// +k8s:deepcopy-gen:interfaces=github.com/cilium/cilium/pkg/bpf.MapKey
type SockmapKey struct {
	DIP    types.IPv6 `align:"$union0"`
	SIP    types.IPv6 `align:"$union1"`
	Family uint8      `align:"family"`
	Pad7   uint8      `align:"pad7"`
	Pad8   uint16     `align:"pad8"`
	SPort  uint32     `align:"sport"`
	DPort  uint32     `align:"dport"`
}

// SockmapValue is the fd of a socket
// +k8s:deepcopy-gen=true
// +k8s:deepcopy-gen:interfaces=github.com/cilium/cilium/pkg/bpf.MapValue
type SockmapValue struct {
	fd uint32
}

// String pretty print the 5-tuple as sip:sport->dip:dport
func (v SockmapKey) String() string {
	return fmt.Sprintf("%s:%d->%s:%d", v.SIP.String(), v.SPort, v.DIP.String(), v.DPort)
}

// String pretty print the file descriptor value, note this is local to agent.
func (v SockmapValue) String() string {
	return fmt.Sprintf("%d", v.fd)
}

// GetValuePtr returns the unsafe pointer to the BPF value.
func (v *SockmapValue) GetValuePtr() unsafe.Pointer { return unsafe.Pointer(v) }

// GetKeyPtr returns the unsafe pointer to the BPF key
func (k *SockmapKey) GetKeyPtr() unsafe.Pointer { return unsafe.Pointer(k) }

// NewValue returns a new empty instance of the structure representing the BPF
// map value
func (k SockmapKey) NewValue() bpf.MapValue { return &SockmapValue{} }

// NewSockmapKey returns a new key using 5-tuple input.
func NewSockmapKey(dip, sip net.IP, sport, dport uint32) SockmapKey {
	result := SockmapKey{}

	if sip4 := sip.To4(); sip4 != nil {
		result.Family = bpf.EndpointKeyIPv4
		copy(result.SIP[:], sip4)
	} else {
		result.Family = bpf.EndpointKeyIPv6
		copy(result.SIP[:], sip)
	}

	if dip4 := dip.To4(); dip4 != nil {
		result.Family = bpf.EndpointKeyIPv4
		copy(result.SIP[:], dip4)
	} else {
		result.Family = bpf.EndpointKeyIPv6
		copy(result.DIP[:], dip)
	}

	result.DPort = dport
	result.SPort = sport
	return result
}

var log = logging.DefaultLogger.WithField(logfields.LogSubsys, "sockmap")

const (
	mapName = "cilium_sock_ops"

	// MaxEntries represents the maximum number of endpoints in the map
	MaxEntries = 65535
)

var (
	buildMap sync.Once
	// SockMap represents the BPF map for sockets
	SockMap *bpf.Map
)

// CreateWithName creates a new sockmap map.
//
// The specified mapName allows non-standard map paths to be used, for instance
// for testing purposes.
func CreateWithName(name string) error {
	buildMap.Do(func() {
		SockMap = bpf.NewMap(name,
			bpf.MapTypeSockHash,
			&SockmapKey{},
			int(unsafe.Sizeof(SockmapKey{})),
			&SockmapValue{},
			4,
			MaxEntries,
			0, 0,
			bpf.ConvertKeyValue,
		)
	})

	_, err := SockMap.OpenOrCreate()
	return err
}

// SockmapCreate will create sockmap map
func SockmapCreate() {
	if err := CreateWithName(mapName); err != nil {
		log.WithError(err).Warning("Unable to open or create socket map")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package http

import (
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"strings"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/probe"
)

const FailureCode int = -1

func containsAny(s string, substrs []string) bool {
	for _, substr := range substrs {
		if strings.Contains(s, substr) {
			return true
		}
	}
	return false
}

func TestHTTPProbeChecker(t *testing.T) {
	handleReq := func(s int, body string) func(w http.ResponseWriter, r *http.Request) {
		return func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(s)
			w.Write([]byte(body))
		}
	}

	prober := New()
	testCases := []struct {
		handler    func(w http.ResponseWriter, r *http.Request)
		reqHeaders http.Header
		health     probe.Result
		// go1.5: error message changed for timeout, need to support
		// both old and new
		accBodies []string
	}{
		// The probe will be filled in below.  This is primarily testing that an HTTP GET happens.
		{
			handleReq(http.StatusOK, "ok body"),
			nil,
			probe.Success,
			[]string{"ok body"},
		},
		{
			// Echo handler that returns the contents of request headers in the body
			func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(200)
				output := ""
				for k, arr := range r.Header {
					for _, v := range arr {
						output += fmt.Sprintf("%s: %s\n", k, v)
					}
				}
				w.Write([]byte(output))
			},
			http.Header{
				"X-Muffins-Or-Cupcakes": {"muffins"},
			},
			probe.Success,
			[]string{
				"X-Muffins-Or-Cupcakes: muffins",
			},
		},
		{
			// Echo handler that returns the contents of Host in the body
			func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(200)
				w.Write([]byte(r.Host))
			},
			http.Header{
				"Host": {"muffins.cupcakes.org"},
			},
			probe.Success,
			[]string{
				"muffins.cupcakes.org",
			},
		},
		{
			handleReq(FailureCode, "fail body"),
			nil,
			probe.Failure,
			[]string{
				fmt.Sprintf("HTTP probe failed with statuscode: %d", FailureCode),
				fmt.Sprintf("malformed HTTP status code \"%d\"", FailureCode),
			},
		},
		{
			func(w http.ResponseWriter, r *http.Request) {
				time.Sleep(3 * time.Second)
			},
			nil,
			probe.Failure,
			[]string{
				"use of closed network connection",
				"request canceled (Client.Timeout exceeded while awaiting headers)",
			},
		},
	}
	for _, test := range testCases {
		func() {
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				test.handler(w, r)
			}))
			defer server.Close()
			u, err := url.Parse(server.URL)
			if err != nil {
				t.Errorf("Unexpected error: %v", err)
			}
			_, port, err := net.SplitHostPort(u.Host)
			if err != nil {
				t.Errorf("Unexpected error: %v", err)
			}
			_, err = strconv.Atoi(port)
			if err != nil {
				t.Errorf("Unexpected error: %v", err)
			}
			health, output, err := prober.Probe(u, test.reqHeaders, 1*time.Second)
			if test.health == probe.Unknown && err == nil {
				t.Errorf("Expected error")
			}
			if test.health != probe.Unknown && err != nil {
				t.Errorf("Unexpected error: %v", err)
			}
			if health != test.health {
				t.Errorf("Expected %v, got %v", test.health, health)
			}
			if !containsAny(output, test.accBodies) {
				t.Errorf("Expected one of %#v, got %v", test.accBodies, output)
			}
		}()
	}
}
<commit_msg>pkg/probe/http: don't compare error strings in tests<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package http

import (
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"strings"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/probe"
)

const FailureCode int = -1

func TestHTTPProbeChecker(t *testing.T) {
	handleReq := func(s int, body string) func(w http.ResponseWriter, r *http.Request) {
		return func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(s)
			w.Write([]byte(body))
		}
	}

	prober := New()
	testCases := []struct {
		handler    func(w http.ResponseWriter, r *http.Request)
		reqHeaders http.Header
		health     probe.Result
		accBody    string
	}{
		// The probe will be filled in below.  This is primarily testing that an HTTP GET happens.
		{
			handler: handleReq(http.StatusOK, "ok body"),
			health:  probe.Success,
			accBody: "ok body",
		},
		{
			// Echo handler that returns the contents of request headers in the body
			handler: func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(200)
				output := ""
				for k, arr := range r.Header {
					for _, v := range arr {
						output += fmt.Sprintf("%s: %s\n", k, v)
					}
				}
				w.Write([]byte(output))
			},
			reqHeaders: http.Header{
				"X-Muffins-Or-Cupcakes": {"muffins"},
			},
			health:  probe.Success,
			accBody: "X-Muffins-Or-Cupcakes: muffins",
		},
		{
			// Echo handler that returns the contents of Host in the body
			handler: func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(200)
				w.Write([]byte(r.Host))
			},
			reqHeaders: http.Header{
				"Host": {"muffins.cupcakes.org"},
			},
			health:  probe.Success,
			accBody: "muffins.cupcakes.org",
		},
		{
			handler: handleReq(FailureCode, "fail body"),
			health:  probe.Failure,
		},
		{
			handler: handleReq(http.StatusInternalServerError, "fail body"),
			health:  probe.Failure,
		},
		{
			handler: func(w http.ResponseWriter, r *http.Request) {
				time.Sleep(3 * time.Second)
			},
			health: probe.Failure,
		},
	}
	for i, test := range testCases {
		func() {
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				test.handler(w, r)
			}))
			defer server.Close()
			u, err := url.Parse(server.URL)
			if err != nil {
				t.Errorf("case %d: unexpected error: %v", i, err)
			}
			_, port, err := net.SplitHostPort(u.Host)
			if err != nil {
				t.Errorf("case %d: unexpected error: %v", i, err)
			}
			_, err = strconv.Atoi(port)
			if err != nil {
				t.Errorf("case %d: unexpected error: %v", i, err)
			}
			health, output, err := prober.Probe(u, test.reqHeaders, 1*time.Second)
			if test.health == probe.Unknown && err == nil {
				t.Errorf("case %d: expected error", i)
			}
			if test.health != probe.Unknown && err != nil {
				t.Errorf("case %d: unexpected error: %v", i, err)
			}
			if health != test.health {
				t.Errorf("case %d: expected %v, got %v", i, test.health, health)
			}
			if health != probe.Failure && test.health != probe.Failure {
				if !strings.Contains(output, test.accBody) {
					t.Errorf("Expected response body to contain %v, got %v", test.accBody, output)
				}
			}
		}()
	}
}
<|endoftext|>


<commit_before>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        self.register_outfeed <- nntp
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }
        // don't use streaming if we have set mode reader
        if mode == "reader" && stream {
          stream = false
        }
        nntp.runConnection(self, false, stream, reader, c)
        self.deregister_outfeed <- nntp
      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          feed.check <- nntp.MessageID()
        }
      }
    case nntp := <- self.ask_for_article:
      for _, feed := range self.feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          log.Println("asking", feed.name, "for", nntp.MessageID())
          feed.article <- nntp.MessageID()
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, c)
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<commit_msg>fix last commit<commit_after>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        self.register_outfeed <- nntp
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }
        // don't use streaming if we have set mode reader
        if mode == "reader" && reader {
          stream = false
        }
        nntp.runConnection(self, false, stream, reader, c)
        self.deregister_outfeed <- nntp
      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          feed.check <- nntp.MessageID()
        }
      }
    case nntp := <- self.ask_for_article:
      for _, feed := range self.feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          log.Println("asking", feed.name, "for", nntp.MessageID())
          feed.article <- nntp.MessageID()
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, c)
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"context"
	"errors"
	"io"
	"os"
)

func query(ctx context.Context, filename, query string, bufSize int) (res []string, err error) {
	file, err := os.OpenFile(filename, os.O_RDWR, 0)
	if err != nil {
		return
	}
	defer file.Close()

	_, err = file.Seek(0, io.SeekStart)
	if err != nil {
		return
	}
	_, err = file.WriteString(query)
	if err != nil {
		return
	}
	_, err = file.Seek(0, io.SeekStart)
	if err != nil {
		return
	}
	buf := make([]byte, bufSize)
	for {
		n, _ := file.Read(buf)
		if n <= 0 {
			break
		}
		res = append(res, string(buf[:n]))
	}
	return
}

func queryCS(ctx context.Context, net, host, service string) (res []string, err error) {
	switch net {
	case "tcp4", "tcp6":
		net = "tcp"
	case "udp4", "udp6":
		net = "udp"
	}
	if host == "" {
		host = "*"
	}
	return query(ctx, netdir+"/cs", net+"!"+host+"!"+service, 128)
}

func queryCS1(ctx context.Context, net string, ip IP, port int) (clone, dest string, err error) {
	ips := "*"
	if len(ip) != 0 && !ip.IsUnspecified() {
		ips = ip.String()
	}
	lines, err := queryCS(ctx, net, ips, itoa(port))
	if err != nil {
		return
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return "", "", errors.New("bad response from ndb/cs")
	}
	clone, dest = f[0], f[1]
	return
}

func queryDNS(ctx context.Context, addr string, typ string) (res []string, err error) {
	return query(ctx, netdir+"/dns", addr+" "+typ, 1024)
}

// toLower returns a lower-case version of in. Restricting us to
// ASCII is sufficient to handle the IP protocol names and allow
// us to not depend on the strings and unicode packages.
func toLower(in string) string {
	for _, c := range in {
		if 'A' <= c && c <= 'Z' {
			// Has upper case; need to fix.
			out := []byte(in)
			for i := 0; i < len(in); i++ {
				c := in[i]
				if 'A' <= c && c <= 'Z' {
					c += 'a' - 'A'
				}
				out[i] = c
			}
			return string(out)
		}
	}
	return in
}

// lookupProtocol looks up IP protocol name and returns
// the corresponding protocol number.
func lookupProtocol(ctx context.Context, name string) (proto int, err error) {
	lines, err := query(ctx, netdir+"/cs", "!protocol="+toLower(name), 128)
	if err != nil {
		return 0, err
	}
	if len(lines) == 0 {
		return 0, UnknownNetworkError(name)
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return 0, UnknownNetworkError(name)
	}
	s := f[1]
	if n, _, ok := dtoi(s[byteIndex(s, '=')+1:]); ok {
		return n, nil
	}
	return 0, UnknownNetworkError(name)
}

func (*Resolver) lookupHost(ctx context.Context, host string) (addrs []string, err error) {
	// Use netdir/cs instead of netdir/dns because cs knows about
	// host names in local network (e.g. from /lib/ndb/local)
	lines, err := queryCS(ctx, "net", host, "1")
	if err != nil {
		if stringsHasSuffix(err.Error(), "dns failure") {
			err = errNoSuchHost
		}
		return
	}
loop:
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 2 {
			continue
		}
		addr := f[1]
		if i := byteIndex(addr, '!'); i >= 0 {
			addr = addr[:i] // remove port
		}
		if ParseIP(addr) == nil {
			continue
		}
		// only return unique addresses
		for _, a := range addrs {
			if a == addr {
				continue loop
			}
		}
		addrs = append(addrs, addr)
	}
	return
}

func (r *Resolver) lookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) {
	lits, err := r.lookupHost(ctx, host)
	if err != nil {
		return
	}
	for _, lit := range lits {
		host, zone := splitHostZone(lit)
		if ip := ParseIP(host); ip != nil {
			addr := IPAddr{IP: ip, Zone: zone}
			addrs = append(addrs, addr)
		}
	}
	return
}

func (*Resolver) lookupPort(ctx context.Context, network, service string) (port int, err error) {
	switch network {
	case "tcp4", "tcp6":
		network = "tcp"
	case "udp4", "udp6":
		network = "udp"
	}
	lines, err := queryCS(ctx, network, "127.0.0.1", toLower(service))
	if err != nil {
		return
	}
	unknownPortError := &AddrError{Err: "unknown port", Addr: network + "/" + service}
	if len(lines) == 0 {
		return 0, unknownPortError
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return 0, unknownPortError
	}
	s := f[1]
	if i := byteIndex(s, '!'); i >= 0 {
		s = s[i+1:] // remove address
	}
	if n, _, ok := dtoi(s); ok {
		return n, nil
	}
	return 0, unknownPortError
}

func (*Resolver) lookupCNAME(ctx context.Context, name string) (cname string, err error) {
	lines, err := queryDNS(ctx, name, "cname")
	if err != nil {
		if stringsHasSuffix(err.Error(), "dns failure") {
			cname = name + "."
			err = nil
		}
		return
	}
	if len(lines) > 0 {
		if f := getFields(lines[0]); len(f) >= 3 {
			return f[2] + ".", nil
		}
	}
	return "", errors.New("bad response from ndb/dns")
}

func (*Resolver) lookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*SRV, err error) {
	var target string
	if service == "" && proto == "" {
		target = name
	} else {
		target = "_" + service + "._" + proto + "." + name
	}
	lines, err := queryDNS(ctx, target, "srv")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 6 {
			continue
		}
		port, _, portOk := dtoi(f[4])
		priority, _, priorityOk := dtoi(f[3])
		weight, _, weightOk := dtoi(f[2])
		if !(portOk && priorityOk && weightOk) {
			continue
		}
		addrs = append(addrs, &SRV{absDomainName([]byte(f[5])), uint16(port), uint16(priority), uint16(weight)})
		cname = absDomainName([]byte(f[0]))
	}
	byPriorityWeight(addrs).sort()
	return
}

func (*Resolver) lookupMX(ctx context.Context, name string) (mx []*MX, err error) {
	lines, err := queryDNS(ctx, name, "mx")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 4 {
			continue
		}
		if pref, _, ok := dtoi(f[2]); ok {
			mx = append(mx, &MX{absDomainName([]byte(f[3])), uint16(pref)})
		}
	}
	byPref(mx).sort()
	return
}

func (*Resolver) lookupNS(ctx context.Context, name string) (ns []*NS, err error) {
	lines, err := queryDNS(ctx, name, "ns")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 3 {
			continue
		}
		ns = append(ns, &NS{absDomainName([]byte(f[2]))})
	}
	return
}

func (*Resolver) lookupTXT(ctx context.Context, name string) (txt []string, err error) {
	lines, err := queryDNS(ctx, name, "txt")
	if err != nil {
		return
	}
	for _, line := range lines {
		if i := byteIndex(line, '\t'); i >= 0 {
			txt = append(txt, absDomainName([]byte(line[i+1:])))
		}
	}
	return
}

func (*Resolver) lookupAddr(ctx context.Context, addr string) (name []string, err error) {
	arpa, err := reverseaddr(addr)
	if err != nil {
		return
	}
	lines, err := queryDNS(ctx, arpa, "ptr")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 3 {
			continue
		}
		name = append(name, absDomainName([]byte(f[2])))
	}
	return
}
<commit_msg>net: fix LookupCNAME on Plan 9<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"context"
	"errors"
	"io"
	"os"
)

func query(ctx context.Context, filename, query string, bufSize int) (res []string, err error) {
	file, err := os.OpenFile(filename, os.O_RDWR, 0)
	if err != nil {
		return
	}
	defer file.Close()

	_, err = file.Seek(0, io.SeekStart)
	if err != nil {
		return
	}
	_, err = file.WriteString(query)
	if err != nil {
		return
	}
	_, err = file.Seek(0, io.SeekStart)
	if err != nil {
		return
	}
	buf := make([]byte, bufSize)
	for {
		n, _ := file.Read(buf)
		if n <= 0 {
			break
		}
		res = append(res, string(buf[:n]))
	}
	return
}

func queryCS(ctx context.Context, net, host, service string) (res []string, err error) {
	switch net {
	case "tcp4", "tcp6":
		net = "tcp"
	case "udp4", "udp6":
		net = "udp"
	}
	if host == "" {
		host = "*"
	}
	return query(ctx, netdir+"/cs", net+"!"+host+"!"+service, 128)
}

func queryCS1(ctx context.Context, net string, ip IP, port int) (clone, dest string, err error) {
	ips := "*"
	if len(ip) != 0 && !ip.IsUnspecified() {
		ips = ip.String()
	}
	lines, err := queryCS(ctx, net, ips, itoa(port))
	if err != nil {
		return
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return "", "", errors.New("bad response from ndb/cs")
	}
	clone, dest = f[0], f[1]
	return
}

func queryDNS(ctx context.Context, addr string, typ string) (res []string, err error) {
	return query(ctx, netdir+"/dns", addr+" "+typ, 1024)
}

// toLower returns a lower-case version of in. Restricting us to
// ASCII is sufficient to handle the IP protocol names and allow
// us to not depend on the strings and unicode packages.
func toLower(in string) string {
	for _, c := range in {
		if 'A' <= c && c <= 'Z' {
			// Has upper case; need to fix.
			out := []byte(in)
			for i := 0; i < len(in); i++ {
				c := in[i]
				if 'A' <= c && c <= 'Z' {
					c += 'a' - 'A'
				}
				out[i] = c
			}
			return string(out)
		}
	}
	return in
}

// lookupProtocol looks up IP protocol name and returns
// the corresponding protocol number.
func lookupProtocol(ctx context.Context, name string) (proto int, err error) {
	lines, err := query(ctx, netdir+"/cs", "!protocol="+toLower(name), 128)
	if err != nil {
		return 0, err
	}
	if len(lines) == 0 {
		return 0, UnknownNetworkError(name)
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return 0, UnknownNetworkError(name)
	}
	s := f[1]
	if n, _, ok := dtoi(s[byteIndex(s, '=')+1:]); ok {
		return n, nil
	}
	return 0, UnknownNetworkError(name)
}

func (*Resolver) lookupHost(ctx context.Context, host string) (addrs []string, err error) {
	// Use netdir/cs instead of netdir/dns because cs knows about
	// host names in local network (e.g. from /lib/ndb/local)
	lines, err := queryCS(ctx, "net", host, "1")
	if err != nil {
		if stringsHasSuffix(err.Error(), "dns failure") {
			err = errNoSuchHost
		}
		return
	}
loop:
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 2 {
			continue
		}
		addr := f[1]
		if i := byteIndex(addr, '!'); i >= 0 {
			addr = addr[:i] // remove port
		}
		if ParseIP(addr) == nil {
			continue
		}
		// only return unique addresses
		for _, a := range addrs {
			if a == addr {
				continue loop
			}
		}
		addrs = append(addrs, addr)
	}
	return
}

func (r *Resolver) lookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) {
	lits, err := r.lookupHost(ctx, host)
	if err != nil {
		return
	}
	for _, lit := range lits {
		host, zone := splitHostZone(lit)
		if ip := ParseIP(host); ip != nil {
			addr := IPAddr{IP: ip, Zone: zone}
			addrs = append(addrs, addr)
		}
	}
	return
}

func (*Resolver) lookupPort(ctx context.Context, network, service string) (port int, err error) {
	switch network {
	case "tcp4", "tcp6":
		network = "tcp"
	case "udp4", "udp6":
		network = "udp"
	}
	lines, err := queryCS(ctx, network, "127.0.0.1", toLower(service))
	if err != nil {
		return
	}
	unknownPortError := &AddrError{Err: "unknown port", Addr: network + "/" + service}
	if len(lines) == 0 {
		return 0, unknownPortError
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return 0, unknownPortError
	}
	s := f[1]
	if i := byteIndex(s, '!'); i >= 0 {
		s = s[i+1:] // remove address
	}
	if n, _, ok := dtoi(s); ok {
		return n, nil
	}
	return 0, unknownPortError
}

func (*Resolver) lookupCNAME(ctx context.Context, name string) (cname string, err error) {
	lines, err := queryDNS(ctx, name, "cname")
	if err != nil {
		if stringsHasSuffix(err.Error(), "dns failure") || stringsHasSuffix(err.Error(), "resource does not exist; negrcode 0") {
			cname = name + "."
			err = nil
		}
		return
	}
	if len(lines) > 0 {
		if f := getFields(lines[0]); len(f) >= 3 {
			return f[2] + ".", nil
		}
	}
	return "", errors.New("bad response from ndb/dns")
}

func (*Resolver) lookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*SRV, err error) {
	var target string
	if service == "" && proto == "" {
		target = name
	} else {
		target = "_" + service + "._" + proto + "." + name
	}
	lines, err := queryDNS(ctx, target, "srv")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 6 {
			continue
		}
		port, _, portOk := dtoi(f[4])
		priority, _, priorityOk := dtoi(f[3])
		weight, _, weightOk := dtoi(f[2])
		if !(portOk && priorityOk && weightOk) {
			continue
		}
		addrs = append(addrs, &SRV{absDomainName([]byte(f[5])), uint16(port), uint16(priority), uint16(weight)})
		cname = absDomainName([]byte(f[0]))
	}
	byPriorityWeight(addrs).sort()
	return
}

func (*Resolver) lookupMX(ctx context.Context, name string) (mx []*MX, err error) {
	lines, err := queryDNS(ctx, name, "mx")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 4 {
			continue
		}
		if pref, _, ok := dtoi(f[2]); ok {
			mx = append(mx, &MX{absDomainName([]byte(f[3])), uint16(pref)})
		}
	}
	byPref(mx).sort()
	return
}

func (*Resolver) lookupNS(ctx context.Context, name string) (ns []*NS, err error) {
	lines, err := queryDNS(ctx, name, "ns")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 3 {
			continue
		}
		ns = append(ns, &NS{absDomainName([]byte(f[2]))})
	}
	return
}

func (*Resolver) lookupTXT(ctx context.Context, name string) (txt []string, err error) {
	lines, err := queryDNS(ctx, name, "txt")
	if err != nil {
		return
	}
	for _, line := range lines {
		if i := byteIndex(line, '\t'); i >= 0 {
			txt = append(txt, absDomainName([]byte(line[i+1:])))
		}
	}
	return
}

func (*Resolver) lookupAddr(ctx context.Context, addr string) (name []string, err error) {
	arpa, err := reverseaddr(addr)
	if err != nil {
		return
	}
	lines, err := queryDNS(ctx, arpa, "ptr")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 3 {
			continue
		}
		name = append(name, absDomainName([]byte(f[2])))
	}
	return
}
<|endoftext|>


<commit_before>package get

import (
	"os"
	"path/filepath"

	log "github.com/op/go-logging"

	"arvika.subliminl.com/developers/subliminl/git"
)

type Flags struct {
	Folder  string
	RepoUrl string
	Version string
}

// Get ensures that flags.Folder contains an up to date copy of flags.RepoUrl checked out to flags.Version.
func Get(log *log.Logger, flags *Flags) error {
	// Expand folder
	var err error
	flags.Folder, err = filepath.Abs(flags.Folder)
	if err != nil {
		return err
	}
	// Make sure a clone exists
	_, err = os.Stat(flags.Folder)
	if os.IsNotExist(err) {
		// Clone repo into folder
		if err := git.Clone(log, flags.RepoUrl, flags.Folder); err != nil {
			return err
		}
	}
	// Change dir to folder
	if err := os.Chdir(flags.Folder); err != nil {
		return err
	}
	// Specific version needed?
	if flags.Version == "" {
		// Get latest version
		if err := git.Pull(log, "origin"); err != nil {
			return err
		}
	} else {
		// Get latest (local) version
		localVersion, err := git.GetLatestTag(nil, flags.Folder)
		if err != nil {
			return err
		}
		if localVersion != flags.Version {
			// Checkout requested version
			log.Info("Found version %s, wanted %s", localVersion, flags.Version)
			// Fetch latest changes
			if err := git.Fetch(log, "origin"); err != nil {
				return err
			}
			// Checkout intended version
			if err := git.Checkout(log, flags.Version); err != nil {
				return err
			}
		} else {
			//log.Info("Found correct version of %s", flags.Folder)
		}
		// Get latest remote version
		remoteVersion, err := git.GetLatestRemoteTag(nil, flags.RepoUrl)
		if err != nil {
			return err
		}
		if remoteVersion != flags.Version {
			log.Warning("Latest remote version '%s' is different from requested version '%s'", remoteVersion, flags.Version)
		}
	}
	return nil
}
<commit_msg>Don't do a git pull after a fresh git clone<commit_after>package get

import (
	"os"
	"path/filepath"

	log "github.com/op/go-logging"

	"arvika.subliminl.com/developers/subliminl/git"
)

type Flags struct {
	Folder  string
	RepoUrl string
	Version string
}

// Get ensures that flags.Folder contains an up to date copy of flags.RepoUrl checked out to flags.Version.
func Get(log *log.Logger, flags *Flags) error {
	// Expand folder
	var err error
	flags.Folder, err = filepath.Abs(flags.Folder)
	if err != nil {
		return err
	}
	// Make sure a clone exists
	_, err = os.Stat(flags.Folder)
	cloned := false
	if os.IsNotExist(err) {
		// Clone repo into folder
		if err := git.Clone(log, flags.RepoUrl, flags.Folder); err != nil {
			return err
		}
		cloned = true
	}
	// Change dir to folder
	if err := os.Chdir(flags.Folder); err != nil {
		return err
	}
	// Specific version needed?
	if flags.Version == "" {
		// Get latest version
		if !cloned {
			if err := git.Pull(log, "origin"); err != nil {
				return err
			}
		}
	} else {
		// Get latest (local) version
		localVersion, err := git.GetLatestTag(nil, flags.Folder)
		if err != nil {
			return err
		}
		if localVersion != flags.Version {
			// Checkout requested version
			log.Info("Found version %s, wanted %s", localVersion, flags.Version)
			// Fetch latest changes
			if err := git.Fetch(log, "origin"); err != nil {
				return err
			}
			// Checkout intended version
			if err := git.Checkout(log, flags.Version); err != nil {
				return err
			}
		} else {
			//log.Info("Found correct version of %s", flags.Folder)
		}
		// Get latest remote version
		remoteVersion, err := git.GetLatestRemoteTag(nil, flags.RepoUrl)
		if err != nil {
			return err
		}
		if remoteVersion != flags.Version {
			log.Warning("Latest remote version '%s' is different from requested version '%s'", remoteVersion, flags.Version)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"sync"
)

func main() {
	// check that we have one or more CFs to poll
	//
	wg := &sync.WaitGroup{}
	wg.Add(1)

	wg.Wait()
}

func getDefaultConfig(name, fallback string) string {
	if val := os.Getenv(name); val != "" {
		return val
	}
	return fallback
}
<commit_msg>Sketch out the initial idea<commit_after>package main

import (
	"os"
	"sync"
	"time"
)

type CFInfo struct {
	API      string
	username string
	password string
}

func main() {
	// check that we have one or more CFs to poll
	//
	zones := parseZones()

	if len(zones) == 0 {
		os.Exit(1)
	}

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go workLoop(zones)
	wg.Wait()
}

func workLoop(zones []CFInfo) {
	t := time.NewTicker(time.Duration(5) * time.Second)

	for {
		select {
		case <-t.C:
			//	for each zone
			//		for each org
			// 			for each space
			//				for each app
			//					fetch metrics
		}
	}
}

func parseZones() []CFInfo {
	zones := make([]CFInfo, 0)

	return zones
}

func getDefaultConfig(name, fallback string) string {
	if val := os.Getenv(name); val != "" {
		return val
	}
	return fallback
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains support functionality for godoc.

package main

import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
	"utf8"
)

// An RWValue wraps a value and permits mutually exclusive
// access to it and records the time the value was last set.
//
type RWValue struct {
	mutex     sync.RWMutex
	value     interface{}
	timestamp int64 // time of last set(), in seconds since epoch
}

func (v *RWValue) set(value interface{}) {
	v.mutex.Lock()
	v.value = value
	v.timestamp = time.Seconds()
	v.mutex.Unlock()
}

func (v *RWValue) get() (interface{}, int64) {
	v.mutex.RLock()
	defer v.mutex.RUnlock()
	return v.value, v.timestamp
}

// TODO(gri) For now, using os.Getwd() is ok here since the functionality
//           based on this code is not invoked for the appengine version,
//           but this is fragile. Determine what the right thing to do is,
//           here (possibly have some Getwd-equivalent in FileSystem).
var cwd, _ = os.Getwd() // ignore errors

// canonicalizePaths takes a list of (directory/file) paths and returns
// the list of corresponding absolute paths in sorted (increasing) order.
// Relative paths are assumed to be relative to the current directory,
// empty and duplicate paths as well as paths for which filter(path) is
// false are discarded. filter may be nil in which case it is not used.
//
func canonicalizePaths(list []string, filter func(path string) bool) []string {
	i := 0
	for _, path := range list {
		path = strings.TrimSpace(path)
		if len(path) == 0 {
			continue // ignore empty paths (don't assume ".")
		}
		// len(path) > 0: normalize path
		if filepath.IsAbs(path) {
			path = filepath.Clean(path)
		} else {
			path = filepath.Join(cwd, path)
		}
		// we have a non-empty absolute path
		if filter != nil && !filter(path) {
			continue
		}
		// keep the path
		list[i] = path
		i++
	}
	list = list[0:i]

	// sort the list and remove duplicate entries
	sort.Strings(list)
	i = 0
	prev := ""
	for _, path := range list {
		if path != prev {
			list[i] = path
			i++
			prev = path
		}
	}

	return list[0:i]
}

// writeFileAtomically writes data to a temporary file and then
// atomically renames that file to the file named by filename.
//
func writeFileAtomically(filename string, data []byte) os.Error {
	// TODO(gri) this won't work on appengine
	f, err := ioutil.TempFile(filepath.Split(filename))
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	f.Close()
	if err != nil {
		return err
	}
	if n < len(data) {
		return io.ErrShortWrite
	}
	return os.Rename(f.Name(), filename)
}

// isText returns true if a significant prefix of s looks like correct UTF-8;
// that is, if it is likely that s is human-readable text.
//
func isText(s []byte) bool {
	const max = 1024 // at least utf8.UTFMax
	if len(s) > max {
		s = s[0:max]
	}
	for i, c := range string(s) {
		if i+utf8.UTFMax > len(s) {
			// last char may be incomplete - ignore
			break
		}
		if c == 0xFFFD || c < ' ' && c != '\n' && c != '\t' {
			// decoding error or control character - not a text file
			return false
		}
	}
	return true
}

// TODO(gri): Should have a mapping from extension to handler, eventually.

// textExt[x] is true if the extension x indicates a text file, and false otherwise.
var textExt = map[string]bool{
	".css": false, // must be served raw
	".js":  false, // must be served raw
}

// isTextFile returns true if the file has a known extension indicating
// a text file, or if a significant chunk of the specified file looks like
// correct UTF-8; that is, if it is likely that the file contains human-
// readable text.
//
func isTextFile(filename string) bool {
	// if the extension is known, use it for decision making
	if isText, found := textExt[filepath.Ext(filename)]; found {
		return isText
	}

	// the extension is not known; read an initial chunk
	// of the file and check if it looks like text
	f, err := fs.Open(filename)
	if err != nil {
		return false
	}
	defer f.Close()

	var buf [1024]byte
	n, err := f.Read(buf[0:])
	if err != nil {
		return false
	}

	return isText(buf[0:n])
}
<commit_msg>godoc: allow form feed in text files<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains support functionality for godoc.

package main

import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
	"utf8"
)

// An RWValue wraps a value and permits mutually exclusive
// access to it and records the time the value was last set.
//
type RWValue struct {
	mutex     sync.RWMutex
	value     interface{}
	timestamp int64 // time of last set(), in seconds since epoch
}

func (v *RWValue) set(value interface{}) {
	v.mutex.Lock()
	v.value = value
	v.timestamp = time.Seconds()
	v.mutex.Unlock()
}

func (v *RWValue) get() (interface{}, int64) {
	v.mutex.RLock()
	defer v.mutex.RUnlock()
	return v.value, v.timestamp
}

// TODO(gri) For now, using os.Getwd() is ok here since the functionality
//           based on this code is not invoked for the appengine version,
//           but this is fragile. Determine what the right thing to do is,
//           here (possibly have some Getwd-equivalent in FileSystem).
var cwd, _ = os.Getwd() // ignore errors

// canonicalizePaths takes a list of (directory/file) paths and returns
// the list of corresponding absolute paths in sorted (increasing) order.
// Relative paths are assumed to be relative to the current directory,
// empty and duplicate paths as well as paths for which filter(path) is
// false are discarded. filter may be nil in which case it is not used.
//
func canonicalizePaths(list []string, filter func(path string) bool) []string {
	i := 0
	for _, path := range list {
		path = strings.TrimSpace(path)
		if len(path) == 0 {
			continue // ignore empty paths (don't assume ".")
		}
		// len(path) > 0: normalize path
		if filepath.IsAbs(path) {
			path = filepath.Clean(path)
		} else {
			path = filepath.Join(cwd, path)
		}
		// we have a non-empty absolute path
		if filter != nil && !filter(path) {
			continue
		}
		// keep the path
		list[i] = path
		i++
	}
	list = list[0:i]

	// sort the list and remove duplicate entries
	sort.Strings(list)
	i = 0
	prev := ""
	for _, path := range list {
		if path != prev {
			list[i] = path
			i++
			prev = path
		}
	}

	return list[0:i]
}

// writeFileAtomically writes data to a temporary file and then
// atomically renames that file to the file named by filename.
//
func writeFileAtomically(filename string, data []byte) os.Error {
	// TODO(gri) this won't work on appengine
	f, err := ioutil.TempFile(filepath.Split(filename))
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	f.Close()
	if err != nil {
		return err
	}
	if n < len(data) {
		return io.ErrShortWrite
	}
	return os.Rename(f.Name(), filename)
}

// isText returns true if a significant prefix of s looks like correct UTF-8;
// that is, if it is likely that s is human-readable text.
//
func isText(s []byte) bool {
	const max = 1024 // at least utf8.UTFMax
	if len(s) > max {
		s = s[0:max]
	}
	for i, c := range string(s) {
		if i+utf8.UTFMax > len(s) {
			// last char may be incomplete - ignore
			break
		}
		if c == 0xFFFD || c < ' ' && c != '\n' && c != '\t' && c != '\f' {
			// decoding error or control character - not a text file
			return false
		}
	}
	return true
}

// TODO(gri): Should have a mapping from extension to handler, eventually.

// textExt[x] is true if the extension x indicates a text file, and false otherwise.
var textExt = map[string]bool{
	".css": false, // must be served raw
	".js":  false, // must be served raw
}

// isTextFile returns true if the file has a known extension indicating
// a text file, or if a significant chunk of the specified file looks like
// correct UTF-8; that is, if it is likely that the file contains human-
// readable text.
//
func isTextFile(filename string) bool {
	// if the extension is known, use it for decision making
	if isText, found := textExt[filepath.Ext(filename)]; found {
		return isText
	}

	// the extension is not known; read an initial chunk
	// of the file and check if it looks like text
	f, err := fs.Open(filename)
	if err != nil {
		return false
	}
	defer f.Close()

	var buf [1024]byte
	n, err := f.Read(buf[0:])
	if err != nil {
		return false
	}

	return isText(buf[0:n])
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag";
	"fmt";
	"go/ast";
	"go/parser";
	"go/printer";
	"io";
	"os";
	"sort";
	"tabwriter";
)

var (
	// operation modes
	silent = flag.Bool("s", false, "silent mode: parsing only");
	verbose = flag.Bool("v", false, "verbose mode: trace parsing");
	exports = flag.Bool("x", false, "show exports only");

	// layout control
	tabwidth = flag.Int("tabwidth", 4, "tab width");
	usetabs = flag.Bool("tabs", false, "align with tabs instead of blanks");
	optcommas = flag.Bool("optcommas", false, "print optional commas");
	optsemis = flag.Bool("optsemis", false, "print optional semicolons");
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [file.go]\n");
	flag.PrintDefaults();
	os.Exit(1);
}

func parserMode() uint {
	mode := parser.ParseComments;
	if *verbose {
		mode |= parser.Trace;
	}
	return mode;
}

func printerMode() uint {
	mode := printer.DocComments;
	if *optcommas {
		mode |= printer.OptCommas;
	}
	if *optsemis {
		mode |= printer.OptSemis;
	}
	return mode;
}

func makeTabwriter(writer io.Writer) *tabwriter.Writer {
	padchar := byte(' ');
	if *usetabs {
		padchar = '\t';
	}
	return tabwriter.NewWriter(writer, *tabwidth, 1, padchar, 0);
}

func main() {
	flag.Parse();

	var filename string;
	switch flag.NArg() {
	case 0: filename = "/dev/stdin";
	case 1: filename = flag.Arg(0);
	default: usage();
	}

	src, err := io.ReadFile(filename);
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %v\n", filename, err);
		os.Exit(1);
	}

	prog, err := parser.Parse(src, parserMode());
	if err != nil {
		if errors, ok := err.(parser.ErrorList); ok {
			sort.Sort(errors);
			for _, e := range errors {
				fmt.Fprintf(os.Stderr, "%s:%v\n", filename, e);
			}
		} else {
			fmt.Fprintf(os.Stderr, "%s: %v\n", filename, err);
		}
		os.Exit(1);
	}

	if !*silent {
		if *exports {
			ast.FilterExports(prog);  // ignore result
		}
		w := makeTabwriter(os.Stdout);
		printer.Fprint(w, prog, printerMode());
		w.Flush();
	}
}
<commit_msg>mention file name only once in error message<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag";
	"fmt";
	"go/ast";
	"go/parser";
	"go/printer";
	"io";
	"os";
	"sort";
	"tabwriter";
)

var (
	// operation modes
	silent = flag.Bool("s", false, "silent mode: parsing only");
	verbose = flag.Bool("v", false, "verbose mode: trace parsing");
	exports = flag.Bool("x", false, "show exports only");

	// layout control
	tabwidth = flag.Int("tabwidth", 4, "tab width");
	usetabs = flag.Bool("tabs", false, "align with tabs instead of blanks");
	optcommas = flag.Bool("optcommas", false, "print optional commas");
	optsemis = flag.Bool("optsemis", false, "print optional semicolons");
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [file.go]\n");
	flag.PrintDefaults();
	os.Exit(1);
}

func parserMode() uint {
	mode := parser.ParseComments;
	if *verbose {
		mode |= parser.Trace;
	}
	return mode;
}

func printerMode() uint {
	mode := printer.DocComments;
	if *optcommas {
		mode |= printer.OptCommas;
	}
	if *optsemis {
		mode |= printer.OptSemis;
	}
	return mode;
}

func makeTabwriter(writer io.Writer) *tabwriter.Writer {
	padchar := byte(' ');
	if *usetabs {
		padchar = '\t';
	}
	return tabwriter.NewWriter(writer, *tabwidth, 1, padchar, 0);
}

func main() {
	flag.Parse();

	var filename string;
	switch flag.NArg() {
	case 0: filename = "/dev/stdin";
	case 1: filename = flag.Arg(0);
	default: usage();
	}

	src, err := io.ReadFile(filename);
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err);
		os.Exit(1);
	}

	prog, err := parser.Parse(src, parserMode());
	if err != nil {
		if errors, ok := err.(parser.ErrorList); ok {
			sort.Sort(errors);
			for _, e := range errors {
				fmt.Fprintf(os.Stderr, "%s:%v\n", filename, e);
			}
		} else {
			fmt.Fprintf(os.Stderr, "%s: %v\n", filename, err);
		}
		os.Exit(1);
	}

	if !*silent {
		if *exports {
			ast.FilterExports(prog);  // ignore result
		}
		w := makeTabwriter(os.Stdout);
		printer.Fprint(w, prog, printerMode());
		w.Flush();
	}
}
<|endoftext|>


<commit_before>package tenho

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

func Start() {
	start := time.Now().UnixNano()
	// 処理
	var i float64
	i = 0
	for {
		i++
		end := time.Now().UnixNano()
		diff := float64(end-start) / 1000000000
		m := i / diff
		out := 0
		if m >= 0 {
			out = int(m)
		}
		seed := time.Now().UnixNano()
		var hai string
		var ok bool
		hai, ok = tryOnce(seed)
		if int(i)%10000 == 0 {
			fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
		}
		if ok {
			fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
			break
		}
		//fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
		//if i > 100000 {
		//	break
		//}
	}
	fmt.Printf("\n")
}

func tryOnce(seed int64) (string, bool) {
	hand := ShuffledHand(seed)
	hai := hand.HaiString()
	ok := hand.Solve()
	return hai, ok
}

// http://d.hatena.ne.jp/hake/20150930/p1
func shuffle(hand Hand) {
	for i := len(hand); i > 1; i-- {
		j := rand.Intn(i) // 0 .. i-1 の乱数発生
		hand[i-1], hand[j] = hand[j], hand[i-1]
	}
}

const HandSize = 14
const MahjongSetSize = 136

var defaultSet []int

func GetMahjongSet() []int {
	if defaultSet == nil {
		size := MahjongSetSize
		defaultSet = make([]int, size, size)
		for i := 0; i < size; i++ {
			defaultSet[i] = i / 4
		}
	}
	return defaultSet
}

func ShuffledHand(seed int64) Hand {
	rand.Seed(seed)

	hand := make(Hand, MahjongSetSize, MahjongSetSize)
	copy(hand, GetMahjongSet())
	hand2 := make(Hand, 0, 0)
	var j int

	for k := MahjongSetSize; k > MahjongSetSize-HandSize; k-- {
		j = rand.Intn(k)
		hand2 = append(hand2, hand[j])
		hand = append(hand[:j], hand[j+1:]...)
	}

	return hand2
}

type Hand []int

// 牌文字への変換(スペース区切り)
func (hand Hand) HaiString() string {
	// http://qiita.com/ruiu/items/2bb83b29baeae2433a79
	// サイズ0、内部バッファの長さ69の[]byteの値を割り当てる
	b := make([]byte, 0, 70)

	// bに文字列を追加
	for j := 0; j < HandSize; j++ {
		// コードポイント上、普通の麻雀牌はU+1F000からの34個。
		// U+1F000 is 'MAHJONG TILE EAST WIND' ('東')
		// https://codepoints.net/U+1F000
		b = append(b, string(hand[j]+0x1F000)...) // appendするには...が必要
		// 自分のMacではスペース区切りでないとうまく表示されないためスペースを挿入する
		// U+0020 is 'SPACE'
		// https://codepoints.net/U+0020
		b = append(b, string(0x20)...) // appendするには...が必要
	}
	return string(b)
}

// 七対子判定
func (hand Hand) solveChitoitsu() bool {
	//カウンタ
	c := map[int]int{}

	for _, v := range hand {
		count, ok := c[v]
		if ok {
			if count == 1 {
				c[v] = 2
			} else {
				// c[v] == 2
				return false
			}
		} else {
			c[v] = 1
		}
		//8個チェック
		if len(c) >= 8 {
			return false
		}
	}
	return true
}

// あがり判定する
func (hand Hand) Solve() bool {
	return hand.solveChitoitsu() || hand.GroupSuit().Solve()
}

type SuitGroup struct {
	innerSuitGroup
	color int
}

type innerSuitGroup []int

func NewSuitGroup(color int) *SuitGroup {
	s := SuitGroup{innerSuitGroup{}, color}
	return &s
}

type SuitsGroupedHand map[int]SuitGroup

const (
	Jihai = iota
	Manzu
	Sozu
	Pinzu
)

// スート分類
func (hand Hand) GroupSuit() SuitsGroupedHand {
	m := SuitsGroupedHand{
		Jihai: *NewSuitGroup(Jihai),
		Manzu: *NewSuitGroup(Manzu),
		Sozu:  *NewSuitGroup(Sozu),
		Pinzu: *NewSuitGroup(Pinzu),
	}
	for _, i := range hand {
		quo := (i - 7 + 9) / 9
		var mod int
		if i-7 >= 0 {
			mod = (i - 7) % 9
		} else {
			mod = i
		}
		s := m[quo]
		s.append(mod)
		m[quo] = s
	}
	return m
}

func (m SuitsGroupedHand) Solve() bool {
	return m.a_pair_existible() && m.valid_33332()
}
func (m SuitsGroupedHand) a_pair_existible() bool {
	//スートのサイズを3で割った時
	//あまりが2であるスートグループが1つであること
	c := 0
	for _, a := range m {
		switch len(a.list()) % 3 {
		case 0:
			// noop
		case 1:
			return false
		case 2:
			c++
		}
	}
	return c == 1
}

func (m SuitsGroupedHand) valid_33332() bool {
	for i := 0; i < 4; i++ {
		if !m[i].valid_suit_group(i) {
			return false
		}
	}
	return true
}

func (a innerSuitGroup) list() innerSuitGroup {
	return a
}

func (a *innerSuitGroup) SetSuitGroup(b innerSuitGroup) {
	*a = b
}

func (a *innerSuitGroup) append(w int) {
	*a = append(*a, w)
}

// 33332形を形成するスートグループがどうかを判定
func (a SuitGroup) valid_suit_group(i int) bool {
	// 対子が含まれているスートグループがただ1つある前提

	//ソート
	sort.Ints(a.list())
	if len(a.list())%3 == 2 {
		//ペアを探す
		pair_numbers := a.pairable_numbers()
		//ペア候補がなかったらぬける
		if len(pair_numbers) == 0 {
			return false
		}
		//ペア候補毎に繰り返し処理
		for _, v := range pair_numbers {
			//ペアとなる２枚を除去
			rest := NewSuitGroup(i)
			c := 2
			for _, w := range a.list() {
				// ペア候補以外は新スライスに入れる
				// ペア候補は３枚目以降は新スライスに入れる
				if w != v || c <= 0 {
					rest.SetSuitGroup(append(rest.list(), w))
				}
				if w == v {
					c--
				}
			}
			if rest.valid_3cards() {
				return true
			} else {
				continue
			}
		}
		return false
	} else if len(a.list())%3 == 0 {
		return a.valid_3cards()
	}
	// 到達しないはず
	panic("到達しないはず")
}

func (a SuitGroup) valid_3cards() bool {
	// 刻子や順子のみで構成されている場合true
	// a is sorted
	// a.size % 3 is0
	// 引数は字牌のとき0
	for {
		if a.remove_kotsu() {
			continue
		}
		if a.color != Jihai {
			if a.remove_shuntsu() {
				continue
			}
		}
		return len(a.list()) == 0
	}
}

func (a *innerSuitGroup) remove_kotsu() bool {
	// 刻子を除去できればtrue
	// a is sorted
	x := *a
	if len(x) < 3 {
		return false
	}
	if x[0] == x[1] && x[0] == x[2] {
		*a = x[3:]
		return true
	}
	return false
}

func (a *innerSuitGroup) remove_shuntsu() bool {
	// 順子を除去できればtrue
	// a is sorted
	rest := innerSuitGroup{}
	first := -1
	second := -1
	found := false
	for _, v := range *a {
		if found {
			rest = append(rest, v)
			continue
		}
		if first == -1 {
			first = v
		} else if second == -1 && first+1 == v {
			second = v
		} else if second != -1 && first+2 == v {
			//flush
			first = -1
			second = -1
			found = true
		} else {
			rest = append(rest, v)
		}
	}
	*a = rest
	return found
}

func (a innerSuitGroup) pairable_numbers() innerSuitGroup {
	// a is sorted
	counter := []int{}
	x := 999 // 2つ前
	y := 999 // 1つ前
	for _, v := range a {
		if y == v && x != v {
			counter = append(counter, v)
		} else {
			y = v
		}
	}
	return counter
}
<commit_msg>Refactor<commit_after>package tenho

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

func Start() {
	start := time.Now().UnixNano()
	// 処理
	var i float64
	i = 0
	for {
		i++
		end := time.Now().UnixNano()
		diff := float64(end-start) / 1000000000
		m := i / diff
		out := 0
		if m >= 0 {
			out = int(m)
		}
		seed := time.Now().UnixNano()
		var hai string
		var ok bool
		hai, ok = tryOnce(seed)
		if int(i)%10000 == 0 {
			fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
		}
		if ok {
			fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
			break
		}
		//fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
		//if i > 100000 {
		//	break
		//}
	}
	fmt.Printf("\n")
}

func tryOnce(seed int64) (string, bool) {
	hand := ShuffledHand(seed)
	hai := hand.HaiString()
	ok := hand.Solve()
	return hai, ok
}

// http://d.hatena.ne.jp/hake/20150930/p1
func shuffle(hand Hand) {
	for i := len(hand); i > 1; i-- {
		j := rand.Intn(i) // 0 .. i-1 の乱数発生
		hand[i-1], hand[j] = hand[j], hand[i-1]
	}
}

const HandSize = 14
const MahjongSetSize = 136

var defaultSet []int

func GetMahjongSet() []int {
	if defaultSet == nil {
		size := MahjongSetSize
		defaultSet = make([]int, size, size)
		for i := 0; i < size; i++ {
			defaultSet[i] = i / 4
		}
	}
	return defaultSet
}

func ShuffledHand(seed int64) Hand {
	rand.Seed(seed)

	hand := make(Hand, MahjongSetSize, MahjongSetSize)
	copy(hand, GetMahjongSet())
	hand2 := make(Hand, 0, 0)
	var j int

	for k := MahjongSetSize; k > MahjongSetSize-HandSize; k-- {
		j = rand.Intn(k)
		hand2 = append(hand2, hand[j])
		hand = append(hand[:j], hand[j+1:]...)
	}

	return hand2
}

type Hand []int

// 牌文字への変換(スペース区切り)
func (hand Hand) HaiString() string {
	// http://qiita.com/ruiu/items/2bb83b29baeae2433a79
	// サイズ0、内部バッファの長さ69の[]byteの値を割り当てる
	b := make([]byte, 0, 70)

	// bに文字列を追加
	for j := 0; j < HandSize; j++ {
		// コードポイント上、普通の麻雀牌はU+1F000からの34個。
		// U+1F000 is 'MAHJONG TILE EAST WIND' ('東')
		// https://codepoints.net/U+1F000
		b = append(b, string(hand[j]+0x1F000)...) // appendするには...が必要
		// 自分のMacではスペース区切りでないとうまく表示されないためスペースを挿入する
		// U+0020 is 'SPACE'
		// https://codepoints.net/U+0020
		b = append(b, string(0x20)...) // appendするには...が必要
	}
	return string(b)
}

// 七対子判定
func (hand Hand) solveChitoitsu() bool {
	//カウンタ
	c := map[int]int{}

	for _, v := range hand {
		count, ok := c[v]
		if ok {
			if count == 1 {
				c[v] = 2
			} else {
				// c[v] == 2
				return false
			}
		} else {
			c[v] = 1
		}
		//8個チェック
		if len(c) >= 8 {
			return false
		}
	}
	return true
}

// あがり判定する
func (hand Hand) Solve() bool {
	return hand.solveChitoitsu() || hand.GroupSuit().Solve()
}

type SuitGroup struct {
	innerSuitGroup
	color int
}

type innerSuitGroup []int

func NewSuitGroup(color int) *SuitGroup {
	s := SuitGroup{innerSuitGroup{}, color}
	return &s
}

type SuitsGroupedHand map[int]SuitGroup

const (
	Jihai = iota
	Manzu
	Sozu
	Pinzu
)

// スート分類
func (hand Hand) GroupSuit() SuitsGroupedHand {
	m := SuitsGroupedHand{
		Jihai: *NewSuitGroup(Jihai),
		Manzu: *NewSuitGroup(Manzu),
		Sozu:  *NewSuitGroup(Sozu),
		Pinzu: *NewSuitGroup(Pinzu),
	}
	for _, i := range hand {
		quo := (i - 7 + 9) / 9
		var mod int
		if i-7 >= 0 {
			mod = (i - 7) % 9
		} else {
			mod = i
		}
		s := m[quo]
		s.append(mod)
		m[quo] = s
	}
	return m
}

func (m SuitsGroupedHand) Solve() bool {
	return m.a_pair_existible() && m.valid_33332()
}
func (m SuitsGroupedHand) a_pair_existible() bool {
	//スートのサイズを3で割った時
	//あまりが2であるスートグループが1つであること
	c := 0
	for _, a := range m {
		switch len(a.list()) % 3 {
		case 0:
			// noop
		case 1:
			return false
		case 2:
			c++
		}
	}
	return c == 1
}

func (m SuitsGroupedHand) valid_33332() bool {
	for i := 0; i < 4; i++ {
		if !m[i].valid_suit_group(i) {
			return false
		}
	}
	return true
}

func (a innerSuitGroup) list() innerSuitGroup {
	return a
}

func (a *innerSuitGroup) SetSuitGroup(b innerSuitGroup) {
	*a = b
}

func (a *innerSuitGroup) append(w int) {
	*a = append(*a, w)
}

// 33332形を形成するスートグループがどうかを判定
func (a SuitGroup) valid_suit_group(i int) bool {
	// 対子が含まれているスートグループがただ1つある前提

	//ソート
	sort.Ints(a.list())
	if len(a.list())%3 == 2 {
		//ペアを探す
		pair_numbers := a.pairable_numbers()
		//ペア候補がなかったらぬける
		if len(pair_numbers) == 0 {
			return false
		}
		//ペア候補毎に繰り返し処理
		for _, v := range pair_numbers {
			//ペアとなる２枚を除去
			rest := NewSuitGroup(i)
			c := 2
			for _, w := range a.list() {
				// ペア候補以外は新スライスに入れる
				// ペア候補は３枚目以降は新スライスに入れる
				if w != v || c <= 0 {
					rest.SetSuitGroup(append(rest.list(), w))
				}
				if w == v {
					c--
				}
			}
			if rest.valid_3cards() {
				return true
			} else {
				continue
			}
		}
		return false
	} else if len(a.list())%3 == 0 {
		return a.valid_3cards()
	}
	// 到達しないはず
	panic("到達しないはず")
}

func (a SuitGroup) valid_3cards() bool {
	// 刻子や順子のみで構成されている場合true
	// a is sorted
	// a.size % 3 is0
	// 引数は字牌のとき0
	for {
		if a.remove_kotsu() {
			continue
		}
		if a.color != Jihai {
			if a.remove_shuntsu() {
				continue
			}
		}
		return len(a.list()) == 0
	}
}

func (a *innerSuitGroup) remove_kotsu() bool {
	// 刻子を除去できればtrue
	// a is sorted
	x := *a
	if len(x) < 3 {
		return false
	}
	if x[0] == x[1] && x[0] == x[2] {
		*a = x[3:]
		return true
	}
	return false
}

func (a *innerSuitGroup) remove_shuntsu() bool {
	// 順子を除去できればtrue
	// a is sorted
	rest := innerSuitGroup{}
	first := -1
	second := -1
	found := false
	for _, v := range *a {
		if found {
			rest = append(rest, v)
			continue
		}
		if first == -1 {
			first = v
		} else if second == -1 && first+1 == v {
			second = v
		} else if second != -1 && first+2 == v {
			//flush
			first = -1
			second = -1
			found = true
		} else {
			rest = append(rest, v)
		}
	}
	*a = rest
	return found
}

func (a innerSuitGroup) pairable_numbers() innerSuitGroup {
	// a is sorted
	counter := []int{}
	var x, y int // 2つ前と1つ前
	for _, v := range a {
		if y == v && x != v {
			counter = append(counter, v)
		} else {
			y = v
		}
	}
	return counter
}
<|endoftext|>