<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package ssh

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/unknwon/com"
	"golang.org/x/crypto/ssh"
	log "unknwon.dev/clog/v2"

	"gogs.io/gogs/internal/conf"
	"gogs.io/gogs/internal/db"
)

func cleanCommand(cmd string) string {
	i := strings.Index(cmd, "git")
	if i == -1 {
		return cmd
	}
	return cmd[i:]
}

func handleServerConn(keyID string, chans <-chan ssh.NewChannel) {
	for newChan := range chans {
		if newChan.ChannelType() != "session" {
			_ = newChan.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}

		ch, reqs, err := newChan.Accept()
		if err != nil {
			log.Error("Error accepting channel: %v", err)
			continue
		}

		go func(in <-chan *ssh.Request) {
			defer func() {
				_ = ch.Close()
			}()
			for req := range in {
				payload := cleanCommand(string(req.Payload))
				switch req.Type {
				case "env":
					args := strings.Split(strings.Replace(payload, "\x00", "", -1), "\v")
					if len(args) != 2 {
						log.Warn("SSH: Invalid env arguments: '%#v'", args)
						continue
					}
					args[0] = strings.TrimLeft(args[0], "\x04")

					// Sometimes the client could send malformed command (i.e. missing "="),
					// see https://discuss.gogs.io/t/ssh/3106.
					if args[0] == "" {
						continue
					}

					_, stderr, err := com.ExecCmd("env", args[0]+"="+args[1])
					if err != nil {
						log.Error("env: %v - %s", err, stderr)
						return
					}

				case "exec":
					cmdName := strings.TrimLeft(payload, "'()")
					log.Trace("SSH: Payload: %v", cmdName)

					args := []string{"serv", "key-" + keyID, "--config=" + conf.CustomConf}
					log.Trace("SSH: Arguments: %v", args)
					cmd := exec.Command(conf.AppPath(), args...)
					cmd.Env = append(os.Environ(), "SSH_ORIGINAL_COMMAND="+cmdName)

					stdout, err := cmd.StdoutPipe()
					if err != nil {
						log.Error("SSH: StdoutPipe: %v", err)
						return
					}
					stderr, err := cmd.StderrPipe()
					if err != nil {
						log.Error("SSH: StderrPipe: %v", err)
						return
					}
					input, err := cmd.StdinPipe()
					if err != nil {
						log.Error("SSH: StdinPipe: %v", err)
						return
					}

					// FIXME: check timeout
					if err = cmd.Start(); err != nil {
						log.Error("SSH: Start: %v", err)
						return
					}

					_ = req.Reply(true, nil)
					go func() {
						_, _ = io.Copy(input, ch)
					}()
					_, _ = io.Copy(ch, stdout)
					_, _ = io.Copy(ch.Stderr(), stderr)

					if err = cmd.Wait(); err != nil {
						log.Error("SSH: Wait: %v", err)
						return
					}

					_, _ = ch.SendRequest("exit-status", false, []byte{0, 0, 0, 0})
					return
				default:
				}
			}
		}(reqs)
	}
}

func listen(config *ssh.ServerConfig, host string, port int) {
	listener, err := net.Listen("tcp", host+":"+com.ToStr(port))
	if err != nil {
		log.Fatal("Failed to start SSH server: %v", err)
	}
	for {
		// Once a ServerConfig has been configured, connections can be accepted.
		conn, err := listener.Accept()
		if err != nil {
			log.Error("SSH: Error accepting incoming connection: %v", err)
			continue
		}

		// Before use, a handshake must be performed on the incoming net.Conn.
		// It must be handled in a separate goroutine,
		// otherwise one user could easily block entire loop.
		// For example, user could be asked to trust server key fingerprint and hangs.
		go func() {
			log.Trace("SSH: Handshaking for %s", conn.RemoteAddr())
			sConn, chans, reqs, err := ssh.NewServerConn(conn, config)
			if err != nil {
				if err == io.EOF {
					log.Warn("SSH: Handshaking was terminated: %v", err)
				} else {
					log.Error("SSH: Error on handshaking: %v", err)
				}
				return
			}

			log.Trace("SSH: Connection from %s (%s)", sConn.RemoteAddr(), sConn.ClientVersion())
			// The incoming Request channel must be serviced.
			go ssh.DiscardRequests(reqs)
			go handleServerConn(sConn.Permissions.Extensions["key-id"], chans)
		}()
	}
}

// Listen starts a SSH server listens on given port.
func Listen(host string, port int, ciphers []string) {
	config := &ssh.ServerConfig{
		Config: ssh.Config{
			Ciphers: ciphers,
		},
		PublicKeyCallback: func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
			pkey, err := db.SearchPublicKeyByContent(strings.TrimSpace(string(ssh.MarshalAuthorizedKey(key))))
			if err != nil {
				log.Error("SearchPublicKeyByContent: %v", err)
				return nil, err
			}
			return &ssh.Permissions{Extensions: map[string]string{"key-id": com.ToStr(pkey.ID)}}, nil
		},
	}

	keyPath := filepath.Join(conf.Server.AppDataPath, "ssh", "gogs.rsa")
	if !com.IsExist(keyPath) {
		if err := os.MkdirAll(filepath.Dir(keyPath), os.ModePerm); err != nil {
			panic(err)
		}
		_, stderr, err := com.ExecCmd(conf.SSH.KeygenPath, "-f", keyPath, "-t", "rsa", "-m", "PEM", "-N", "")
		if err != nil {
			panic(fmt.Sprintf("Failed to generate private key: %v - %s", err, stderr))
		}
		log.Trace("SSH: New private key is generateed: %s", keyPath)
	}

	privateBytes, err := ioutil.ReadFile(keyPath)
	if err != nil {
		panic("SSH: Failed to load private key: " + err.Error())
	}
	private, err := ssh.ParsePrivateKey(privateBytes)
	if err != nil {
		panic("SSH: Failed to parse private key: " + err.Error())
	}
	config.AddHostKey(private)

	go listen(config, host, port)
}
<commit_msg>ssh: improve env command processing (#6095)<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package ssh

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/unknwon/com"
	"golang.org/x/crypto/ssh"
	log "unknwon.dev/clog/v2"

	"gogs.io/gogs/internal/conf"
	"gogs.io/gogs/internal/db"
)

func cleanCommand(cmd string) string {
	i := strings.Index(cmd, "git")
	if i == -1 {
		return cmd
	}
	return cmd[i:]
}

func handleServerConn(keyID string, chans <-chan ssh.NewChannel) {
	for newChan := range chans {
		if newChan.ChannelType() != "session" {
			_ = newChan.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}

		ch, reqs, err := newChan.Accept()
		if err != nil {
			log.Error("Error accepting channel: %v", err)
			continue
		}

		go func(in <-chan *ssh.Request) {
			defer func() {
				_ = ch.Close()
			}()
			for req := range in {
				payload := cleanCommand(string(req.Payload))
				switch req.Type {
				case "env":
					var env struct {
						Name  string
						Value string
					}
					if err := ssh.Unmarshal(req.Payload, &env); err != nil {
						log.Warn("SSH: Invalid env payload %q: %v", req.Payload, err)
						continue
					}
					// Sometimes the client could send malformed command (i.e. missing "="),
					// see https://discuss.gogs.io/t/ssh/3106.
					if env.Name == "" || env.Value == "" {
						log.Warn("SSH: Invalid env arguments: %+v", env)
						continue
					}

					_, stderr, err := com.ExecCmd("env", fmt.Sprintf("%s=%s", env.Name, env.Value))
					if err != nil {
						log.Error("env: %v - %s", err, stderr)
						return
					}

				case "exec":
					cmdName := strings.TrimLeft(payload, "'()")
					log.Trace("SSH: Payload: %v", cmdName)

					args := []string{"serv", "key-" + keyID, "--config=" + conf.CustomConf}
					log.Trace("SSH: Arguments: %v", args)
					cmd := exec.Command(conf.AppPath(), args...)
					cmd.Env = append(os.Environ(), "SSH_ORIGINAL_COMMAND="+cmdName)

					stdout, err := cmd.StdoutPipe()
					if err != nil {
						log.Error("SSH: StdoutPipe: %v", err)
						return
					}
					stderr, err := cmd.StderrPipe()
					if err != nil {
						log.Error("SSH: StderrPipe: %v", err)
						return
					}
					input, err := cmd.StdinPipe()
					if err != nil {
						log.Error("SSH: StdinPipe: %v", err)
						return
					}

					// FIXME: check timeout
					if err = cmd.Start(); err != nil {
						log.Error("SSH: Start: %v", err)
						return
					}

					_ = req.Reply(true, nil)
					go func() {
						_, _ = io.Copy(input, ch)
					}()
					_, _ = io.Copy(ch, stdout)
					_, _ = io.Copy(ch.Stderr(), stderr)

					if err = cmd.Wait(); err != nil {
						log.Error("SSH: Wait: %v", err)
						return
					}

					_, _ = ch.SendRequest("exit-status", false, []byte{0, 0, 0, 0})
					return
				default:
				}
			}
		}(reqs)
	}
}

func listen(config *ssh.ServerConfig, host string, port int) {
	listener, err := net.Listen("tcp", host+":"+com.ToStr(port))
	if err != nil {
		log.Fatal("Failed to start SSH server: %v", err)
	}
	for {
		// Once a ServerConfig has been configured, connections can be accepted.
		conn, err := listener.Accept()
		if err != nil {
			log.Error("SSH: Error accepting incoming connection: %v", err)
			continue
		}

		// Before use, a handshake must be performed on the incoming net.Conn.
		// It must be handled in a separate goroutine,
		// otherwise one user could easily block entire loop.
		// For example, user could be asked to trust server key fingerprint and hangs.
		go func() {
			log.Trace("SSH: Handshaking for %s", conn.RemoteAddr())
			sConn, chans, reqs, err := ssh.NewServerConn(conn, config)
			if err != nil {
				if err == io.EOF {
					log.Warn("SSH: Handshaking was terminated: %v", err)
				} else {
					log.Error("SSH: Error on handshaking: %v", err)
				}
				return
			}

			log.Trace("SSH: Connection from %s (%s)", sConn.RemoteAddr(), sConn.ClientVersion())
			// The incoming Request channel must be serviced.
			go ssh.DiscardRequests(reqs)
			go handleServerConn(sConn.Permissions.Extensions["key-id"], chans)
		}()
	}
}

// Listen starts a SSH server listens on given port.
func Listen(host string, port int, ciphers []string) {
	config := &ssh.ServerConfig{
		Config: ssh.Config{
			Ciphers: ciphers,
		},
		PublicKeyCallback: func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
			pkey, err := db.SearchPublicKeyByContent(strings.TrimSpace(string(ssh.MarshalAuthorizedKey(key))))
			if err != nil {
				log.Error("SearchPublicKeyByContent: %v", err)
				return nil, err
			}
			return &ssh.Permissions{Extensions: map[string]string{"key-id": com.ToStr(pkey.ID)}}, nil
		},
	}

	keyPath := filepath.Join(conf.Server.AppDataPath, "ssh", "gogs.rsa")
	if !com.IsExist(keyPath) {
		if err := os.MkdirAll(filepath.Dir(keyPath), os.ModePerm); err != nil {
			panic(err)
		}
		_, stderr, err := com.ExecCmd(conf.SSH.KeygenPath, "-f", keyPath, "-t", "rsa", "-m", "PEM", "-N", "")
		if err != nil {
			panic(fmt.Sprintf("Failed to generate private key: %v - %s", err, stderr))
		}
		log.Trace("SSH: New private key is generateed: %s", keyPath)
	}

	privateBytes, err := ioutil.ReadFile(keyPath)
	if err != nil {
		panic("SSH: Failed to load private key: " + err.Error())
	}
	private, err := ssh.ParsePrivateKey(privateBytes)
	if err != nil {
		panic("SSH: Failed to parse private key: " + err.Error())
	}
	config.AddHostKey(private)

	go listen(config, host, port)
}
<|endoftext|>


<commit_before>package atbash

import "testing"

var tests = []struct {
	expected string
	s        string
}{
	{"ml", "no"},
	{"ml", "no"},
	{"bvh", "yes"},
	{"lnt", "OMG"},
	{"lnt", "O M G"},
	{"nrmwy oldrm tob", "mindblowingly"},
	{"gvhgr mt123 gvhgr mt", "Testing, 1 2 3, testing."},
	{"gifgs rhurx grlm", "Truth is fiction."},
	{"gsvjf rxpyi ldmul cqfnk hlevi gsvoz abwlt", "The quick brown fox jumps over the lazy dog."},
}

func TestAtbash(t *testing.T) {
	for _, test := range tests {
		actual := Atbash(test.s)
		if actual != test.expected {
			t.Errorf("Atbash(%s): expected %s, actual %s", test.s, test.expected, actual)
		}
	}
}
<commit_msg>Added benchmark in go/atbash-cipher<commit_after>package atbash

import "testing"

var tests = []struct {
	expected string
	s        string
}{
	{"ml", "no"},
	{"ml", "no"},
	{"bvh", "yes"},
	{"lnt", "OMG"},
	{"lnt", "O M G"},
	{"nrmwy oldrm tob", "mindblowingly"},
	{"gvhgr mt123 gvhgr mt", "Testing, 1 2 3, testing."},
	{"gifgs rhurx grlm", "Truth is fiction."},
	{"gsvjf rxpyi ldmul cqfnk hlevi gsvoz abwlt", "The quick brown fox jumps over the lazy dog."},
}

func TestAtbash(t *testing.T) {
	for _, test := range tests {
		actual := Atbash(test.s)
		if actual != test.expected {
			t.Errorf("Atbash(%s): expected %s, actual %s", test.s, test.expected, actual)
		}
	}
}

func BenchmarkAtbash(b *testing.B) {
	b.StopTimer()
	for _, test := range tests {
		b.StartTimer()

		for i := 0; i < b.N; i++ {
			Atbash(test.s)
		}

		b.StopTimer()
	}
}
<|endoftext|>


<commit_before>package vault

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
)

func NewRecipient(recipient string) (*VaultRecipient, error) {
	args := strings.Split(recipient, ":")

	if len(args) != 2 {
		return nil, fmt.Errorf("\nInvalid format \"%s\"\nRecipient should be in the form of fingerprint:name\n", recipient)
	}

	if len(args[0]) == 0 || len(args[1]) == 0 {
		return nil, fmt.Errorf("\nInvalid format \"%s\"\nRecipient should be in the form of fingerprint:name\n", recipient)
	}

	recipientFingerprint := strings.Split(recipient, ":")[0]
	recipientFingerprint = strings.Replace(recipientFingerprint, " ", "", -1)

	if hexFingerprint, err := hex.DecodeString(recipientFingerprint); err != nil {
		return nil, fmt.Errorf("\nSupplied fingerprint \"%s\" does not have the correct format\n", recipientFingerprint)
	} else {

		if len(hexFingerprint) != 16 && len(hexFingerprint) != 20 {
			return nil, fmt.Errorf("\nSupplied fingerprint \"%s\" does not have the correct size\n", hexFingerprint)
		}

	}

	recipientName := strings.Split(recipient, ":")[1]

	return &VaultRecipient{Fingerprint: recipientFingerprint, Name: recipientName}, nil
}

func LoadVaultfile() (*Vaultfile, error) {
	return loadVaultfileRecursive(GetHomeDir())
}

func loadVaultfileRecursive(currentPath string) (*Vaultfile, error) {
	v := &Vaultfile{}
	if currentPath == path.Dir(currentPath) {
		return v, nil
	}

	content, err := ioutil.ReadFile(path.Join(currentPath, "Vaultfile"))

	if os.IsNotExist(err) {
		return loadVaultfileRecursive(path.Dir(currentPath))
	} else if err != nil {
		fmt.Println(err.(*os.PathError))
		return v, nil
	}
	if err := json.Unmarshal(content, v); err != nil {
		return nil, fmt.Errorf("\nCouldn't read Vaultfile.\n")
	} else {
		return v, nil
	}

}

type VaultRecipient struct {
	Name        string
	Fingerprint string
}

func (v VaultRecipient) ToString() string {
	return v.Fingerprint + ":" + v.Name
}

type Vaultfile struct {
	Recipients []VaultRecipient
}

func (v Vaultfile) Save() error {
	js, err := json.MarshalIndent(v, "", "  ")
	if err != nil {
		return err
	}

	err2 := ioutil.WriteFile(path.Join(GetHomeDir(), "Vaultfile"), js, 0644)
	if err2 != nil {
		return err2
	}
	return nil
}
<commit_msg>Print correct fingerprint on error msg<commit_after>package vault

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
)

func NewRecipient(recipient string) (*VaultRecipient, error) {
	args := strings.Split(recipient, ":")

	if len(args) != 2 {
		return nil, fmt.Errorf("\nInvalid format \"%s\"\nRecipient should be in the form of fingerprint:name\n", recipient)
	}

	if len(args[0]) == 0 || len(args[1]) == 0 {
		return nil, fmt.Errorf("\nInvalid format \"%s\"\nRecipient should be in the form of fingerprint:name\n", recipient)
	}

	recipientFingerprint := strings.Split(recipient, ":")[0]
	recipientFingerprint = strings.Replace(recipientFingerprint, " ", "", -1)

	if hexFingerprint, err := hex.DecodeString(recipientFingerprint); err != nil {
		return nil, fmt.Errorf("\nSupplied fingerprint \"%s\" does not have the correct format\n", recipientFingerprint)
	} else {

		if len(hexFingerprint) != 16 && len(hexFingerprint) != 20 {
			return nil, fmt.Errorf("\nSupplied fingerprint \"%s\" does not have the correct size\n", recipientFingerprint)
		}

	}

	recipientName := strings.Split(recipient, ":")[1]

	return &VaultRecipient{Fingerprint: recipientFingerprint, Name: recipientName}, nil
}

func LoadVaultfile() (*Vaultfile, error) {
	return loadVaultfileRecursive(GetHomeDir())
}

func loadVaultfileRecursive(currentPath string) (*Vaultfile, error) {
	v := &Vaultfile{}
	if currentPath == path.Dir(currentPath) {
		return v, nil
	}

	content, err := ioutil.ReadFile(path.Join(currentPath, "Vaultfile"))

	if os.IsNotExist(err) {
		return loadVaultfileRecursive(path.Dir(currentPath))
	} else if err != nil {
		fmt.Println(err.(*os.PathError))
		return v, nil
	}
	if err := json.Unmarshal(content, v); err != nil {
		return nil, fmt.Errorf("\nCouldn't read Vaultfile.\n")
	} else {
		return v, nil
	}

}

type VaultRecipient struct {
	Name        string
	Fingerprint string
}

func (v VaultRecipient) ToString() string {
	return v.Fingerprint + ":" + v.Name
}

type Vaultfile struct {
	Recipients []VaultRecipient
}

func (v Vaultfile) Save() error {
	js, err := json.MarshalIndent(v, "", "  ")
	if err != nil {
		return err
	}

	err2 := ioutil.WriteFile(path.Join(GetHomeDir(), "Vaultfile"), js, 0644)
	if err2 != nil {
		return err2
	}
	return nil
}
<|endoftext|>


<commit_before>package renter

import (
	"io"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// When a file contract is within this many blocks of expiring, the renter
	// will attempt to reupload the data covered by the contract.
	renewThreshold = 2000

	hostTimeout = 15 * time.Second
)

// chunkHosts returns the IPs of the hosts storing a given chunk.
func (f *file) chunkHosts(index uint64) []modules.NetAddress {
	var hosts []modules.NetAddress
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			if p.Chunk == index {
				hosts = append(hosts, fc.IP)
				break
			}
		}
	}
	return hosts
}

// removeExpiredContracts deletes contracts in the file object that have
// expired.
func (f *file) removeExpiredContracts(currentHeight types.BlockHeight) {
	f.mu.Lock()
	defer f.mu.Unlock()
	var expired []types.FileContractID
	for id, fc := range f.contracts {
		if currentHeight >= fc.WindowStart {
			expired = append(expired, id)
		}
	}
	for _, id := range expired {
		delete(f.contracts, id)
	}
}

// incompleteChunks returns a map of chunks in need of repair.
func (f *file) incompleteChunks() map[uint64][]uint64 {
	present := make([][]bool, f.numChunks())
	for i := range present {
		present[i] = make([]bool, f.erasureCode.NumPieces())
	}
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			present[p.Chunk][p.Piece] = true
		}
	}
	incomplete := make(map[uint64][]uint64)
	for chunkIndex, pieceBools := range present {
		for pieceIndex, ok := range pieceBools {
			if !ok {
				incomplete[uint64(chunkIndex)] = append(incomplete[uint64(chunkIndex)], uint64(pieceIndex))
			}
		}
	}
	return incomplete
}

// expiringChunks returns a map of chunks whose pieces will expire within
// 'renewThreshold' blocks.
func (f *file) expiringChunks(currentHeight types.BlockHeight) map[uint64][]uint64 {
	expiring := make(map[uint64][]uint64)
	for _, fc := range f.contracts {
		if currentHeight >= fc.WindowStart-renewThreshold {
			// mark every piece in the chunk
			for _, p := range fc.Pieces {
				expiring[p.Chunk] = append(expiring[p.Chunk], p.Piece)
			}
		}
	}
	return expiring
}

// offlineChunks returns a map of chunks whose pieces are not
// immediately available for download.
func (f *file) offlineChunks() map[uint64][]uint64 {
	offline := make(map[uint64][]uint64)
	var mapLock sync.Mutex
	var wg sync.WaitGroup
	wg.Add(len(f.contracts))
	for _, fc := range f.contracts {
		go func(fc fileContract) {
			defer wg.Done()
			conn, err := net.DialTimeout("tcp", string(fc.IP), hostTimeout)
			if err == nil {
				conn.Close()
				return
			}
			// host did not respond in time; mark all pieces as offline
			mapLock.Lock()
			for _, p := range fc.Pieces {
				offline[p.Chunk] = append(offline[p.Chunk], p.Piece)
			}
			mapLock.Unlock()
		}(fc)
	}
	wg.Wait()
	return offline
}

// repair attempts to repair a file by uploading missing pieces to more hosts.
func (f *file) repair(r io.ReaderAt, pieceMap map[uint64][]uint64, hosts []uploader) error {
	// For each chunk with missing pieces, re-encode the chunk and upload each
	// missing piece.
	var wg sync.WaitGroup
	for chunkIndex, missingPieces := range pieceMap {
		// can only upload to hosts that aren't already storing this chunk
		// TODO: what if we're renewing?
		// 	curHosts := f.chunkHosts(chunkIndex)
		// 	var newHosts []uploader
		// outer:
		// 	for _, h := range hosts {
		// 		for _, ip := range curHosts {
		// 			if ip == h.addr() {

		// 				continue outer
		// 			}
		// 		}
		// 		newHosts = append(newHosts, h)
		// 	}
		newHosts := hosts
		// don't bother encoding if there aren't any hosts to upload to
		if len(newHosts) == 0 {
			continue
		}

		// read chunk data and encode
		chunk := make([]byte, f.chunkSize())
		_, err := r.ReadAt(chunk, int64(chunkIndex*f.chunkSize()))
		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
			return err
		}
		pieces, err := f.erasureCode.Encode(chunk)
		if err != nil {
			return err
		}

		// upload pieces, split evenly among hosts
		wg.Add(len(missingPieces))
		for j, pieceIndex := range missingPieces {
			host := newHosts[j%len(newHosts)]
			up := uploadPiece{pieces[pieceIndex], chunkIndex, pieceIndex}
			//go func(host uploader, up uploadPiece) {
			_ = host.addPiece(up)
			wg.Done()
			//}(host, up)
		}
		wg.Wait()

		// update contracts
		f.mu.Lock()
		for _, h := range hosts {
			contract := h.fileContract()
			f.contracts[contract.ID] = contract
		}
		f.mu.Unlock()
	}

	return nil
}

// threadedRepairUploads improves the health of files tracked by the renter by
// reuploading their missing pieces. Multiple repair attempts may be necessary
// before the file reaches full redundancy.
func (r *Renter) threadedRepairUploads() {
	// a primitive blacklist is used to augment the hostdb's weights. Each
	// negotiation failure increments the integer, and the probability of
	// selecting the host for upload is 1/n.
	blacklist := make(map[modules.NetAddress]int)

	for {
		time.Sleep(5 * time.Second)

		if !r.wallet.Unlocked() {
			continue
		}

		// make copy of repair set under lock
		repairing := make(map[string]string)
		id := r.mu.RLock()
		for name, path := range r.repairSet {
			repairing[name] = path
		}
		r.mu.RUnlock(id)

		for name, path := range repairing {
			// retrieve file object and get current height
			id = r.mu.RLock()
			f, ok := r.files[name]
			//height := r.blockHeight
			r.mu.RUnlock(id)
			if !ok {
				r.log.Printf("failed to repair %v: no longer tracking that file", name)
				id = r.mu.Lock()
				delete(r.repairSet, name)
				r.mu.Unlock(id)
				continue
			}

			// delete any expired contracts
			//f.removeExpiredContracts(height)

			// determine file health
			badChunks := f.incompleteChunks()
			if len(badChunks) == 0 {
				//badChunks = f.expiringChunks(height)
				// if len(badChunks) == 0 {
				// 	// nothing to do
				// 	continue
				// }
				continue
			}

			r.log.Printf("repairing %v chunks of %v", len(badChunks), name)

			// open file handle
			handle, err := os.Open(path)
			if err != nil {
				r.log.Printf("failed to repair %v: %v", name, err)
				id = r.mu.Lock()
				delete(r.repairSet, name)
				r.mu.Unlock(id)
				continue
			}

			// build host list
			bytesPerHost := f.pieceSize * f.numChunks()
			var hosts []uploader
			randHosts := r.hostDB.RandomHosts(f.erasureCode.NumPieces() * 2)
			for _, h := range randHosts {
				// probabilistically filter out known bad hosts
				// unresponsive hosts will be selected with probability 1/(1+nFailures)
				nFailures, ok := blacklist[h.IPAddress]
				if n, _ := crypto.RandIntn(1 + nFailures); ok && n != 0 {
					continue
				}

				// TODO: use smarter duration
				hostUploader, err := r.newHostUploader(h, bytesPerHost, defaultDuration, f.masterKey)
				if err != nil {
					// penalize unresponsive hosts
					if strings.Contains(err.Error(), "timeout") {
						blacklist[h.IPAddress]++
					}
					continue
				}
				hosts = append(hosts, hostUploader)
				if len(hosts) >= f.erasureCode.NumPieces() {
					break
				}
			}

			if len(hosts) < f.erasureCode.MinPieces() {
				r.log.Printf("failed to repair %v: not enough hosts", name)
				continue
			}

			err = f.repair(handle, badChunks, hosts)
			if err != nil {
				r.log.Printf("failed to repair %v: %v", name, err)
				id = r.mu.Lock()
				delete(r.repairSet, name)
				r.mu.Unlock(id)
			}

			// save the repaired file data
			err = r.saveFile(f)
			if err != nil {
				// definitely bad, but we probably shouldn't delete from the
				// repair set if this happens
				r.log.Printf("failed to save repaired file %v: %v", name, err)
			}

			// close the file
			handle.Close()

			// close the host connections
			for i := range hosts {
				hosts[i].(*hostUploader).Close()
			}
		}

	}
}
<commit_msg>simplify repair loop structure<commit_after>package renter

import (
	"io"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// When a file contract is within this many blocks of expiring, the renter
	// will attempt to reupload the data covered by the contract.
	renewThreshold = 2000

	hostTimeout = 15 * time.Second
)

// chunkHosts returns the IPs of the hosts storing a given chunk.
func (f *file) chunkHosts(index uint64) []modules.NetAddress {
	var hosts []modules.NetAddress
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			if p.Chunk == index {
				hosts = append(hosts, fc.IP)
				break
			}
		}
	}
	return hosts
}

// removeExpiredContracts deletes contracts in the file object that have
// expired.
func (f *file) removeExpiredContracts(currentHeight types.BlockHeight) {
	f.mu.Lock()
	defer f.mu.Unlock()
	var expired []types.FileContractID
	for id, fc := range f.contracts {
		if currentHeight >= fc.WindowStart {
			expired = append(expired, id)
		}
	}
	for _, id := range expired {
		delete(f.contracts, id)
	}
}

// incompleteChunks returns a map of chunks in need of repair.
func (f *file) incompleteChunks() map[uint64][]uint64 {
	present := make([][]bool, f.numChunks())
	for i := range present {
		present[i] = make([]bool, f.erasureCode.NumPieces())
	}
	for _, fc := range f.contracts {
		for _, p := range fc.Pieces {
			present[p.Chunk][p.Piece] = true
		}
	}
	incomplete := make(map[uint64][]uint64)
	for chunkIndex, pieceBools := range present {
		for pieceIndex, ok := range pieceBools {
			if !ok {
				incomplete[uint64(chunkIndex)] = append(incomplete[uint64(chunkIndex)], uint64(pieceIndex))
			}
		}
	}
	return incomplete
}

// expiringChunks returns a map of chunks whose pieces will expire within
// 'renewThreshold' blocks.
func (f *file) expiringChunks(currentHeight types.BlockHeight) map[uint64][]uint64 {
	expiring := make(map[uint64][]uint64)
	for _, fc := range f.contracts {
		if currentHeight >= fc.WindowStart-renewThreshold {
			// mark every piece in the chunk
			for _, p := range fc.Pieces {
				expiring[p.Chunk] = append(expiring[p.Chunk], p.Piece)
			}
		}
	}
	return expiring
}

// offlineChunks returns a map of chunks whose pieces are not
// immediately available for download.
func (f *file) offlineChunks() map[uint64][]uint64 {
	offline := make(map[uint64][]uint64)
	var mapLock sync.Mutex
	var wg sync.WaitGroup
	wg.Add(len(f.contracts))
	for _, fc := range f.contracts {
		go func(fc fileContract) {
			defer wg.Done()
			conn, err := net.DialTimeout("tcp", string(fc.IP), hostTimeout)
			if err == nil {
				conn.Close()
				return
			}
			// host did not respond in time; mark all pieces as offline
			mapLock.Lock()
			for _, p := range fc.Pieces {
				offline[p.Chunk] = append(offline[p.Chunk], p.Piece)
			}
			mapLock.Unlock()
		}(fc)
	}
	wg.Wait()
	return offline
}

// repair attempts to repair a file by uploading missing pieces to more hosts.
func (f *file) repair(r io.ReaderAt, pieceMap map[uint64][]uint64, hosts []uploader) error {
	// For each chunk with missing pieces, re-encode the chunk and upload each
	// missing piece.
	var wg sync.WaitGroup
	for chunkIndex, missingPieces := range pieceMap {
		// can only upload to hosts that aren't already storing this chunk
		// TODO: what if we're renewing?
		// 	curHosts := f.chunkHosts(chunkIndex)
		// 	var newHosts []uploader
		// outer:
		// 	for _, h := range hosts {
		// 		for _, ip := range curHosts {
		// 			if ip == h.addr() {

		// 				continue outer
		// 			}
		// 		}
		// 		newHosts = append(newHosts, h)
		// 	}
		newHosts := hosts
		// don't bother encoding if there aren't any hosts to upload to
		if len(newHosts) == 0 {
			continue
		}

		// read chunk data and encode
		chunk := make([]byte, f.chunkSize())
		_, err := r.ReadAt(chunk, int64(chunkIndex*f.chunkSize()))
		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
			return err
		}
		pieces, err := f.erasureCode.Encode(chunk)
		if err != nil {
			return err
		}

		// upload pieces, split evenly among hosts
		wg.Add(len(missingPieces))
		for j, pieceIndex := range missingPieces {
			host := newHosts[j%len(newHosts)]
			up := uploadPiece{pieces[pieceIndex], chunkIndex, pieceIndex}
			//go func(host uploader, up uploadPiece) {
			_ = host.addPiece(up)
			wg.Done()
			//}(host, up)
		}
		wg.Wait()

		// update contracts
		f.mu.Lock()
		for _, h := range hosts {
			contract := h.fileContract()
			f.contracts[contract.ID] = contract
		}
		f.mu.Unlock()
	}

	return nil
}

// threadedRepairUploads improves the health of files tracked by the renter by
// reuploading their missing pieces. Multiple repair attempts may be necessary
// before the file reaches full redundancy.
func (r *Renter) threadedRepairUploads() {
	// a primitive blacklist is used to augment the hostdb's weights. Each
	// negotiation failure increments the integer, and the probability of
	// selecting the host for upload is 1/n.
	blacklist := make(map[modules.NetAddress]int)

	for {
		time.Sleep(5 * time.Second)

		if !r.wallet.Unlocked() {
			continue
		}

		// make copy of repair set under lock
		repairing := make(map[string]string)
		id := r.mu.RLock()
		for name, path := range r.repairSet {
			repairing[name] = path
		}
		r.mu.RUnlock(id)

		for name, path := range repairing {
			// retrieve file object and get current height
			id = r.mu.RLock()
			f, ok := r.files[name]
			//height := r.blockHeight
			r.mu.RUnlock(id)
			if !ok {
				r.log.Printf("failed to repair %v: no longer tracking that file", name)
				id = r.mu.Lock()
				delete(r.repairSet, name)
				r.mu.Unlock(id)
				continue
			}

			// delete any expired contracts
			//f.removeExpiredContracts(height)

			// determine file health
			badChunks := f.incompleteChunks()
			if len(badChunks) == 0 {
				//badChunks = f.expiringChunks(height)
				// if len(badChunks) == 0 {
				// 	// nothing to do
				// 	continue
				// }
				continue
			}

			r.log.Printf("repairing %v chunks of %v", len(badChunks), name)

			// defer is really convenient for cleaning up resources, so an
			// inline function is justified
			err := func() error {
				// open file handle
				handle, err := os.Open(path)
				if err != nil {
					return err
				}
				defer handle.Close()

				// build host list
				bytesPerHost := f.pieceSize * f.numChunks()
				var hosts []uploader
				randHosts := r.hostDB.RandomHosts(f.erasureCode.NumPieces() * 2)
				for _, h := range randHosts {
					// probabilistically filter out known bad hosts
					// unresponsive hosts will be selected with probability 1/(1+nFailures)
					nFailures, ok := blacklist[h.IPAddress]
					if n, _ := crypto.RandIntn(1 + nFailures); ok && n != 0 {
						continue
					}

					// TODO: use smarter duration
					hostUploader, err := r.newHostUploader(h, bytesPerHost, defaultDuration, f.masterKey)
					if err != nil {
						// penalize unresponsive hosts
						if strings.Contains(err.Error(), "timeout") {
							blacklist[h.IPAddress]++
						}
						continue
					}
					defer hostUploader.Close()

					hosts = append(hosts, hostUploader)
					if len(hosts) >= f.erasureCode.NumPieces() {
						break
					}
				}

				if len(hosts) < f.erasureCode.MinPieces() {
					// don't return an error in this case, since the file
					// should not be removed from the repair set
					r.log.Printf("failed to repair %v: not enough hosts", name)
					return nil
				}

				return f.repair(handle, badChunks, hosts)
			}()

			if err != nil {
				r.log.Printf("%v cannot be repaired: %v", name, err)
				id = r.mu.Lock()
				delete(r.repairSet, name)
				r.mu.Unlock(id)
			}

			// save the repaired file data
			err = r.saveFile(f)
			if err != nil {
				// definitely bad, but we probably shouldn't delete from the
				// repair set if this happens
				r.log.Printf("failed to save repaired file %v: %v", name, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>package resource

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/logging"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/v2/internal/diagutils"
	grpcplugin "github.com/hashicorp/terraform-plugin-sdk/v2/internal/helper/plugin"
	proto "github.com/hashicorp/terraform-plugin-sdk/v2/internal/tfplugin5"
	"github.com/hashicorp/terraform-plugin-sdk/v2/plugin"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
	tftest "github.com/hashicorp/terraform-plugin-test/v2"
	testing "github.com/mitchellh/go-testing-interface"
)

func runProviderCommand(t testing.T, f func() error, wd *tftest.WorkingDir, factories map[string]func() (*schema.Provider, error)) error {
	t.Helper()

	// Run the provider in the same process as the test runner using the
	// reattach behavior in Terraform. This ensures we get test coverage
	// and enables the use of delve as a debugger.

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// this is needed so Terraform doesn't default to expecting protocol 4;
	// we're skipping the handshake because Terraform didn't launch the
	// plugin.
	os.Setenv("PLUGIN_PROTOCOL_VERSIONS", "5")

	var namespaces []string
	host := "registry.terraform.io"
	if v := os.Getenv("TF_ACC_PROVIDER_NAMESPACE"); v != "" {
		namespaces = append(namespaces, v)
	} else {
		// unfortunately, we need to populate both of them
		// Terraform 0.12.26 and higher uses the legacy mode ("-")
		// Terraform 0.13.0 and higher uses the default mode ("hashicorp")
		// because of the change in how providers are addressed in 0.13
		namespaces = append(namespaces, "-", "hashicorp")
	}
	if v := os.Getenv("TF_ACC_PROVIDER_HOST"); v != "" {
		host = v
	}

	// Spin up gRPC servers for every provider factory, start a
	// WaitGroup to listen for all of the close channels.
	wg := sync.WaitGroup{}
	wg.Add(len(factories))
	reattachInfo := map[string]plugin.ReattachConfig{}
	for providerName, factory := range factories {
		// providerName may be returned as terraform-provider-foo, and we need
		// just foo. So let's fix that.
		providerName = strings.TrimPrefix(providerName, "terraform-provider-")

		provider, err := factory()
		if err != nil {
			return fmt.Errorf("unable to create provider %q from factory: %w", providerName, err)
		}

		// PT: should this actually be called here? does it not get called by TF itself already?
		diags := provider.Configure(ctx, terraform.NewResourceConfigRaw(nil))
		if diags.HasError() {
			return fmt.Errorf("unable to configure provider %q: %w", providerName, diagutils.ErrorDiags(diags))
		}

		opts := &plugin.ServeOpts{
			GRPCProviderFunc: func() proto.ProviderServer {
				return grpcplugin.NewGRPCProviderServer(provider)
			},
			Logger: hclog.New(&hclog.LoggerOptions{
				Name:   "plugintest",
				Level:  hclog.Trace,
				Output: ioutil.Discard,
			}),
		}

		config, closeCh, err := plugin.DebugServe(ctx, opts)
		if err != nil {
			return fmt.Errorf("unable to server provider %q: %w", providerName, err)
		}

		go func(c <-chan struct{}) {
			<-c
			wg.Done()
		}(closeCh)

		// Copy reattach info for any additional namespaces needed for testing
		for _, ns := range namespaces {
			reattachInfo[strings.TrimSuffix(host, "/")+"/"+
				strings.TrimSuffix(ns, "/")+"/"+
				providerName] = config
		}
	}

	// plugin.DebugServe hijacks our log output location, so let's reset it
	logging.SetOutput()

	reattachStr, err := json.Marshal(reattachInfo)
	if err != nil {
		return err
	}
	wd.Setenv("TF_REATTACH_PROVIDERS", string(reattachStr))

	// ok, let's call whatever Terraform command the test was trying to
	// call, now that we know it'll attach back to that server we just
	// started.
	err = f()
	if err != nil {
		log.Printf("[WARN] Got error running Terraform: %s", err)
	}

	// cancel the server so it'll return. Otherwise, this closeCh won't get
	// closed, and we'll hang here.
	cancel()

	// wait for the server to actually shut down; it may take a moment for
	// it to clean up, or whatever.
	// TODO: add a timeout here?
	wg.Wait()

	// once we've run the Terraform command, let's remove the reattach
	// information from the WorkingDir's environment. The WorkingDir will
	// persist until the next call, but the server in the reattach info
	// doesn't exist anymore at this point, so the reattach info is no
	// longer valid. In theory it should be overwritten in the next call,
	// but just to avoid any confusing bug reports, let's just unset the
	// environment variable altogether.
	wd.Unsetenv("TF_REATTACH_PROVIDERS")

	// return any error returned from the orchestration code running
	// Terraform commands
	return err
}
<commit_msg>Some minor cleanup.<commit_after>package resource

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/logging"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/v2/internal/diagutils"
	grpcplugin "github.com/hashicorp/terraform-plugin-sdk/v2/internal/helper/plugin"
	proto "github.com/hashicorp/terraform-plugin-sdk/v2/internal/tfplugin5"
	"github.com/hashicorp/terraform-plugin-sdk/v2/plugin"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
	tftest "github.com/hashicorp/terraform-plugin-test/v2"
	testing "github.com/mitchellh/go-testing-interface"
)

func runProviderCommand(t testing.T, f func() error, wd *tftest.WorkingDir, factories map[string]func() (*schema.Provider, error)) error {
	// don't point to this as a test failure location
	// point to whatever called it
	t.Helper()

	// Run the providers in the same process as the test runner using the
	// reattach behavior in Terraform. This ensures we get test coverage
	// and enables the use of delve as a debugger.

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// this is needed so Terraform doesn't default to expecting protocol 4;
	// we're skipping the handshake because Terraform didn't launch the
	// plugins.
	os.Setenv("PLUGIN_PROTOCOL_VERSIONS", "5")

	// Terraform 0.12.X and 0.13.X+ treat namespaceless providers
	// differently in terms of what namespace they default to. So we're
	// going to set both variations, as we don't know which version of
	// Terraform we're talking to. We're also going to allow overriding
	// the host or namespace using environment variables.
	var namespaces []string
	host := "registry.terraform.io"
	if v := os.Getenv("TF_ACC_PROVIDER_NAMESPACE"); v != "" {
		namespaces = append(namespaces, v)
	} else {
		namespaces = append(namespaces, "-", "hashicorp")
	}
	if v := os.Getenv("TF_ACC_PROVIDER_HOST"); v != "" {
		host = v
	}

	// Spin up gRPC servers for every provider factory, start a
	// WaitGroup to listen for all of the close channels.
	var wg sync.WaitGroup
	reattachInfo := map[string]plugin.ReattachConfig{}
	for providerName, factory := range factories {
		// providerName may be returned as terraform-provider-foo, and
		// we need just foo. So let's fix that.
		providerName = strings.TrimPrefix(providerName, "terraform-provider-")

		provider, err := factory()
		if err != nil {
			return fmt.Errorf("unable to create provider %q from factory: %w", providerName, err)
		}

		// keep track of the running factory, so we can make sure it's
		// shut down.
		wg.Add(1)

		// PT: should this actually be called here? does it not get called by TF itself already?
		// PC: it should be. Why was it added? Did something not work without it?
		diags := provider.Configure(ctx, terraform.NewResourceConfigRaw(nil))
		if diags.HasError() {
			return fmt.Errorf("unable to configure provider %q: %w", providerName, diagutils.ErrorDiags(diags))
		}

		// configure the settings our plugin will be served with
		// the GRPCProviderFunc wraps a non-gRPC provider server
		// into a gRPC interface, and the logger just discards logs
		// from go-plugin.
		opts := &plugin.ServeOpts{
			GRPCProviderFunc: func() proto.ProviderServer {
				return grpcplugin.NewGRPCProviderServer(provider)
			},
			Logger: hclog.New(&hclog.LoggerOptions{
				Name:   "plugintest",
				Level:  hclog.Trace,
				Output: ioutil.Discard,
			}),
		}

		// let's actually start the provider server
		config, closeCh, err := plugin.DebugServe(ctx, opts)
		if err != nil {
			return fmt.Errorf("unable to serve provider %q: %w", providerName, err)
		}

		// plugin.DebugServe hijacks our log output location, so let's
		// reset it
		logging.SetOutput()

		// when the provider exits, remove one from the waitgroup
		// so we can track when everything is done
		go func(c <-chan struct{}) {
			<-c
			wg.Done()
		}(closeCh)

		// set our provider's reattachinfo in our map, once
		// for every namespace that different Terraform versions
		// may expect.
		for _, ns := range namespaces {
			reattachInfo[strings.TrimSuffix(host, "/")+"/"+
				strings.TrimSuffix(ns, "/")+"/"+
				providerName] = config
		}
	}

	// set the environment variable that will tell Terraform how to
	// connect to our various running servers.
	reattachStr, err := json.Marshal(reattachInfo)
	if err != nil {
		return err
	}
	wd.Setenv("TF_REATTACH_PROVIDERS", string(reattachStr))

	// ok, let's call whatever Terraform command the test was trying to
	// call, now that we know it'll attach back to those servers we just
	// started.
	err = f()
	if err != nil {
		log.Printf("[WARN] Got error running Terraform: %s", err)
	}

	// cancel the servers so they'll return. Otherwise, this closeCh won't
	// get closed, and we'll hang here.
	cancel()

	// wait for the servers to actually shut down; it may take a moment for
	// them to clean up, or whatever.
	// TODO: add a timeout here?
	// PC: do we need one? The test will time out automatically...
	wg.Wait()

	// once we've run the Terraform command, let's remove the reattach
	// information from the WorkingDir's environment. The WorkingDir will
	// persist until the next call, but the server in the reattach info
	// doesn't exist anymore at this point, so the reattach info is no
	// longer valid. In theory it should be overwritten in the next call,
	// but just to avoid any confusing bug reports, let's just unset the
	// environment variable altogether.
	wd.Unsetenv("TF_REATTACH_PROVIDERS")

	// return any error returned from the orchestration code running
	// Terraform commands
	return err
}
<|endoftext|>


<commit_before>package docker_helpers

import (
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
)

var dockerDialer = &net.Dialer{
	Timeout:   30 * time.Second,
	KeepAlive: 30 * time.Second,
}

func httpTransportFix(host string, client Client) {
	dockerClient, ok := client.(*docker.Client)
	if !ok {
		return
	}

	logrus.WithField("host", host).Debugln("Applying docker.Client transport fix:", dockerClient)
	dockerClient.Dialer = dockerDialer
	dockerClient.HTTPClient = &http.Client{
		Transport: &http.Transport{
			Proxy:               http.ProxyFromEnvironment,
			Dial:                dockerDialer.Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig:     dockerClient.TLSConfig,
		},
	}
}

func New(c DockerCredentials, apiVersion string) (client Client, err error) {
	endpoint := "unix:///var/run/docker.sock"
	tlsVerify := false
	tlsCertPath := ""

	defer func() {
		if client != nil {
			httpTransportFix(endpoint, client)
		}
	}()

	if c.Host != "" {
		// read docker config from config
		endpoint = c.Host
		if c.CertPath != "" {
			tlsVerify = true
			tlsCertPath = c.CertPath
		}
	} else if host := os.Getenv("DOCKER_HOST"); host != "" {
		// read docker config from environment
		endpoint = host
		tlsVerify, _ = strconv.ParseBool(os.Getenv("DOCKER_TLS_VERIFY"))
		tlsCertPath = os.Getenv("DOCKER_CERT_PATH")
	}

	if tlsVerify {
		client, err = docker.NewVersionnedTLSClient(
			endpoint,
			filepath.Join(tlsCertPath, "cert.pem"),
			filepath.Join(tlsCertPath, "key.pem"),
			filepath.Join(tlsCertPath, "ca.pem"),
			apiVersion,
		)
		return
	}

	client, err = docker.NewVersionedClient(endpoint, apiVersion)
	return
}

func Close(client Client) {
	dockerClient, ok := client.(*docker.Client)
	if !ok {
		return
	}

	// Nuke all connections
	if transport, ok := dockerClient.HTTPClient.Transport.(*http.Transport); ok && transport != http.DefaultTransport {
		transport.DisableKeepAlives = true
		transport.CloseIdleConnections()
		logrus.Debugln("Closed all idle connections for docker.Client:", dockerClient)
	}
}
<commit_msg>Fix nil casting issue on docker client creation<commit_after>package docker_helpers

import (
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
)

var dockerDialer = &net.Dialer{
	Timeout:   30 * time.Second,
	KeepAlive: 30 * time.Second,
}

func httpTransportFix(host string, client Client) {
	dockerClient, ok := client.(*docker.Client)
	if !ok || dockerClient == nil {
		return
	}

	logrus.WithField("host", host).Debugln("Applying docker.Client transport fix:", dockerClient)
	dockerClient.Dialer = dockerDialer
	dockerClient.HTTPClient = &http.Client{
		Transport: &http.Transport{
			Proxy:               http.ProxyFromEnvironment,
			Dial:                dockerDialer.Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig:     dockerClient.TLSConfig,
		},
	}
}

func New(c DockerCredentials, apiVersion string) (client Client, err error) {
	endpoint := "unix:///var/run/docker.sock"
	tlsVerify := false
	tlsCertPath := ""

	defer func() {
		if client != nil {
			httpTransportFix(endpoint, client)
		}
	}()

	if c.Host != "" {
		// read docker config from config
		endpoint = c.Host
		if c.CertPath != "" {
			tlsVerify = true
			tlsCertPath = c.CertPath
		}
	} else if host := os.Getenv("DOCKER_HOST"); host != "" {
		// read docker config from environment
		endpoint = host
		tlsVerify, _ = strconv.ParseBool(os.Getenv("DOCKER_TLS_VERIFY"))
		tlsCertPath = os.Getenv("DOCKER_CERT_PATH")
	}

	if tlsVerify {
		client, err = docker.NewVersionnedTLSClient(
			endpoint,
			filepath.Join(tlsCertPath, "cert.pem"),
			filepath.Join(tlsCertPath, "key.pem"),
			filepath.Join(tlsCertPath, "ca.pem"),
			apiVersion,
		)
		if err != nil {
			logrus.Errorln("Error while TLS Docker client creation:", err)
		}
		return
	}

	client, err = docker.NewVersionedClient(endpoint, apiVersion)
	if err != nil {
		logrus.Errorln("Error while Docker client creation:", err)
	}
	return
}

func Close(client Client) {
	dockerClient, ok := client.(*docker.Client)
	if !ok {
		return
	}

	// Nuke all connections
	if transport, ok := dockerClient.HTTPClient.Transport.(*http.Transport); ok && transport != http.DefaultTransport {
		transport.DisableKeepAlives = true
		transport.CloseIdleConnections()
		logrus.Debugln("Closed all idle connections for docker.Client:", dockerClient)
	}
}
<|endoftext|>


<commit_before>package dtls

import (
	"encoding/binary"
	"fmt"
	"hash"
)

// CipherSuiteID is an ID for our supported CipherSuites
type CipherSuiteID uint16

// Supported Cipher Suites
const (
	// AES-128-CCM
	TLS_ECDHE_ECDSA_WITH_AES_128_CCM   CipherSuiteID = 0xc0ac //nolint:golint,stylecheck
	TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 CipherSuiteID = 0xc0ae //nolint:golint,stylecheck

	// AES-128-GCM-SHA256
	TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 CipherSuiteID = 0xc02b //nolint:golint,stylecheck
	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256   CipherSuiteID = 0xc02f //nolint:golint,stylecheck

	// AES-256-CBC-SHA
	TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA CipherSuiteID = 0xc00a //nolint:golint,stylecheck
	TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA   CipherSuiteID = 0xc014 //nolint:golint,stylecheck

	TLS_PSK_WITH_AES_128_CCM        CipherSuiteID = 0xc0a4 //nolint:golint,stylecheck
	TLS_PSK_WITH_AES_128_CCM_8      CipherSuiteID = 0xc0a8 //nolint:golint,stylecheck
	TLS_PSK_WITH_AES_128_GCM_SHA256 CipherSuiteID = 0x00a8 //nolint:golint,stylecheck
	TLS_PSK_WITH_AES_128_CBC_SHA256 CipherSuiteID = 0x00ae //nolint:golint,stylecheck
)

var _ = allCipherSuites() // Necessary until this function isn't only used by Go 1.14

func (c CipherSuiteID) String() string {
	switch c {
	case TLS_ECDHE_ECDSA_WITH_AES_128_CCM:
		return "TLS_ECDHE_ECDSA_WITH_AES_128_CCM"
	case TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8:
		return "TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8"
	case TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:
		return "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"
	case TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:
		return "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
	case TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA:
		return "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA"
	case TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:
		return "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA"
	case TLS_PSK_WITH_AES_128_CCM:
		return "TLS_PSK_WITH_AES_128_CCM"
	case TLS_PSK_WITH_AES_128_CCM_8:
		return "TLS_PSK_WITH_AES_128_CCM_8"
	case TLS_PSK_WITH_AES_128_GCM_SHA256:
		return "TLS_PSK_WITH_AES_128_GCM_SHA256"
	case TLS_PSK_WITH_AES_128_CBC_SHA256:
		return "TLS_PSK_WITH_AES_128_CBC_SHA256"
	default:
		return fmt.Sprintf("unknown(%v)", uint16(c))
	}
}

type cipherSuite interface {
	String() string
	ID() CipherSuiteID
	certificateType() clientCertificateType
	hashFunc() func() hash.Hash
	isPSK() bool
	isInitialized() bool

	// Generate the internal encryption state
	init(masterSecret, clientRandom, serverRandom []byte, isClient bool) error

	encrypt(pkt *recordLayer, raw []byte) ([]byte, error)
	decrypt(in []byte) ([]byte, error)
}

// CipherSuiteName provides the same functionality as tls.CipherSuiteName
// that appeared first in Go 1.14.
//
// Our implementation differs slightly in that it takes in a CiperSuiteID,
// like the rest of our library, instead of a uint16 like crypto/tls.
func CipherSuiteName(id CipherSuiteID) string {
	suite := cipherSuiteForID(id)
	if suite != nil {
		return suite.String()
	}
	return fmt.Sprintf("0x%04X", uint16(id))
}

// Taken from https://www.iana.org/assignments/tls-parameters/tls-parameters.xml
// A cipherSuite is a specific combination of key agreement, cipher and MAC
// function.
func cipherSuiteForID(id CipherSuiteID) cipherSuite {
	switch id {
	case TLS_ECDHE_ECDSA_WITH_AES_128_CCM:
		return newCipherSuiteTLSEcdheEcdsaWithAes128Ccm()
	case TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8:
		return newCipherSuiteTLSEcdheEcdsaWithAes128Ccm8()
	case TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:
		return &cipherSuiteTLSEcdheEcdsaWithAes128GcmSha256{}
	case TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:
		return &cipherSuiteTLSEcdheRsaWithAes128GcmSha256{}
	case TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA:
		return &cipherSuiteTLSEcdheEcdsaWithAes256CbcSha{}
	case TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:
		return &cipherSuiteTLSEcdheRsaWithAes256CbcSha{}
	case TLS_PSK_WITH_AES_128_CCM:
		return newCipherSuiteTLSPskWithAes128Ccm()
	case TLS_PSK_WITH_AES_128_CCM_8:
		return newCipherSuiteTLSPskWithAes128Ccm8()
	case TLS_PSK_WITH_AES_128_GCM_SHA256:
		return &cipherSuiteTLSPskWithAes128GcmSha256{}
	case TLS_PSK_WITH_AES_128_CBC_SHA256:
		return &cipherSuiteTLSPskWithAes128CbcSha256{}
	}
	return nil
}

// CipherSuites we support in order of preference
func defaultCipherSuites() []cipherSuite {
	return []cipherSuite{
		&cipherSuiteTLSEcdheEcdsaWithAes128GcmSha256{},
		&cipherSuiteTLSEcdheRsaWithAes128GcmSha256{},
		&cipherSuiteTLSEcdheEcdsaWithAes256CbcSha{},
		&cipherSuiteTLSEcdheRsaWithAes256CbcSha{},
	}
}

func allCipherSuites() []cipherSuite {
	return []cipherSuite{
		newCipherSuiteTLSEcdheEcdsaWithAes128Ccm(),
		newCipherSuiteTLSEcdheEcdsaWithAes128Ccm8(),
		&cipherSuiteTLSEcdheEcdsaWithAes128GcmSha256{},
		&cipherSuiteTLSEcdheRsaWithAes128GcmSha256{},
		&cipherSuiteTLSEcdheEcdsaWithAes256CbcSha{},
		&cipherSuiteTLSEcdheRsaWithAes256CbcSha{},
		newCipherSuiteTLSPskWithAes128Ccm(),
		newCipherSuiteTLSPskWithAes128Ccm8(),
		&cipherSuiteTLSPskWithAes128GcmSha256{},
	}
}

func decodeCipherSuites(buf []byte) ([]cipherSuite, error) {
	if len(buf) < 2 {
		return nil, errDTLSPacketInvalidLength
	}
	cipherSuitesCount := int(binary.BigEndian.Uint16(buf[0:])) / 2
	rtrn := []cipherSuite{}
	for i := 0; i < cipherSuitesCount; i++ {
		if len(buf) < (i*2 + 4) {
			return nil, errBufferTooSmall
		}
		id := CipherSuiteID(binary.BigEndian.Uint16(buf[(i*2)+2:]))
		if c := cipherSuiteForID(id); c != nil {
			rtrn = append(rtrn, c)
		}
	}
	return rtrn, nil
}

func encodeCipherSuites(cipherSuites []cipherSuite) []byte {
	out := []byte{0x00, 0x00}
	binary.BigEndian.PutUint16(out[len(out)-2:], uint16(len(cipherSuites)*2))
	for _, c := range cipherSuites {
		out = append(out, []byte{0x00, 0x00}...)
		binary.BigEndian.PutUint16(out[len(out)-2:], uint16(c.ID()))
	}
	return out
}

func parseCipherSuites(userSelectedSuites []CipherSuiteID, nonPSK, PSK bool) ([]cipherSuite, error) {
	if !nonPSK && !PSK {
		return nil, errNoAvailableCipherSuites
	}

	cipherSuitesForIDs := func(ids []CipherSuiteID) ([]cipherSuite, error) {
		cipherSuites := []cipherSuite{}
		for _, id := range ids {
			c := cipherSuiteForID(id)
			if c == nil {
				return nil, &invalidCipherSuite{id}
			}
			cipherSuites = append(cipherSuites, c)
		}
		return cipherSuites, nil
	}

	var (
		cipherSuites []cipherSuite
		err          error
		i            int
	)
	if len(userSelectedSuites) != 0 {
		cipherSuites, err = cipherSuitesForIDs(userSelectedSuites)
		if err != nil {
			return nil, err
		}
	} else {
		cipherSuites = defaultCipherSuites()
	}

	var foundPSK, foundNonPSK bool
	for _, c := range cipherSuites {
		if PSK && c.isPSK() {
			foundPSK = true
		} else if nonPSK && !c.isPSK() {
			foundNonPSK = true
		} else {
			continue
		}
		cipherSuites[i] = c
		i++
	}

	if PSK && !foundPSK {
		return nil, errNoAvailablePSKCipherSuite
	}
	if nonPSK && !foundNonPSK {
		return nil, errNoAvailableNonPSKCipherSuite
	}

	return cipherSuites[:i], nil
}
<commit_msg>Clean up parseCipherSuites<commit_after>package dtls

import (
	"encoding/binary"
	"fmt"
	"hash"
)

// CipherSuiteID is an ID for our supported CipherSuites
type CipherSuiteID uint16

// Supported Cipher Suites
const (
	// AES-128-CCM
	TLS_ECDHE_ECDSA_WITH_AES_128_CCM   CipherSuiteID = 0xc0ac //nolint:golint,stylecheck
	TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 CipherSuiteID = 0xc0ae //nolint:golint,stylecheck

	// AES-128-GCM-SHA256
	TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 CipherSuiteID = 0xc02b //nolint:golint,stylecheck
	TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256   CipherSuiteID = 0xc02f //nolint:golint,stylecheck

	// AES-256-CBC-SHA
	TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA CipherSuiteID = 0xc00a //nolint:golint,stylecheck
	TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA   CipherSuiteID = 0xc014 //nolint:golint,stylecheck

	TLS_PSK_WITH_AES_128_CCM        CipherSuiteID = 0xc0a4 //nolint:golint,stylecheck
	TLS_PSK_WITH_AES_128_CCM_8      CipherSuiteID = 0xc0a8 //nolint:golint,stylecheck
	TLS_PSK_WITH_AES_128_GCM_SHA256 CipherSuiteID = 0x00a8 //nolint:golint,stylecheck
	TLS_PSK_WITH_AES_128_CBC_SHA256 CipherSuiteID = 0x00ae //nolint:golint,stylecheck
)

var _ = allCipherSuites() // Necessary until this function isn't only used by Go 1.14

func (c CipherSuiteID) String() string {
	switch c {
	case TLS_ECDHE_ECDSA_WITH_AES_128_CCM:
		return "TLS_ECDHE_ECDSA_WITH_AES_128_CCM"
	case TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8:
		return "TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8"
	case TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:
		return "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"
	case TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:
		return "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
	case TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA:
		return "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA"
	case TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:
		return "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA"
	case TLS_PSK_WITH_AES_128_CCM:
		return "TLS_PSK_WITH_AES_128_CCM"
	case TLS_PSK_WITH_AES_128_CCM_8:
		return "TLS_PSK_WITH_AES_128_CCM_8"
	case TLS_PSK_WITH_AES_128_GCM_SHA256:
		return "TLS_PSK_WITH_AES_128_GCM_SHA256"
	case TLS_PSK_WITH_AES_128_CBC_SHA256:
		return "TLS_PSK_WITH_AES_128_CBC_SHA256"
	default:
		return fmt.Sprintf("unknown(%v)", uint16(c))
	}
}

type cipherSuite interface {
	String() string
	ID() CipherSuiteID
	certificateType() clientCertificateType
	hashFunc() func() hash.Hash
	isPSK() bool
	isInitialized() bool

	// Generate the internal encryption state
	init(masterSecret, clientRandom, serverRandom []byte, isClient bool) error

	encrypt(pkt *recordLayer, raw []byte) ([]byte, error)
	decrypt(in []byte) ([]byte, error)
}

// CipherSuiteName provides the same functionality as tls.CipherSuiteName
// that appeared first in Go 1.14.
//
// Our implementation differs slightly in that it takes in a CiperSuiteID,
// like the rest of our library, instead of a uint16 like crypto/tls.
func CipherSuiteName(id CipherSuiteID) string {
	suite := cipherSuiteForID(id)
	if suite != nil {
		return suite.String()
	}
	return fmt.Sprintf("0x%04X", uint16(id))
}

// Taken from https://www.iana.org/assignments/tls-parameters/tls-parameters.xml
// A cipherSuite is a specific combination of key agreement, cipher and MAC
// function.
func cipherSuiteForID(id CipherSuiteID) cipherSuite {
	switch id {
	case TLS_ECDHE_ECDSA_WITH_AES_128_CCM:
		return newCipherSuiteTLSEcdheEcdsaWithAes128Ccm()
	case TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8:
		return newCipherSuiteTLSEcdheEcdsaWithAes128Ccm8()
	case TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:
		return &cipherSuiteTLSEcdheEcdsaWithAes128GcmSha256{}
	case TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:
		return &cipherSuiteTLSEcdheRsaWithAes128GcmSha256{}
	case TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA:
		return &cipherSuiteTLSEcdheEcdsaWithAes256CbcSha{}
	case TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:
		return &cipherSuiteTLSEcdheRsaWithAes256CbcSha{}
	case TLS_PSK_WITH_AES_128_CCM:
		return newCipherSuiteTLSPskWithAes128Ccm()
	case TLS_PSK_WITH_AES_128_CCM_8:
		return newCipherSuiteTLSPskWithAes128Ccm8()
	case TLS_PSK_WITH_AES_128_GCM_SHA256:
		return &cipherSuiteTLSPskWithAes128GcmSha256{}
	case TLS_PSK_WITH_AES_128_CBC_SHA256:
		return &cipherSuiteTLSPskWithAes128CbcSha256{}
	}
	return nil
}

// CipherSuites we support in order of preference
func defaultCipherSuites() []cipherSuite {
	return []cipherSuite{
		&cipherSuiteTLSEcdheEcdsaWithAes128GcmSha256{},
		&cipherSuiteTLSEcdheRsaWithAes128GcmSha256{},
		&cipherSuiteTLSEcdheEcdsaWithAes256CbcSha{},
		&cipherSuiteTLSEcdheRsaWithAes256CbcSha{},
	}
}

func allCipherSuites() []cipherSuite {
	return []cipherSuite{
		newCipherSuiteTLSEcdheEcdsaWithAes128Ccm(),
		newCipherSuiteTLSEcdheEcdsaWithAes128Ccm8(),
		&cipherSuiteTLSEcdheEcdsaWithAes128GcmSha256{},
		&cipherSuiteTLSEcdheRsaWithAes128GcmSha256{},
		&cipherSuiteTLSEcdheEcdsaWithAes256CbcSha{},
		&cipherSuiteTLSEcdheRsaWithAes256CbcSha{},
		newCipherSuiteTLSPskWithAes128Ccm(),
		newCipherSuiteTLSPskWithAes128Ccm8(),
		&cipherSuiteTLSPskWithAes128GcmSha256{},
	}
}

func decodeCipherSuites(buf []byte) ([]cipherSuite, error) {
	if len(buf) < 2 {
		return nil, errDTLSPacketInvalidLength
	}
	cipherSuitesCount := int(binary.BigEndian.Uint16(buf[0:])) / 2
	rtrn := []cipherSuite{}
	for i := 0; i < cipherSuitesCount; i++ {
		if len(buf) < (i*2 + 4) {
			return nil, errBufferTooSmall
		}
		id := CipherSuiteID(binary.BigEndian.Uint16(buf[(i*2)+2:]))
		if c := cipherSuiteForID(id); c != nil {
			rtrn = append(rtrn, c)
		}
	}
	return rtrn, nil
}

func encodeCipherSuites(cipherSuites []cipherSuite) []byte {
	out := []byte{0x00, 0x00}
	binary.BigEndian.PutUint16(out[len(out)-2:], uint16(len(cipherSuites)*2))
	for _, c := range cipherSuites {
		out = append(out, []byte{0x00, 0x00}...)
		binary.BigEndian.PutUint16(out[len(out)-2:], uint16(c.ID()))
	}
	return out
}

func parseCipherSuites(userSelectedSuites []CipherSuiteID, includeNonPSK, includePSK bool) ([]cipherSuite, error) {
	if !includeNonPSK && !includePSK {
		return nil, errNoAvailableCipherSuites
	}

	cipherSuitesForIDs := func(ids []CipherSuiteID) ([]cipherSuite, error) {
		cipherSuites := []cipherSuite{}
		for _, id := range ids {
			c := cipherSuiteForID(id)
			if c == nil {
				return nil, &invalidCipherSuite{id}
			}
			cipherSuites = append(cipherSuites, c)
		}
		return cipherSuites, nil
	}

	var (
		cipherSuites []cipherSuite
		err          error
		i            int
	)
	if len(userSelectedSuites) != 0 {
		cipherSuites, err = cipherSuitesForIDs(userSelectedSuites)
		if err != nil {
			return nil, err
		}
	} else {
		cipherSuites = defaultCipherSuites()
	}

	var foundPSK, foundNonPSK bool
	for _, c := range cipherSuites {
		switch {
		case includePSK && c.isPSK():
			foundPSK = true
		case includeNonPSK && !c.isPSK():
			foundNonPSK = true
		default:
			continue
		}
		cipherSuites[i] = c
		i++
	}

	if includePSK && !foundPSK {
		return nil, errNoAvailablePSKCipherSuite
	}
	if includeNonPSK && !foundNonPSK {
		return nil, errNoAvailableNonPSKCipherSuite
	}

	return cipherSuites[:i], nil
}
<|endoftext|>


<commit_before>package cjk2num

import (
	"fmt"
	"testing"
)

// test Convert
func TestConvert(t *testing.T) {
	check(t, "一億三千万二百十五", 130000215)
	check(t, "一億", 100000000)
	check(t, "千拾", 1010)
	check(t, "百", 100)
	check(t, "二百三十一兆五十五億二千万千五百一", 231005520001501)
	check(t, "二百三十一兆五十五億二千万千五〇一", 231005520001501)
	check(t, "二十三人", 23)
	check(t, "７十1", 71)

	// expect error
	if _, err := Convert("一億万"); err == nil {
		t.Errorf("passed invalid format \n")
	}
}

//Check
func check(t *testing.T, input string, ans int64) {
	res, err := Convert(input)
	if err != nil {
		t.Errorf("%s\n", err.Error())
	}

	if res != ans {
		t.Errorf("%s =\n%d\n, want \n%d", input, res, ans)
		return
	}
}

// Example 1 千や百と言った記号を用いないパターン
func Example_case1() {
	res, _ := Convert("一九八四")
	fmt.Printf("%d", res)
	//Output:1984
}

// Example 2　億や千などの記号を用いるパターン
func Example_case2() {
	res, _ := Convert("一億二千三百四十五万六千七百八十九")
	fmt.Printf("%d", res)
	//Output:123456789
}

// Example 3 大字を使ったパターン
func Example_case3() {
	res, _ := Convert("壱萬弐千参百")
	fmt.Printf("%d", res)
	//Output:12300
}

// Example 4 〇(れい:まると違う)を含むパターン
func Example_case4() {
	res, _ := Convert("一〇九")
	fmt.Printf("%d", res)
	//Output:109
}

// Example 5 中文
func Example_case5() {
	res, _ := Convert("壹億貳仟叁佰肆拾伍萬陸仟柒佰捌拾玖")
	fmt.Printf("%d", res)
	//Output:123456789
}

// Example 6 한글
func Example_case6() {
	res, _ := Convert("오만육천칠백팔십구")
	fmt.Printf("%d", res)
	//Output:56789
}

// Example ７変則的な単位
func Example_case7() {
	res, _ := Convert("3万１ダース")
	fmt.Printf("%d", res)
	//Output:360012
}

// Example 8　オリジナルの桁を定義
func Example_case8() {
	presetSymbols := GetPresetSymols()                    //プリセットされた記号定義を取得
	originalSymbol := BreakSymbol{"たこ", 8}                //オリジナルの単位を作成
	presetSymbols = append(presetSymbols, originalSymbol) //プリセットに加える
	res, _ := ConvertBy("10たこ", presetSymbols)
	fmt.Printf("%d", res)
	//Output:80
}
<commit_msg>add example<commit_after>package cjk2num

import (
	"fmt"
	"testing"
)

// test Convert
func TestConvert(t *testing.T) {
	check(t, "一億三千万二百十五", 130000215)
	check(t, "一億", 100000000)
	check(t, "千拾", 1010)
	check(t, "百", 100)
	check(t, "二百三十一兆五十五億二千万千五百一", 231005520001501)
	check(t, "二百三十一兆五十五億二千万千五〇一", 231005520001501)
	check(t, "二十三人", 23)
	check(t, "７十1", 71)

	// expect error
	if _, err := Convert("一億万"); err == nil {
		t.Errorf("passed invalid format \n")
	}
}

//Check
func check(t *testing.T, input string, ans int64) {
	res, err := Convert(input)
	if err != nil {
		t.Errorf("%s\n", err.Error())
	}

	if res != ans {
		t.Errorf("%s =\n%d\n, want \n%d", input, res, ans)
		return
	}
}

// Example 1 千や百と言った記号を用いないパターン
func Example_case1() {
	res, _ := Convert("一九八四")
	fmt.Printf("%d", res)
	//Output:1984
}

// Example 2　億や千などの記号を用いるパターン
func Example_case2() {
	res, _ := Convert("一億二千三百四十五万六千七百八十九")
	fmt.Printf("%d", res)
	//Output:123456789
}

// Example 3 大字を使ったパターン
func Example_case3() {
	res, _ := Convert("壱萬弐千参百")
	fmt.Printf("%d", res)
	//Output:12300
}

// Example 4 〇(れい:まると違う)を含むパターン
func Example_case4() {
	res, _ := Convert("一〇九")
	fmt.Printf("%d", res)
	//Output:109
}

// Example 5 中文
func Example_case5() {
	res, _ := Convert("壹億貳仟叁佰肆拾伍萬陸仟柒佰捌拾玖")
	fmt.Printf("%d", res)
	//Output:123456789
}

// Example 6 한글
func Example_case6() {
	res, _ := Convert("오만육천칠백팔십구")
	fmt.Printf("%d", res)
	//Output:56789
}

// Example ７変則的な単位
func Example_case7() {
	res, _ := Convert("3万１ダース")
	fmt.Printf("%d", res)
	//Output:360012
}

// Example 8　オリジナルの桁を定義
func Example_case8() {
	presetSymbols := GetPresetSymols()                    //プリセットされた記号定義を取得
	originalSymbol := BreakSymbol{"たこ", 8}                //オリジナルの単位を作成
	presetSymbols = append(presetSymbols, originalSymbol) //プリセットに加える
	res, _ := ConvertBy("10たこ", presetSymbols)
	fmt.Printf("%d", res)
	//Output:80
}

// Example 9　オリジナルの数字を定義
func Example_case9() {
	//タミル文字(தமிழ்)
	originalSymbols := []Symbol{
		NumberSymbol{"௦", 0},
		NumberSymbol{"௧", 1},
		NumberSymbol{"௨", 2},
		NumberSymbol{"௩", 3},
		NumberSymbol{"௪", 4},
		NumberSymbol{"௫", 5},
		NumberSymbol{"௬", 6},
		NumberSymbol{"௭", 7},
		NumberSymbol{"௮", 8},
		NumberSymbol{"௯", 9},
	}
	res, _ := ConvertBy("௯௯௫௬௦௮௩", originalSymbols)
	fmt.Printf("%d", res)
	//Output:9956083
}
<|endoftext|>


<commit_before>// +build linux

package fscommon

import (
	"fmt"
	"math"
	"strconv"
	"strings"

	"github.com/opencontainers/runc/libcontainer/cgroups"
)

var (
	// Deprecated: use cgroups.OpenFile instead.
	OpenFile = cgroups.OpenFile
	// Deprecated: use cgroups.ReadFile instead.
	ReadFile = cgroups.ReadFile
	// Deprecated: use cgroups.WriteFile instead.
	WriteFile = cgroups.WriteFile
)

// ParseUint converts a string to an uint64 integer.
// Negative values are returned at zero as, due to kernel bugs,
// some of the memory cgroup stats can be negative.
func ParseUint(s string, base, bitSize int) (uint64, error) {
	value, err := strconv.ParseUint(s, base, bitSize)
	if err != nil {
		intValue, intErr := strconv.ParseInt(s, base, bitSize)
		// 1. Handle negative values greater than MinInt64 (and)
		// 2. Handle negative values lesser than MinInt64
		if intErr == nil && intValue < 0 {
			return 0, nil
		} else if intErr != nil && intErr.(*strconv.NumError).Err == strconv.ErrRange && intValue < 0 {
			return 0, nil
		}

		return value, err
	}

	return value, nil
}

// ParseKeyValue parses a space-separated "name value" kind of cgroup
// parameter and returns its key as a string, and its value as uint64
// (ParseUint is used to convert the value). For example,
// "io_service_bytes 1234" will be returned as "io_service_bytes", 1234.
func ParseKeyValue(t string) (string, uint64, error) {
	parts := strings.SplitN(t, " ", 3)
	if len(parts) != 2 {
		return "", 0, fmt.Errorf("line %q is not in key value format", t)
	}

	value, err := ParseUint(parts[1], 10, 64)
	if err != nil {
		return "", 0, err
	}

	return parts[0], value, nil
}

// GetValueByKey reads a key-value pairs from the specified cgroup file,
// and returns a value of the specified key. ParseUint is used for value
// conversion.
func GetValueByKey(path, file, key string) (uint64, error) {
	content, err := cgroups.ReadFile(path, file)
	if err != nil {
		return 0, err
	}

	lines := strings.Split(string(content), "\n")
	for _, line := range lines {
		arr := strings.Split(line, " ")
		if len(arr) == 2 && arr[0] == key {
			return ParseUint(arr[1], 10, 64)
		}
	}

	return 0, nil
}

// GetCgroupParamUint reads a single uint64 value from the specified cgroup file.
// If the value read is "max", the math.MaxUint64 is returned.
func GetCgroupParamUint(path, file string) (uint64, error) {
	contents, err := GetCgroupParamString(path, file)
	if err != nil {
		return 0, err
	}
	contents = strings.TrimSpace(contents)
	if contents == "max" {
		return math.MaxUint64, nil
	}

	res, err := ParseUint(contents, 10, 64)
	if err != nil {
		return res, fmt.Errorf("unable to parse file %q", path+"/"+file)
	}
	return res, nil
}

// GetCgroupParamInt reads a single int64 value from specified cgroup file.
// If the value read is "max", the math.MaxInt64 is returned.
func GetCgroupParamInt(path, file string) (int64, error) {
	contents, err := cgroups.ReadFile(path, file)
	if err != nil {
		return 0, err
	}
	contents = strings.TrimSpace(contents)
	if contents == "max" {
		return math.MaxInt64, nil
	}

	res, err := strconv.ParseInt(contents, 10, 64)
	if err != nil {
		return res, fmt.Errorf("unable to parse %q as a int from Cgroup file %q", contents, path+"/"+file)
	}
	return res, nil
}

// GetCgroupParamString reads a string from the specified cgroup file.
func GetCgroupParamString(path, file string) (string, error) {
	contents, err := cgroups.ReadFile(path, file)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(contents), nil
}
<commit_msg>libct/cg/fscommon: introduce and use ParseError<commit_after>// +build linux

package fscommon

import (
	"fmt"
	"math"
	"path"
	"strconv"
	"strings"

	"github.com/opencontainers/runc/libcontainer/cgroups"
)

var (
	// Deprecated: use cgroups.OpenFile instead.
	OpenFile = cgroups.OpenFile
	// Deprecated: use cgroups.ReadFile instead.
	ReadFile = cgroups.ReadFile
	// Deprecated: use cgroups.WriteFile instead.
	WriteFile = cgroups.WriteFile
)

// ParseError records a parse error details, including the file path.
type ParseError struct {
	Path string
	File string
	Err  error
}

func (e *ParseError) Error() string {
	return "unable to parse " + path.Join(e.Path, e.File) + ": " + e.Err.Error()
}

func (e *ParseError) Unwrap() error { return e.Err }

// ParseUint converts a string to an uint64 integer.
// Negative values are returned at zero as, due to kernel bugs,
// some of the memory cgroup stats can be negative.
func ParseUint(s string, base, bitSize int) (uint64, error) {
	value, err := strconv.ParseUint(s, base, bitSize)
	if err != nil {
		intValue, intErr := strconv.ParseInt(s, base, bitSize)
		// 1. Handle negative values greater than MinInt64 (and)
		// 2. Handle negative values lesser than MinInt64
		if intErr == nil && intValue < 0 {
			return 0, nil
		} else if intErr != nil && intErr.(*strconv.NumError).Err == strconv.ErrRange && intValue < 0 {
			return 0, nil
		}

		return value, err
	}

	return value, nil
}

// ParseKeyValue parses a space-separated "name value" kind of cgroup
// parameter and returns its key as a string, and its value as uint64
// (ParseUint is used to convert the value). For example,
// "io_service_bytes 1234" will be returned as "io_service_bytes", 1234.
func ParseKeyValue(t string) (string, uint64, error) {
	parts := strings.SplitN(t, " ", 3)
	if len(parts) != 2 {
		return "", 0, fmt.Errorf("line %q is not in key value format", t)
	}

	value, err := ParseUint(parts[1], 10, 64)
	if err != nil {
		return "", 0, err
	}

	return parts[0], value, nil
}

// GetValueByKey reads a key-value pairs from the specified cgroup file,
// and returns a value of the specified key. ParseUint is used for value
// conversion.
func GetValueByKey(path, file, key string) (uint64, error) {
	content, err := cgroups.ReadFile(path, file)
	if err != nil {
		return 0, err
	}

	lines := strings.Split(string(content), "\n")
	for _, line := range lines {
		arr := strings.Split(line, " ")
		if len(arr) == 2 && arr[0] == key {
			val, err := ParseUint(arr[1], 10, 64)
			if err != nil {
				err = &ParseError{Path: path, File: file, Err: err}
			}
			return val, err
		}
	}

	return 0, nil
}

// GetCgroupParamUint reads a single uint64 value from the specified cgroup file.
// If the value read is "max", the math.MaxUint64 is returned.
func GetCgroupParamUint(path, file string) (uint64, error) {
	contents, err := GetCgroupParamString(path, file)
	if err != nil {
		return 0, err
	}
	contents = strings.TrimSpace(contents)
	if contents == "max" {
		return math.MaxUint64, nil
	}

	res, err := ParseUint(contents, 10, 64)
	if err != nil {
		return res, &ParseError{Path: path, File: file, Err: err}
	}
	return res, nil
}

// GetCgroupParamInt reads a single int64 value from specified cgroup file.
// If the value read is "max", the math.MaxInt64 is returned.
func GetCgroupParamInt(path, file string) (int64, error) {
	contents, err := cgroups.ReadFile(path, file)
	if err != nil {
		return 0, err
	}
	contents = strings.TrimSpace(contents)
	if contents == "max" {
		return math.MaxInt64, nil
	}

	res, err := strconv.ParseInt(contents, 10, 64)
	if err != nil {
		return res, &ParseError{Path: path, File: file, Err: err}
	}
	return res, nil
}

// GetCgroupParamString reads a string from the specified cgroup file.
func GetCgroupParamString(path, file string) (string, error) {
	contents, err := cgroups.ReadFile(path, file)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(contents), nil
}
<|endoftext|>


<commit_before>// goforever - processes management
// Copyright (c) 2013 Garrett Woodworth (https://github.com/gwoo).

// sphere-director - Ninja processes management
// Copyright (c) 2014 Ninja Blocks Inc. (https://github.com/ninjablocks).

package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/gwoo/greq"
	"github.com/mitchellh/osext"
	"github.com/ninjasphere/go-ninja/logger"
)

var config *Config
var daemon *Process
var log = logger.GetLogger("Director")

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
	usage := `
Commands
  list              List processes.
  show [name]       Show main proccess or named process.
  start [name]      Start main proccess or named process.
  stop [name]       Stop main proccess or named process.
  restart [name]    Restart main proccess or named process.
`
	fmt.Fprintln(os.Stderr, usage)
}

func init() {
	flag.Usage = Usage
	flag.Parse()
	setConfig()

	// Make sure we use the same binary for the daemon
	currentBinary, err := osext.Executable()
	log.Infof("Binary: %s", currentBinary)
	if err != nil {
		currentBinary = "/opt/ninjablocks/bin/sphere-director"
		log.Infof("Couldn't get current binary. ? Setting to '%s'. error: %s", currentBinary, err)
	}

	daemon = &Process{
		ID: "sphere-director",
		Info: packageJson{
			Name:        "Sphere Director",
			Description: "Manages the processes that make up Ninja Sphere.",
		},
		Args:    []string{},
		Command: currentBinary,
		Pidfile: Pidfile(homeDir + "sphere-director.pid"),
		Logfile: homeDir + "sphere-director.log",
		Errfile: homeDir + "sphere-director.log",
	}
}

func main() {

	if len(flag.Args()) > 0 {
		fmt.Printf("%s", Cli())
		return
	}
	if len(flag.Args()) == 0 {
		RunDaemon()
		Mqtt()
		HttpServer()
	}
}

func Cli() string {
	var o []byte
	var err error
	sub := flag.Arg(0)
	name := flag.Arg(1)
	req := greq.New(host(), true)
	if sub == "list" {
		o, _, err = req.Get("/")
	}
	if name == "" {
		if sub == "start" {
			daemon.Args = append(daemon.Args, os.Args[2:]...)
			return daemon.start()
		}
		_, _, err = daemon.find()
		if err != nil {
			return fmt.Sprintf("Error: %s.\n", err)
		}
		if sub == "show" {
			return fmt.Sprintf("%s.\n", daemon.String())
		}
		if sub == "stop" {
			message := daemon.stop()
			return message
		}
		if sub == "restart" {
			ch, message := daemon.restart()
			fmt.Print(message)
			return fmt.Sprintf("%s\n", <-ch)
		}
	}
	if name != "" {
		path := fmt.Sprintf("/%s", name)
		switch sub {
		case "show":
			o, _, err = req.Get(path)
		case "start":
			o, _, err = req.Post(path, nil)
		case "stop":
			o, _, err = req.Delete(path)
		case "restart":
			o, _, err = req.Put(path, nil)
		}
	}
	if err != nil {
		fmt.Printf("Process error: %s", err)
	}
	return fmt.Sprintf("%s\n", o)
}

func RunDaemon() {
	daemon.children = config.Processes
	//daemon.run()
}

func setConfig() {
	var err error
	config, err = LoadConfig()
	if err != nil {
		log.Fatalf("%s", err)
		return
	}
	config.FindProcesses()
}

func host() string {
	scheme := "https"
	if isHttps() == false {
		scheme = "http"
	}
	return fmt.Sprintf("%s://%s:%s@:%d",
		scheme, config.Username, config.Password, config.Port,
	)
}
<commit_msg>Update director pid file to the one upstart uses<commit_after>// goforever - processes management
// Copyright (c) 2013 Garrett Woodworth (https://github.com/gwoo).

// sphere-director - Ninja processes management
// Copyright (c) 2014 Ninja Blocks Inc. (https://github.com/ninjablocks).

package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/gwoo/greq"
	"github.com/mitchellh/osext"
	"github.com/ninjasphere/go-ninja/logger"
)

var config *Config
var daemon *Process
var log = logger.GetLogger("Director")

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
	usage := `
Commands
  list              List processes.
  show [name]       Show main proccess or named process.
  start [name]      Start main proccess or named process.
  stop [name]       Stop main proccess or named process.
  restart [name]    Restart main proccess or named process.
`
	fmt.Fprintln(os.Stderr, usage)
}

func init() {
	flag.Usage = Usage
	flag.Parse()
	setConfig()

	// Make sure we use the same binary for the daemon
	currentBinary, err := osext.Executable()
	log.Infof("Binary: %s", currentBinary)
	if err != nil {
		currentBinary = "/opt/ninjablocks/bin/sphere-director"
		log.Infof("Couldn't get current binary. ? Setting to '%s'. error: %s", currentBinary, err)
	}

	daemon = &Process{
		ID: "sphere-director",
		Info: packageJson{
			Name:        "Sphere Director",
			Description: "Manages the processes that make up Ninja Sphere.",
		},
		Args:    []string{},
		Command: currentBinary,
		Pidfile: Pidfile("/var/run/sphere-director.pid"),
		Logfile: homeDir + "sphere-director.log",
		Errfile: homeDir + "sphere-director.log",
	}
}

func main() {

	if len(flag.Args()) > 0 {
		fmt.Printf("%s", Cli())
		return
	}
	if len(flag.Args()) == 0 {
		RunDaemon()
		Mqtt()
		HttpServer()
	}
}

func Cli() string {
	var o []byte
	var err error
	sub := flag.Arg(0)
	name := flag.Arg(1)
	req := greq.New(host(), true)
	if sub == "list" {
		o, _, err = req.Get("/")
	}
	if name == "" {
		if sub == "start" {
			daemon.Args = append(daemon.Args, os.Args[2:]...)
			return daemon.start()
		}
		_, _, err = daemon.find()
		if err != nil {
			return fmt.Sprintf("Error: %s.\n", err)
		}
		if sub == "show" {
			return fmt.Sprintf("%s.\n", daemon.String())
		}
		if sub == "stop" {
			message := daemon.stop()
			return message
		}
		if sub == "restart" {
			ch, message := daemon.restart()
			fmt.Print(message)
			return fmt.Sprintf("%s\n", <-ch)
		}
	}
	if name != "" {
		path := fmt.Sprintf("/%s", name)
		switch sub {
		case "show":
			o, _, err = req.Get(path)
		case "start":
			o, _, err = req.Post(path, nil)
		case "stop":
			o, _, err = req.Delete(path)
		case "restart":
			o, _, err = req.Put(path, nil)
		}
	}
	if err != nil {
		fmt.Printf("Process error: %s", err)
	}
	return fmt.Sprintf("%s\n", o)
}

func RunDaemon() {
	daemon.children = config.Processes
	//daemon.run()
}

func setConfig() {
	var err error
	config, err = LoadConfig()
	if err != nil {
		log.Fatalf("%s", err)
		return
	}
	config.FindProcesses()
}

func host() string {
	scheme := "https"
	if isHttps() == false {
		scheme = "http"
	}
	return fmt.Sprintf("%s://%s:%s@:%d",
		scheme, config.Username, config.Password, config.Port,
	)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/zenazn/goji/web"
	"gopkg.in/mgo.v2"
)

var AddDeviceMissingDeviceIdError = ErrorJson{"POST to /devices must be a JSON with a DeviceId property."}

type SidewinderDirector struct {
	MongoDB          string
	session          *mgo.Session
	ApnsCommunicator *APNSCommunicator
}

func NewSidewinderDirector(mongoDB string, communicator *APNSCommunicator) (*SidewinderDirector, error) {
	session, err := mgo.Dial("mongo,localhost")
	if err != nil {
		return nil, err
	}
	return &SidewinderDirector{mongoDB, session, communicator}, nil
}

func (self *SidewinderDirector) Store() *SidewinderStore {
	return &SidewinderStore{self.MongoDB, self.session.Copy()}
}

func (self *SidewinderDirector) DatastoreInfo(context web.C, writer http.ResponseWriter, request *http.Request) error {
	session := self.Store().session

	buildInfo, err := session.BuildInfo()
	if err != nil {
		return fmt.Errorf("Could not connect to MongoDB.\n%v", err.Error())
	}
	writer.Header().Set("Content-Type", "application/json")

	databases, err := session.DatabaseNames()
	if err != nil {
		return fmt.Errorf("Could not retrieve database names.\n%v", err.Error())
	}

	dataStoreInfo := DatastoreInfo{buildInfo, session.LiveServers(), databases}

	return json.NewEncoder(writer).Encode(&dataStoreInfo)
}

func (self *SidewinderDirector) postDevice(context web.C, writer http.ResponseWriter, request *http.Request) error {
	sentJSON := decodeDeviceDocument(request)
	if sentJSON == nil {
		return writeJson(400, AddDeviceMissingDeviceIdError, writer)
	}
	recordWasCreated, err := self.Store().AddDevice(sentJSON.DeviceId)
	if err != nil {
		return err
	} else if recordWasCreated {
		return writeJson(201, sentJSON, writer)
	} else {
		return writeJson(200, sentJSON, writer)
	}
}

func decodeDeviceDocument(request *http.Request) *DeviceDocument {
	var sentJSON DeviceDocument
	if decodeErr := json.NewDecoder(request.Body).Decode(&sentJSON); decodeErr == nil && sentJSON.DeviceId != "" {
		return &sentJSON
	} else {
		return nil
	}
}

type DeviceHandler func(id string, writer http.ResponseWriter, request *http.Request) error

func (self DeviceHandler) ServeHTTPC(context web.C, writer http.ResponseWriter, request *http.Request) {
	deviceId := context.URLParams["id"]
	err := self(deviceId, writer, request)
	if err != nil {
		writeJson(500, ErrorJson{err.Error()}, writer)
	}
}

func (self *SidewinderDirector) deleteDevice(deviceId string, writer http.ResponseWriter, request *http.Request) error {
	result, err := self.Store().FindDevice(deviceId)
	if err != nil {
		return err
	}

	if err := self.Store().DeleteDevice(deviceId); err != nil {
		return err
	}

	return writeJson(200, result, writer)
}

func (self *SidewinderDirector) PostNotification(deviceId string, writer http.ResponseWriter, request *http.Request) error {
	var notification map[string]string
	if decodeErr := json.NewDecoder(request.Body).Decode(&notification); decodeErr != nil {
		return decodeErr
	}

	if err := self.ApnsCommunicator.sendPushNotification(deviceId, notification["Alert"]); err != nil {
		return err
	}
	return writeJson(201, notification, writer)
}

func (self *SidewinderDirector) CircleNotify(context web.C, writer http.ResponseWriter, request *http.Request) error {
	var buffer bytes.Buffer
	buffer.ReadFrom(request.Body)
	fmt.Printf("Sent body:\n%v", buffer.String())
	return nil
}
<commit_msg>Trying to read the sent JSON. Something seems to be missing at the moment.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/zenazn/goji/web"
	"gopkg.in/mgo.v2"
)

var AddDeviceMissingDeviceIdError = ErrorJson{"POST to /devices must be a JSON with a DeviceId property."}

type SidewinderDirector struct {
	MongoDB          string
	session          *mgo.Session
	ApnsCommunicator *APNSCommunicator
}

func NewSidewinderDirector(mongoDB string, communicator *APNSCommunicator) (*SidewinderDirector, error) {
	session, err := mgo.Dial("mongo,localhost")
	if err != nil {
		return nil, err
	}
	return &SidewinderDirector{mongoDB, session, communicator}, nil
}

func (self *SidewinderDirector) Store() *SidewinderStore {
	return &SidewinderStore{self.MongoDB, self.session.Copy()}
}

func (self *SidewinderDirector) DatastoreInfo(context web.C, writer http.ResponseWriter, request *http.Request) error {
	session := self.Store().session

	buildInfo, err := session.BuildInfo()
	if err != nil {
		return fmt.Errorf("Could not connect to MongoDB.\n%v", err.Error())
	}
	writer.Header().Set("Content-Type", "application/json")

	databases, err := session.DatabaseNames()
	if err != nil {
		return fmt.Errorf("Could not retrieve database names.\n%v", err.Error())
	}

	dataStoreInfo := DatastoreInfo{buildInfo, session.LiveServers(), databases}

	return json.NewEncoder(writer).Encode(&dataStoreInfo)
}

func (self *SidewinderDirector) postDevice(context web.C, writer http.ResponseWriter, request *http.Request) error {
	sentJSON := decodeDeviceDocument(request)
	if sentJSON == nil {
		return writeJson(400, AddDeviceMissingDeviceIdError, writer)
	}
	recordWasCreated, err := self.Store().AddDevice(sentJSON.DeviceId)
	if err != nil {
		return err
	} else if recordWasCreated {
		return writeJson(201, sentJSON, writer)
	} else {
		return writeJson(200, sentJSON, writer)
	}
}

func decodeDeviceDocument(request *http.Request) *DeviceDocument {
	var sentJSON DeviceDocument
	if decodeErr := json.NewDecoder(request.Body).Decode(&sentJSON); decodeErr == nil && sentJSON.DeviceId != "" {
		return &sentJSON
	} else {
		return nil
	}
}

type DeviceHandler func(id string, writer http.ResponseWriter, request *http.Request) error

func (self DeviceHandler) ServeHTTPC(context web.C, writer http.ResponseWriter, request *http.Request) {
	deviceId := context.URLParams["id"]
	err := self(deviceId, writer, request)
	if err != nil {
		writeJson(500, ErrorJson{err.Error()}, writer)
	}
}

func (self *SidewinderDirector) deleteDevice(deviceId string, writer http.ResponseWriter, request *http.Request) error {
	result, err := self.Store().FindDevice(deviceId)
	if err != nil {
		return err
	}

	if err := self.Store().DeleteDevice(deviceId); err != nil {
		return err
	}

	return writeJson(200, result, writer)
}

func (self *SidewinderDirector) PostNotification(deviceId string, writer http.ResponseWriter, request *http.Request) error {
	var notification map[string]string
	if decodeErr := json.NewDecoder(request.Body).Decode(&notification); decodeErr != nil {
		return decodeErr
	}

	if err := self.ApnsCommunicator.sendPushNotification(deviceId, notification["Alert"]); err != nil {
		return err
	}
	return writeJson(201, notification, writer)
}

func (self *SidewinderDirector) CircleNotify(context web.C, writer http.ResponseWriter, request *http.Request) error {
	var notification map[string]string
	if decodeErr := json.NewDecoder(request.Body).Decode(&notification); decodeErr != nil {
		return decodeErr
	}
	fmt.Printf("Sent body:\n%v", notification)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/sha1"
	"errors"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/zeebo/bencode"

	"github.com/rakoo/rakoshare/pkg/sharesession"
)

var (
	errNewFile    = errors.New("Got new file")
	errInvalidDir = errors.New("Invalid watched dir")
)

type state int

const (
	IDEM = iota
	CHANGED
)

type Watcher struct {
	session    *sharesession.Session
	watchedDir string
	lock       sync.Mutex

	PingNewTorrent chan string
}

func NewWatcher(session *sharesession.Session, watchedDir string, canWrite bool) (w *Watcher, err error) {
	w = &Watcher{
		session:        session,
		watchedDir:     watchedDir,
		PingNewTorrent: make(chan string),
	}

	if canWrite {
		go w.watch()
	}

	// Initialization, only if there is something in the dir
	if _, err := os.Stat(watchedDir); err != nil {
		return nil, err
	}

	dir, err := os.Open(watchedDir)
	if err != nil {
		return nil, err
	}

	names, err := dir.Readdirnames(1)
	if len(names) == 0 || err != nil && err != io.EOF {
		return nil, errInvalidDir
	}

	if err == nil {
		go func() {
			w.PingNewTorrent <- session.GetCurrentInfohash()
		}()
	}

	return
}

func (w *Watcher) watch() {
	var previousState, currentState state
	currentState = IDEM

	compareTime := w.session.GetLastModTime()

	for _ = range time.Tick(10 * time.Second) {
		w.lock.Lock()

		err := torrentWalk(w.watchedDir, func(path string, info os.FileInfo, perr error) (err error) {
			if info.ModTime().After(compareTime) {
				fmt.Printf("[newer] %s\n", path)
				return errNewFile
			}
			return
		})

		w.lock.Unlock()

		previousState = currentState

		if err == errNewFile {
			currentState = CHANGED
		} else if err == nil {
			currentState = IDEM
		} else {
			log.Println("Error while walking dir:", err)
		}

		compareTime = time.Now()

		if currentState == IDEM && previousState == CHANGED {
			// Note that we may be in the CHANGED state for multiple
			// iterations, such as when changes take more than 10 seconds to
			// finish. When we go back to "idle" state, we kick in the
			// metadata creation.

			// Block until we completely manage it. We will take
			// care of other changes in the next run of the loop.
			ih, err := w.torrentify()
			if err != nil {
				log.Printf("Couldn't torrentify: ", err)
				continue
			}
			w.PingNewTorrent <- ih
		}
	}
}

func (w *Watcher) torrentify() (ih string, err error) {
	w.lock.Lock()
	defer w.lock.Unlock()

	meta, err := createMeta(w.watchedDir)
	if err != nil {
		log.Println(err)
		return
	}

	err = w.saveMetainfo(meta)

	return meta.InfoHash, err
}

func (w *Watcher) saveMetainfo(meta *MetaInfo) error {
	var buf bytes.Buffer
	err := bencode.NewEncoder(&buf).Encode(meta)
	if err != nil {
		return err
	}
	w.session.SaveTorrent(buf.Bytes(), meta.InfoHash, time.Now().Format(time.RFC3339))
	return nil
}

func createMeta(dir string) (meta *MetaInfo, err error) {
	blockSize := int64(1 << 20) // 1MiB

	fileDicts := make([]*FileDict, 0)

	hasher := NewBlockHasher(blockSize)
	err = torrentWalk(dir, func(path string, info os.FileInfo, perr error) (err error) {
		if perr != nil {
			return perr
		}

		f, err := os.Open(path)
		if err != nil {
			return errors.New(fmt.Sprintf("Couldn't open %s for hashing: %s\n", path, err))
		}
		defer f.Close()

		_, err = io.Copy(hasher, f)
		if err != nil {
			log.Printf("Couldn't hash %s: %s\n", path, err)
			return err
		}

		relPath, err := filepath.Rel(dir, path)
		if err != nil {
			return
		}

		fileDict := &FileDict{
			Length: info.Size(),
			Path:   strings.Split(relPath, string(os.PathSeparator)),
		}
		fileDicts = append(fileDicts, fileDict)

		return
	})
	if err != nil {
		return
	}

	end := hasher.Close()
	if end != nil {
		return
	}

	meta = &MetaInfo{
		Info: &InfoDict{
			Pieces:      string(hasher.Pieces),
			PieceLength: blockSize,
			Private:     0,
			Name:        "rakoshare",
			Files:       fileDicts,
		},
	}

	hash := sha1.New()
	err = bencode.NewEncoder(hash).Encode(meta.Info)
	if err != nil {
		return
	}
	meta.InfoHash = string(hash.Sum(nil))

	return
}

type BlockHasher struct {
	sha1er    hash.Hash
	left      int64
	blockSize int64
	Pieces    []byte
}

func NewBlockHasher(blockSize int64) (h *BlockHasher) {
	return &BlockHasher{
		blockSize: blockSize,
		sha1er:    sha1.New(),
		left:      blockSize,
	}
}

// You shouldn't use this one
func (h *BlockHasher) Write(p []byte) (n int, err error) {
	n2, err := h.ReadFrom(bytes.NewReader(p))
	return int(n2), err
}

func (h *BlockHasher) ReadFrom(rd io.Reader) (n int64, err error) {
	var stop bool

	for {
		if h.left > 0 {
			thisN, err := io.CopyN(h.sha1er, rd, h.left)
			if err != nil {
				if err == io.EOF {
					stop = true
				} else {
					return n, err
				}
			}
			h.left -= thisN
			n += thisN
		}
		if h.left == 0 {
			h.Pieces = h.sha1er.Sum(h.Pieces)
			h.sha1er = sha1.New()
			h.left = h.blockSize
		}

		if stop {
			break
		}
	}

	return
}

func (h *BlockHasher) Close() (err error) {
	if h.left == h.blockSize {
		// We're at the end of a blockSize, we don't have any buffered data
		return
	}
	h.Pieces = h.sha1er.Sum(h.Pieces)
	return
}

func torrentWalk(root string, fn filepath.WalkFunc) (err error) {
	return filepath.Walk(root, func(path string, info os.FileInfo, perr error) (err error) {
		if !info.Mode().IsRegular() {
			return
		}

		// Torrents can't have empty files
		if info.Size() == 0 {
			return
		}

		if strings.HasPrefix(filepath.Base(path), ".") {
			return
		}

		if filepath.Ext(path) == ".part" {
			return
		}

		return fn(path, info, perr)
	})
}
<commit_msg>Allow empty dirs<commit_after>package main

import (
	"bytes"
	"crypto/sha1"
	"errors"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/zeebo/bencode"

	"github.com/rakoo/rakoshare/pkg/sharesession"
)

var (
	errNewFile    = errors.New("Got new file")
	errInvalidDir = errors.New("Invalid watched dir")
)

type state int

const (
	IDEM = iota
	CHANGED
)

type Watcher struct {
	session    *sharesession.Session
	watchedDir string
	lock       sync.Mutex

	PingNewTorrent chan string
}

func NewWatcher(session *sharesession.Session, watchedDir string, canWrite bool) (w *Watcher, err error) {
	w = &Watcher{
		session:        session,
		watchedDir:     watchedDir,
		PingNewTorrent: make(chan string),
	}

	if canWrite {
		go w.watch()
	}

	// Initialization, only if there is something in the dir
	if _, err := os.Stat(watchedDir); err != nil {
		return nil, err
	}

	st, err := os.Stat(watchedDir)
	if err != nil {
		return nil, err
	}
	if !st.IsDir() {
		return nil, errInvalidDir
	}

	go func() {
		w.PingNewTorrent <- session.GetCurrentInfohash()
	}()

	return
}

func (w *Watcher) watch() {
	var previousState, currentState state
	currentState = IDEM

	compareTime := w.session.GetLastModTime()

	for _ = range time.Tick(10 * time.Second) {
		w.lock.Lock()

		err := torrentWalk(w.watchedDir, func(path string, info os.FileInfo, perr error) (err error) {
			if info.ModTime().After(compareTime) {
				fmt.Printf("[newer] %s\n", path)
				return errNewFile
			}
			return
		})

		w.lock.Unlock()

		previousState = currentState

		if err == errNewFile {
			currentState = CHANGED
		} else if err == nil {
			currentState = IDEM
		} else {
			log.Println("Error while walking dir:", err)
		}

		compareTime = time.Now()

		if currentState == IDEM && previousState == CHANGED {
			// Note that we may be in the CHANGED state for multiple
			// iterations, such as when changes take more than 10 seconds to
			// finish. When we go back to "idle" state, we kick in the
			// metadata creation.

			// Block until we completely manage it. We will take
			// care of other changes in the next run of the loop.
			ih, err := w.torrentify()
			if err != nil {
				log.Printf("Couldn't torrentify: ", err)
				continue
			}
			w.PingNewTorrent <- ih
		}
	}
}

func (w *Watcher) torrentify() (ih string, err error) {
	w.lock.Lock()
	defer w.lock.Unlock()

	meta, err := createMeta(w.watchedDir)
	if err != nil {
		log.Println(err)
		return
	}

	err = w.saveMetainfo(meta)

	return meta.InfoHash, err
}

func (w *Watcher) saveMetainfo(meta *MetaInfo) error {
	var buf bytes.Buffer
	err := bencode.NewEncoder(&buf).Encode(meta)
	if err != nil {
		return err
	}
	w.session.SaveTorrent(buf.Bytes(), meta.InfoHash, time.Now().Format(time.RFC3339))
	return nil
}

func createMeta(dir string) (meta *MetaInfo, err error) {
	blockSize := int64(1 << 20) // 1MiB

	fileDicts := make([]*FileDict, 0)

	hasher := NewBlockHasher(blockSize)
	err = torrentWalk(dir, func(path string, info os.FileInfo, perr error) (err error) {
		if perr != nil {
			return perr
		}

		f, err := os.Open(path)
		if err != nil {
			return errors.New(fmt.Sprintf("Couldn't open %s for hashing: %s\n", path, err))
		}
		defer f.Close()

		_, err = io.Copy(hasher, f)
		if err != nil {
			log.Printf("Couldn't hash %s: %s\n", path, err)
			return err
		}

		relPath, err := filepath.Rel(dir, path)
		if err != nil {
			return
		}

		fileDict := &FileDict{
			Length: info.Size(),
			Path:   strings.Split(relPath, string(os.PathSeparator)),
		}
		fileDicts = append(fileDicts, fileDict)

		return
	})
	if err != nil {
		return
	}

	end := hasher.Close()
	if end != nil {
		return
	}

	meta = &MetaInfo{
		Info: &InfoDict{
			Pieces:      string(hasher.Pieces),
			PieceLength: blockSize,
			Private:     0,
			Name:        "rakoshare",
			Files:       fileDicts,
		},
	}

	hash := sha1.New()
	err = bencode.NewEncoder(hash).Encode(meta.Info)
	if err != nil {
		return
	}
	meta.InfoHash = string(hash.Sum(nil))

	return
}

type BlockHasher struct {
	sha1er    hash.Hash
	left      int64
	blockSize int64
	Pieces    []byte
}

func NewBlockHasher(blockSize int64) (h *BlockHasher) {
	return &BlockHasher{
		blockSize: blockSize,
		sha1er:    sha1.New(),
		left:      blockSize,
	}
}

// You shouldn't use this one
func (h *BlockHasher) Write(p []byte) (n int, err error) {
	n2, err := h.ReadFrom(bytes.NewReader(p))
	return int(n2), err
}

func (h *BlockHasher) ReadFrom(rd io.Reader) (n int64, err error) {
	var stop bool

	for {
		if h.left > 0 {
			thisN, err := io.CopyN(h.sha1er, rd, h.left)
			if err != nil {
				if err == io.EOF {
					stop = true
				} else {
					return n, err
				}
			}
			h.left -= thisN
			n += thisN
		}
		if h.left == 0 {
			h.Pieces = h.sha1er.Sum(h.Pieces)
			h.sha1er = sha1.New()
			h.left = h.blockSize
		}

		if stop {
			break
		}
	}

	return
}

func (h *BlockHasher) Close() (err error) {
	if h.left == h.blockSize {
		// We're at the end of a blockSize, we don't have any buffered data
		return
	}
	h.Pieces = h.sha1er.Sum(h.Pieces)
	return
}

func torrentWalk(root string, fn filepath.WalkFunc) (err error) {
	return filepath.Walk(root, func(path string, info os.FileInfo, perr error) (err error) {
		if !info.Mode().IsRegular() {
			return
		}

		// Torrents can't have empty files
		if info.Size() == 0 {
			return
		}

		if strings.HasPrefix(filepath.Base(path), ".") {
			return
		}

		if filepath.Ext(path) == ".part" {
			return
		}

		return fn(path, info, perr)
	})
}
<|endoftext|>