<commit_before>package userPermission

import (
	"errors"
	"net/http"
	"github.com/ewhal/nyaa/model"
	"github.com/ewhal/nyaa/service/user"
	"github.com/ewhal/nyaa/util/log"
)

// HasAdmin checks that user has an admin permission.
func HasAdmin(user *model.User) bool {
	name := "admin"
	for _, role := range user.Roles {
		log.Debugf("HasAdmin role.Name : %s", role.Name)
		if role.Name == name {
			return true
		}
	}
	return false
}

// CurrentOrAdmin check that user has admin permission or user is the current user.
func CurrentOrAdmin(user *model.User, userId uint) bool {
	log.Debugf("user.Id == userId %d %d %s", user.Id, userId, user.Id == userId)
	return (HasAdmin(user) || user.Id == userId)
}

// CurrentUserIdentical check that userId is same as current user's Id.
func CurrentUserIdentical(r *http.Request, userId uint) (bool, error) {
	currentUser, err := userService.CurrentUser(r)
	if err != nil {
		return false, errors.New("Auth failed.")
	}
	if currentUser.Id != userId {
		return false, errors.New("User is not identical.")
	}

	return true, nil
}
<commit_msg>Try to fix travis<commit_after>package userPermission

import (
	"errors"
	"net/http"
	"github.com/ewhal/nyaa/model"
	"github.com/ewhal/nyaa/service/user"
	"github.com/ewhal/nyaa/util/log"
)

// HasAdmin checks that user has an admin permission.
func HasAdmin(user *model.User) bool {
	return user.Status == 2
}

// CurrentOrAdmin check that user has admin permission or user is the current user.
func CurrentOrAdmin(user *model.User, userId uint) bool {
	log.Debugf("user.Id == userId %d %d %s", user.Id, userId, user.Id == userId)
	return (HasAdmin(user) || user.Id == userId)
}

// CurrentUserIdentical check that userId is same as current user's Id.
func CurrentUserIdentical(r *http.Request, userId uint) (bool, error) {
	currentUser, err := userService.CurrentUser(r)
	if err != nil {
		return false, errors.New("Auth failed.")
	}
	if currentUser.Id != userId {
		return false, errors.New("User is not identical.")
	}

	return true, nil
}
<|endoftext|>


<commit_before>// +build !production

package main

import (
	"flag"
	"fmt"
	"path/filepath"
	"strings"

	"github.com/ostrost/ostent/share/assets"
)

const packageName = "share/assets"

func main() {
	flag.Parse()
	target := flag.Arg(0)

	var lines []string
	for _, line := range assets.JsAssetNames(false) {
		lines = append(lines, filepath.Join(packageName, line))
	}

	if target != "" {
		fmt.Printf("%s: %s\n", target, strings.Join(lines, " "))
		return
	}
	for _, line := range lines {
		fmt.Println(line)
	}
}
<commit_msg>jsmakerule: forgotten JsAssetNames use<commit_after>// +build !production

package main

import (
	"flag"
	"fmt"
	"path/filepath"
	"strings"

	"github.com/ostrost/ostent/assets"
	shareassets "github.com/ostrost/ostent/share/assets"
)

const packageName = "share/assets"

func main() {
	flag.Parse()
	target := flag.Arg(0)

	var lines []string
	for _, line := range assets.JsAssetNames(shareassets.AssetNames(), false) {
		lines = append(lines, filepath.Join(packageName, line))
	}

	if target != "" {
		fmt.Printf("%s: %s\n", target, strings.Join(lines, " "))
		return
	}
	for _, line := range lines {
		fmt.Println(line)
	}
}
<|endoftext|>


<commit_before>// Package up implements a plugin to check if a web host is up or down.
package up

import (
	"fmt"
	bot "github.com/StalkR/goircbot"
	"net"
	"net/http"
	"regexp"
	"strings"
	"time"
)

func timeoutDialer(d time.Duration) func(net, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		return net.DialTimeout(netw, addr, d)
	}
}

// Probe gets an URL and returns a boolean if it worked within 3 seconds.
func Probe(url string) bool {
	client := &http.Client{
		Transport: &http.Transport{
			Dial: timeoutDialer(3 * time.Second),
		},
	}
	resp, err := client.Get(url)
	if err != nil {
		return false
	}
	defer resp.Body.Close()
	return true
}

func Up(b *bot.Bot, e *bot.Event) {
	arg := strings.TrimSpace(e.Args)
	if len(arg) == 0 {
		return
	}
	matched, err := regexp.Match("^[\\w._-]+$", []byte(arg))
	if err != nil || !matched {
		return
	}
	url := fmt.Sprintf("http://%s", arg)
	if !Probe(url) {
		b.Conn.Privmsg(e.Target, fmt.Sprintf("%s is down from here.", url))
		return
	}
	b.Conn.Privmsg(e.Target, fmt.Sprintf("%s is up from here.", url))
}

// Register registers the plugin with a bot.
func Register(b *bot.Bot) {
	b.AddCommand("up", bot.Command{
		Help:    "check if a web host is up or down",
		Handler: Up,
		Pub:     true,
		Priv:    true,
		Hidden:  false})
}
<commit_msg>up: disable TLS verification<commit_after>// Package up implements a plugin to check if a web host is up or down.
package up

import (
	"crypto/tls"
	"fmt"
	bot "github.com/StalkR/goircbot"
	"net"
	"net/http"
	"regexp"
	"strings"
	"time"
)

func timeoutDialer(d time.Duration) func(net, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		return net.DialTimeout(netw, addr, d)
	}
}

// Probe gets an URL and returns a boolean if it worked within 3 seconds.
func Probe(url string) bool {
	client := &http.Client{
		Transport: &http.Transport{
			Dial:            timeoutDialer(3 * time.Second),
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}
	resp, err := client.Get(url)
	if err != nil {
		return false
	}
	defer resp.Body.Close()
	return true
}

func Up(b *bot.Bot, e *bot.Event) {
	arg := strings.TrimSpace(e.Args)
	if len(arg) == 0 {
		return
	}
	matched, err := regexp.Match("^[\\w._-]+$", []byte(arg))
	if err != nil || !matched {
		return
	}
	url := fmt.Sprintf("http://%s", arg)
	if !Probe(url) {
		b.Conn.Privmsg(e.Target, fmt.Sprintf("%s is down from here.", url))
		return
	}
	b.Conn.Privmsg(e.Target, fmt.Sprintf("%s is up from here.", url))
}

// Register registers the plugin with a bot.
func Register(b *bot.Bot) {
	b.AddCommand("up", bot.Command{
		Help:    "check if a web host is up or down",
		Handler: Up,
		Pub:     true,
		Priv:    true,
		Hidden:  false})
}
<|endoftext|>


<commit_before>package tq

import (
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/lfsapi"
	"github.com/git-lfs/git-lfs/tools"
)

const (
	BasicAdapterName   = "basic"
	defaultContentType = "application/octet-stream"
)

// Adapter for basic uploads (non resumable)
type basicUploadAdapter struct {
	*adapterBase
}

func (a *basicUploadAdapter) ClearTempStorage() error {
	// Should be empty already but also remove dir
	return os.RemoveAll(a.tempDir())
}

func (a *basicUploadAdapter) tempDir() string {
	// Must be dedicated to this adapter as deleted by ClearTempStorage
	d := filepath.Join(os.TempDir(), "git-lfs-basic-temp")
	if err := os.MkdirAll(d, 0755); err != nil {
		return os.TempDir()
	}
	return d
}

func (a *basicUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
	return nil, nil
}
func (a *basicUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
}

func (a *basicUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
	rel, err := t.Rel("upload")
	if err != nil {
		return err
	}
	if rel == nil {
		return errors.Errorf("No upload action for object: %s", t.Oid)
	}

	req, err := a.newHTTPRequest("PUT", rel)
	if err != nil {
		return err
	}

	if req.Header.Get("Transfer-Encoding") == "chunked" {
		req.TransferEncoding = []string{"chunked"}
	} else {
		req.Header.Set("Content-Length", strconv.FormatInt(t.Size, 10))
	}

	req.ContentLength = t.Size

	f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
	if err != nil {
		return errors.Wrap(err, "basic upload")
	}
	defer f.Close()

	if err := setContentTypeFor(req, f); err != nil {
		return err
	}

	// Ensure progress callbacks made while uploading
	// Wrap callback to give name context
	ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		if cb != nil {
			return cb(t.Name, totalSize, readSoFar, readSinceLast)
		}
		return nil
	}

	cbr := tools.NewBodyWithCallback(f, t.Size, ccb)
	var reader lfsapi.ReadSeekCloser = cbr

	// Signal auth was ok on first read; this frees up other workers to start
	if authOkFunc != nil {
		reader = newStartCallbackReader(reader, func() error {
			authOkFunc()
			return nil
		})
	}

	req.Body = reader

	req = a.apiClient.LogRequest(req, "lfs.data.upload")
	res, err := a.doHTTP(t, req)
	if err != nil {
		// We're about to return a retriable error, meaning that this
		// transfer will either be retried, or it will fail.
		//
		// Either way, let's decrement the number of bytes that we've
		// read _so far_, so that the next iteration doesn't re-transfer
		// those bytes, according to the progress meter.
		if perr := cbr.ResetProgress(); perr != nil {
			err = errors.Wrap(err, perr.Error())
		}

		return errors.NewRetriableError(err)
	}

	// A status code of 403 likely means that an authentication token for the
	// upload has expired. This can be safely retried.
	if res.StatusCode == 403 {
		err = errors.New("http: received status 403")
		return errors.NewRetriableError(err)
	}

	if res.StatusCode > 299 {
		return errors.Wrapf(nil, "Invalid status for %s %s: %d",
			req.Method,
			strings.SplitN(req.URL.String(), "?", 2)[0],
			res.StatusCode,
		)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return verifyUpload(a.apiClient, a.remote, t)
}

// startCallbackReader is a reader wrapper which calls a function as soon as the
// first Read() call is made. This callback is only made once
type startCallbackReader struct {
	cb     func() error
	cbDone bool
	lfsapi.ReadSeekCloser
}

func (s *startCallbackReader) Read(p []byte) (n int, err error) {
	if !s.cbDone && s.cb != nil {
		if err := s.cb(); err != nil {
			return 0, err
		}
		s.cbDone = true
	}
	return s.ReadSeekCloser.Read(p)
}
func newStartCallbackReader(r lfsapi.ReadSeekCloser, cb func() error) *startCallbackReader {
	return &startCallbackReader{
		ReadSeekCloser: r,
		cb:             cb,
	}
}

func configureBasicUploadAdapter(m *Manifest) {
	m.RegisterNewAdapterFunc(BasicAdapterName, Upload, func(name string, dir Direction) Adapter {
		switch dir {
		case Upload:
			bu := &basicUploadAdapter{newAdapterBase(m.fs, name, dir, nil)}
			// self implements impl
			bu.transferImpl = bu
			return bu
		case Download:
			panic("Should never ask this func for basic download")
		}
		return nil
	})
}

func setContentTypeFor(req *http.Request, r io.ReadSeeker) error {
	if len(req.Header.Get("Content-Type")) != 0 {
		return nil
	}

	buffer := make([]byte, 512)
	n, err := r.Read(buffer)
	if err != nil && err != io.EOF {
		return errors.Wrap(err, "content type detect")
	}

	contentType := http.DetectContentType(buffer[:n])
	if _, err := r.Seek(0, 0); err != nil {
		return errors.Wrap(err, "content type rewind")
	}

	if contentType == "" {
		contentType = defaultContentType
	}

	req.Header.Set("Content-Type", contentType)
	return nil
}
<commit_msg>tq/basic_upload.go: move 'setContentTypeFor()' up<commit_after>package tq

import (
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/lfsapi"
	"github.com/git-lfs/git-lfs/tools"
)

const (
	BasicAdapterName   = "basic"
	defaultContentType = "application/octet-stream"
)

// Adapter for basic uploads (non resumable)
type basicUploadAdapter struct {
	*adapterBase
}

func (a *basicUploadAdapter) ClearTempStorage() error {
	// Should be empty already but also remove dir
	return os.RemoveAll(a.tempDir())
}

func (a *basicUploadAdapter) tempDir() string {
	// Must be dedicated to this adapter as deleted by ClearTempStorage
	d := filepath.Join(os.TempDir(), "git-lfs-basic-temp")
	if err := os.MkdirAll(d, 0755); err != nil {
		return os.TempDir()
	}
	return d
}

func (a *basicUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
	return nil, nil
}
func (a *basicUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
}

func (a *basicUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
	rel, err := t.Rel("upload")
	if err != nil {
		return err
	}
	if rel == nil {
		return errors.Errorf("No upload action for object: %s", t.Oid)
	}

	req, err := a.newHTTPRequest("PUT", rel)
	if err != nil {
		return err
	}

	if req.Header.Get("Transfer-Encoding") == "chunked" {
		req.TransferEncoding = []string{"chunked"}
	} else {
		req.Header.Set("Content-Length", strconv.FormatInt(t.Size, 10))
	}

	req.ContentLength = t.Size

	f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
	if err != nil {
		return errors.Wrap(err, "basic upload")
	}
	defer f.Close()

	if err := setContentTypeFor(req, f); err != nil {
		return err
	}

	// Ensure progress callbacks made while uploading
	// Wrap callback to give name context
	ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		if cb != nil {
			return cb(t.Name, totalSize, readSoFar, readSinceLast)
		}
		return nil
	}

	cbr := tools.NewBodyWithCallback(f, t.Size, ccb)
	var reader lfsapi.ReadSeekCloser = cbr

	// Signal auth was ok on first read; this frees up other workers to start
	if authOkFunc != nil {
		reader = newStartCallbackReader(reader, func() error {
			authOkFunc()
			return nil
		})
	}

	req.Body = reader

	req = a.apiClient.LogRequest(req, "lfs.data.upload")
	res, err := a.doHTTP(t, req)
	if err != nil {
		// We're about to return a retriable error, meaning that this
		// transfer will either be retried, or it will fail.
		//
		// Either way, let's decrement the number of bytes that we've
		// read _so far_, so that the next iteration doesn't re-transfer
		// those bytes, according to the progress meter.
		if perr := cbr.ResetProgress(); perr != nil {
			err = errors.Wrap(err, perr.Error())
		}

		return errors.NewRetriableError(err)
	}

	// A status code of 403 likely means that an authentication token for the
	// upload has expired. This can be safely retried.
	if res.StatusCode == 403 {
		err = errors.New("http: received status 403")
		return errors.NewRetriableError(err)
	}

	if res.StatusCode > 299 {
		return errors.Wrapf(nil, "Invalid status for %s %s: %d",
			req.Method,
			strings.SplitN(req.URL.String(), "?", 2)[0],
			res.StatusCode,
		)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return verifyUpload(a.apiClient, a.remote, t)
}

func setContentTypeFor(req *http.Request, r io.ReadSeeker) error {
	if len(req.Header.Get("Content-Type")) != 0 {
		return nil
	}

	buffer := make([]byte, 512)
	n, err := r.Read(buffer)
	if err != nil && err != io.EOF {
		return errors.Wrap(err, "content type detect")
	}

	contentType := http.DetectContentType(buffer[:n])
	if _, err := r.Seek(0, 0); err != nil {
		return errors.Wrap(err, "content type rewind")
	}

	if contentType == "" {
		contentType = defaultContentType
	}

	req.Header.Set("Content-Type", contentType)
	return nil
}

// startCallbackReader is a reader wrapper which calls a function as soon as the
// first Read() call is made. This callback is only made once
type startCallbackReader struct {
	cb     func() error
	cbDone bool
	lfsapi.ReadSeekCloser
}

func (s *startCallbackReader) Read(p []byte) (n int, err error) {
	if !s.cbDone && s.cb != nil {
		if err := s.cb(); err != nil {
			return 0, err
		}
		s.cbDone = true
	}
	return s.ReadSeekCloser.Read(p)
}
func newStartCallbackReader(r lfsapi.ReadSeekCloser, cb func() error) *startCallbackReader {
	return &startCallbackReader{
		ReadSeekCloser: r,
		cb:             cb,
	}
}

func configureBasicUploadAdapter(m *Manifest) {
	m.RegisterNewAdapterFunc(BasicAdapterName, Upload, func(name string, dir Direction) Adapter {
		switch dir {
		case Upload:
			bu := &basicUploadAdapter{newAdapterBase(m.fs, name, dir, nil)}
			// self implements impl
			bu.transferImpl = bu
			return bu
		case Download:
			panic("Should never ask this func for basic download")
		}
		return nil
	})
}
<|endoftext|>


<commit_before>package tracker

import (
	"runtime"
	"sync"
	"time"

	"github.com/chihaya/chihaya/stats"
	"github.com/chihaya/chihaya/tracker/models"
)

type Storage struct {
	users  map[string]*models.User
	usersM sync.RWMutex

	torrents  map[string]*models.Torrent
	torrentsM sync.RWMutex

	clients  map[string]bool
	clientsM sync.RWMutex
}

func NewStorage() *Storage {
	return &Storage{
		users:    make(map[string]*models.User),
		torrents: make(map[string]*models.Torrent),
		clients:  make(map[string]bool),
	}
}

func (s *Storage) TouchTorrent(infohash string) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.LastAction = time.Now().Unix()

	return nil
}

func (s *Storage) FindTorrent(infohash string) (*models.Torrent, error) {
	s.torrentsM.RLock()
	defer s.torrentsM.RUnlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return nil, models.ErrTorrentDNE
	}

	return &*torrent, nil
}

func (s *Storage) PutTorrent(torrent *models.Torrent) {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	s.torrents[torrent.Infohash] = &*torrent
}

func (s *Storage) DeleteTorrent(infohash string) {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	delete(s.torrents, infohash)
}

func (s *Storage) IncrementTorrentSnatches(infohash string) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.Snatches++

	return nil
}

func (s *Storage) PutLeecher(infohash string, p *models.Peer) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.Leechers.Put(*p)

	return nil
}

func (s *Storage) DeleteLeecher(infohash string, p *models.Peer) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.Leechers.Delete(p.Key())

	return nil
}

func (s *Storage) PutSeeder(infohash string, p *models.Peer) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.Seeders.Put(*p)

	return nil
}

func (s *Storage) DeleteSeeder(infohash string, p *models.Peer) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.Seeders.Delete(p.Key())

	return nil
}

func (s *Storage) PurgeInactiveTorrent(infohash string) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	if torrent.PeerCount() == 0 {
		delete(s.torrents, infohash)
	}

	return nil
}

func (s *Storage) PurgeInactivePeers(purgeEmptyTorrents bool, before time.Time) error {
	unixtime := before.Unix()

	// Build a list of keys to process.
	s.torrentsM.RLock()
	index := 0
	keys := make([]string, len(s.torrents))

	for infohash := range s.torrents {
		keys[index] = infohash
		index++
	}
	s.torrentsM.RUnlock()

	// Process the keys while allowing other goroutines to run.
	for _, infohash := range keys {
		runtime.Gosched()

		s.torrentsM.Lock()
		torrent := s.torrents[infohash]

		if torrent == nil {
			// The torrent has already been deleted since keys were computed.
			continue
		}

		torrent.Seeders.Purge(unixtime)
		torrent.Leechers.Purge(unixtime)

		peers := torrent.PeerCount()
		s.torrentsM.Unlock()

		if purgeEmptyTorrents && peers == 0 {
			s.PurgeInactiveTorrent(infohash)
			stats.RecordEvent(stats.ReapedTorrent)
		}
	}

	return nil
}

func (s *Storage) FindUser(passkey string) (*models.User, error) {
	s.usersM.RLock()
	defer s.usersM.RUnlock()

	user, exists := s.users[passkey]
	if !exists {
		return nil, models.ErrUserDNE
	}

	return &*user, nil
}

func (s *Storage) PutUser(user *models.User) {
	s.usersM.Lock()
	defer s.usersM.Unlock()

	s.users[user.Passkey] = &*user
}

func (s *Storage) DeleteUser(passkey string) {
	s.usersM.Lock()
	defer s.usersM.Unlock()

	delete(s.users, passkey)
}

func (s *Storage) ClientApproved(peerID string) error {
	s.clientsM.RLock()
	defer s.clientsM.RUnlock()

	_, exists := s.clients[peerID]
	if !exists {
		return models.ErrClientUnapproved
	}

	return nil
}

func (s *Storage) PutClient(peerID string) {
	s.clientsM.Lock()
	defer s.clientsM.Unlock()

	s.clients[peerID] = true
}

func (s *Storage) DeleteClient(peerID string) {
	s.clientsM.Lock()
	defer s.clientsM.Unlock()

	delete(s.clients, peerID)
}
<commit_msg>Unlock torrent mutex when continuing purge loop.<commit_after>package tracker

import (
	"runtime"
	"sync"
	"time"

	"github.com/chihaya/chihaya/stats"
	"github.com/chihaya/chihaya/tracker/models"
)

type Storage struct {
	users  map[string]*models.User
	usersM sync.RWMutex

	torrents  map[string]*models.Torrent
	torrentsM sync.RWMutex

	clients  map[string]bool
	clientsM sync.RWMutex
}

func NewStorage() *Storage {
	return &Storage{
		users:    make(map[string]*models.User),
		torrents: make(map[string]*models.Torrent),
		clients:  make(map[string]bool),
	}
}

func (s *Storage) TouchTorrent(infohash string) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.LastAction = time.Now().Unix()

	return nil
}

func (s *Storage) FindTorrent(infohash string) (*models.Torrent, error) {
	s.torrentsM.RLock()
	defer s.torrentsM.RUnlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return nil, models.ErrTorrentDNE
	}

	return &*torrent, nil
}

func (s *Storage) PutTorrent(torrent *models.Torrent) {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	s.torrents[torrent.Infohash] = &*torrent
}

func (s *Storage) DeleteTorrent(infohash string) {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	delete(s.torrents, infohash)
}

func (s *Storage) IncrementTorrentSnatches(infohash string) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.Snatches++

	return nil
}

func (s *Storage) PutLeecher(infohash string, p *models.Peer) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.Leechers.Put(*p)

	return nil
}

func (s *Storage) DeleteLeecher(infohash string, p *models.Peer) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.Leechers.Delete(p.Key())

	return nil
}

func (s *Storage) PutSeeder(infohash string, p *models.Peer) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.Seeders.Put(*p)

	return nil
}

func (s *Storage) DeleteSeeder(infohash string, p *models.Peer) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	torrent.Seeders.Delete(p.Key())

	return nil
}

func (s *Storage) PurgeInactiveTorrent(infohash string) error {
	s.torrentsM.Lock()
	defer s.torrentsM.Unlock()

	torrent, exists := s.torrents[infohash]
	if !exists {
		return models.ErrTorrentDNE
	}

	if torrent.PeerCount() == 0 {
		delete(s.torrents, infohash)
	}

	return nil
}

func (s *Storage) PurgeInactivePeers(purgeEmptyTorrents bool, before time.Time) error {
	unixtime := before.Unix()

	// Build a list of keys to process.
	s.torrentsM.RLock()
	index := 0
	keys := make([]string, len(s.torrents))

	for infohash := range s.torrents {
		keys[index] = infohash
		index++
	}
	s.torrentsM.RUnlock()

	// Process the keys while allowing other goroutines to run.
	for _, infohash := range keys {
		runtime.Gosched()

		s.torrentsM.Lock()
		torrent := s.torrents[infohash]

		if torrent == nil {
			// The torrent has already been deleted since keys were computed.
			s.torrentsM.Unlock()
			continue
		}

		torrent.Seeders.Purge(unixtime)
		torrent.Leechers.Purge(unixtime)

		peers := torrent.PeerCount()
		s.torrentsM.Unlock()

		if purgeEmptyTorrents && peers == 0 {
			s.PurgeInactiveTorrent(infohash)
			stats.RecordEvent(stats.ReapedTorrent)
		}
	}

	return nil
}

func (s *Storage) FindUser(passkey string) (*models.User, error) {
	s.usersM.RLock()
	defer s.usersM.RUnlock()

	user, exists := s.users[passkey]
	if !exists {
		return nil, models.ErrUserDNE
	}

	return &*user, nil
}

func (s *Storage) PutUser(user *models.User) {
	s.usersM.Lock()
	defer s.usersM.Unlock()

	s.users[user.Passkey] = &*user
}

func (s *Storage) DeleteUser(passkey string) {
	s.usersM.Lock()
	defer s.usersM.Unlock()

	delete(s.users, passkey)
}

func (s *Storage) ClientApproved(peerID string) error {
	s.clientsM.RLock()
	defer s.clientsM.RUnlock()

	_, exists := s.clients[peerID]
	if !exists {
		return models.ErrClientUnapproved
	}

	return nil
}

func (s *Storage) PutClient(peerID string) {
	s.clientsM.Lock()
	defer s.clientsM.Unlock()

	s.clients[peerID] = true
}

func (s *Storage) DeleteClient(peerID string) {
	s.clientsM.Lock()
	defer s.clientsM.Unlock()

	delete(s.clients, peerID)
}
<|endoftext|>


<commit_before>package testutil

import (
	"bufio"
	"bytes"
	"os"
	"os/exec"
	"strings"
	"text/template"
	"unicode"
)

// dedent is a helper function that trims repeated, leading spaces from several
// lines of text. This is useful for long, inline commands that are indented to
// match the surrounding code but should be interpreted without the space:
// e.g.:
// func() {
//   BashCmd(`
//     cat <<EOF
//     This is a test
//     EOF
//   `)
// }
// Should evaluate:
// $ cat <<EOF
// > This is a test
// > EOF
// not:
// $      cat <<EOF
// >      This is a test
// >      EOF  # doesn't terminate heredoc b/c of space
// such that the heredoc doesn't end properly
func dedent(cmd string) string {
	notSpace := func(r rune) bool {
		return !unicode.IsSpace(r)
	}

	prefix := "-" // non-space character indicates no prefix has been established
	s := bufio.NewScanner(strings.NewReader(cmd))
	var dedentedCmd bytes.Buffer
	for s.Scan() {
		i := strings.IndexFunc(s.Text(), notSpace)
		if i == -1 {
			continue // blank line (or spaces-only line)--ignore completely
		} else if prefix == "-" {
			prefix = s.Text()[:i] // first non-blank line, set prefix
			dedentedCmd.WriteString(s.Text()[i:])
			dedentedCmd.WriteRune('\n')
		} else if strings.HasPrefix(s.Text(), prefix) {
			dedentedCmd.WriteString(s.Text()[len(prefix):]) // remove prefix
			dedentedCmd.WriteRune('\n')
		} else {
			return cmd // no common prefix--break early
		}
	}
	return dedentedCmd.String()
}

// Cmd is a convenience function that replaces exec.Command. It's both shorter
// and it uses the current process's stderr as output for the command, which
// makes debugging failures much easier (i.e. you get an error message
// rather than "exit status 1")
func Cmd(name string, args ...string) *exec.Cmd {
	cmd := exec.Command(name, args...)
	cmd.Stderr = os.Stderr
	// for convenience, simulate hitting "enter" after any prompt. This can easily
	// be replaced
	cmd.Stdin = strings.NewReader("\n")
	cmd.Env = os.Environ()
	return cmd
}

// BashCmd is a convenience function that:
// 1. Performs a Go template substitution on 'cmd' using the strings in 'subs'
// 2. Returns a command that runs the result string from 1 as a Bash script
func BashCmd(cmd string, subs ...string) *exec.Cmd {
	if len(subs)%2 == 1 {
		panic("some variable does not have a corresponding value")
	}

	// copy 'subs' into a map
	subsMap := make(map[string]string)
	for i := 0; i < len(subs); i += 2 {
		subsMap[subs[i]] = subs[i+1]
	}

	// Warn users that they must install 'match' if they want to run tests with
	// this library, and enable 'pipefail' so that if any 'match' in a chain
	// fails, the whole command fails.
	buf := &bytes.Buffer{}
	buf.WriteString(`
set -e -o pipefail

# Try to ignore pipefail errors (encountered when writing to a closed pipe).
# Processes like 'yes' are essentially guaranteed to hit this, and because of
# -e -o pipefail they will crash the whole script. We need these options,
# though, for 'match' to work, so for now we work around pipefail errors on a
# cmd-by-cmd basis. See "The Infamous SIGPIPE Signal"
# http://www.tldp.org/LDP/lpg/node20.html
pipeerr=141 # typical error code returned by unix utils when SIGPIPE is raised
function yes {
	/usr/bin/yes || test "$?" -eq "${pipeerr}"
}
export -f yes # use in subshells too

which match >/dev/null || {
	echo "You must have 'match' installed to run these tests. Please run:" >&2
	echo "  cd ${GOPATH}/src/github.com/pachyderm/pachyderm/ && go install ./src/testing/match" >&2
	exit 1
}`)
	buf.WriteRune('\n')

	// do the substitution
	template.Must(template.New("").Parse(dedent(cmd))).Execute(buf, subsMap)
	res := exec.Command("/bin/bash")
	res.Stderr = os.Stderr
	// useful for debugging, but makes travis too noisy:
	// res.Stdout = os.Stdout
	res.Stdin = buf
	res.Env = os.Environ()
	return res
}
<commit_msg>Fixed installation instruction<commit_after>package testutil

import (
	"bufio"
	"bytes"
	"os"
	"os/exec"
	"strings"
	"text/template"
	"unicode"
)

// dedent is a helper function that trims repeated, leading spaces from several
// lines of text. This is useful for long, inline commands that are indented to
// match the surrounding code but should be interpreted without the space:
// e.g.:
// func() {
//   BashCmd(`
//     cat <<EOF
//     This is a test
//     EOF
//   `)
// }
// Should evaluate:
// $ cat <<EOF
// > This is a test
// > EOF
// not:
// $      cat <<EOF
// >      This is a test
// >      EOF  # doesn't terminate heredoc b/c of space
// such that the heredoc doesn't end properly
func dedent(cmd string) string {
	notSpace := func(r rune) bool {
		return !unicode.IsSpace(r)
	}

	prefix := "-" // non-space character indicates no prefix has been established
	s := bufio.NewScanner(strings.NewReader(cmd))
	var dedentedCmd bytes.Buffer
	for s.Scan() {
		i := strings.IndexFunc(s.Text(), notSpace)
		if i == -1 {
			continue // blank line (or spaces-only line)--ignore completely
		} else if prefix == "-" {
			prefix = s.Text()[:i] // first non-blank line, set prefix
			dedentedCmd.WriteString(s.Text()[i:])
			dedentedCmd.WriteRune('\n')
		} else if strings.HasPrefix(s.Text(), prefix) {
			dedentedCmd.WriteString(s.Text()[len(prefix):]) // remove prefix
			dedentedCmd.WriteRune('\n')
		} else {
			return cmd // no common prefix--break early
		}
	}
	return dedentedCmd.String()
}

// Cmd is a convenience function that replaces exec.Command. It's both shorter
// and it uses the current process's stderr as output for the command, which
// makes debugging failures much easier (i.e. you get an error message
// rather than "exit status 1")
func Cmd(name string, args ...string) *exec.Cmd {
	cmd := exec.Command(name, args...)
	cmd.Stderr = os.Stderr
	// for convenience, simulate hitting "enter" after any prompt. This can easily
	// be replaced
	cmd.Stdin = strings.NewReader("\n")
	cmd.Env = os.Environ()
	return cmd
}

// BashCmd is a convenience function that:
// 1. Performs a Go template substitution on 'cmd' using the strings in 'subs'
// 2. Returns a command that runs the result string from 1 as a Bash script
func BashCmd(cmd string, subs ...string) *exec.Cmd {
	if len(subs)%2 == 1 {
		panic("some variable does not have a corresponding value")
	}

	// copy 'subs' into a map
	subsMap := make(map[string]string)
	for i := 0; i < len(subs); i += 2 {
		subsMap[subs[i]] = subs[i+1]
	}

	// Warn users that they must install 'match' if they want to run tests with
	// this library, and enable 'pipefail' so that if any 'match' in a chain
	// fails, the whole command fails.
	buf := &bytes.Buffer{}
	buf.WriteString(`
set -e -o pipefail

# Try to ignore pipefail errors (encountered when writing to a closed pipe).
# Processes like 'yes' are essentially guaranteed to hit this, and because of
# -e -o pipefail they will crash the whole script. We need these options,
# though, for 'match' to work, so for now we work around pipefail errors on a
# cmd-by-cmd basis. See "The Infamous SIGPIPE Signal"
# http://www.tldp.org/LDP/lpg/node20.html
pipeerr=141 # typical error code returned by unix utils when SIGPIPE is raised
function yes {
	/usr/bin/yes || test "$?" -eq "${pipeerr}"
}
export -f yes # use in subshells too

which match >/dev/null || {
	echo "You must have 'match' installed to run these tests. Please run:" >&2
	echo "  go install ./src/testing/match" >&2
	exit 1
}`)
	buf.WriteRune('\n')

	// do the substitution
	template.Must(template.New("").Parse(dedent(cmd))).Execute(buf, subsMap)
	res := exec.Command("/bin/bash")
	res.Stderr = os.Stderr
	// useful for debugging, but makes travis too noisy:
	// res.Stdout = os.Stdout
	res.Stdin = buf
	res.Env = os.Environ()
	return res
}
<|endoftext|>


<commit_before>package transfer

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/github/git-lfs/tools"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/subprocess"
	"github.com/rubyist/tracerx"

	"github.com/github/git-lfs/config"
)

// Adapter for custom transfer via external process
type customAdapter struct {
	*adapterBase
	path                string
	args                string
	concurrent          bool
	originalConcurrency int
}

// Struct to capture stderr and write to trace
type traceWriter struct {
	buf         bytes.Buffer
	processName string
}

func (t *traceWriter) Write(b []byte) (int, error) {
	n, err := t.buf.Write(b)
	t.Flush()
	return n, err
}
func (t *traceWriter) Flush() {
	var err error
	for err == nil {
		var s string
		s, err = t.buf.ReadString('\n')
		if len(s) > 0 {
			tracerx.Printf("xfer[%v]: %v", t.processName, strings.TrimSpace(s))
		}
	}
}

type customAdapterWorkerContext struct {
	workerNum   int
	cmd         *exec.Cmd
	stdout      io.ReadCloser
	bufferedOut *bufio.Reader
	stdin       io.WriteCloser
	errTracer   *traceWriter
}

type customAdapterInitRequest struct {
	Id                  string `json:"id"`
	Operation           string `json:"operation"`
	Concurrent          bool   `json:"concurrent"`
	ConcurrentTransfers int    `json:"concurrenttransfers"`
}

func NewCustomAdapterInitRequest(op string, concurrent bool, concurrentTransfers int) *customAdapterInitRequest {
	return &customAdapterInitRequest{"init", op, concurrent, concurrentTransfers}
}

type customAdapterTransferRequest struct { // common between upload/download
	Id     string            `json:"id"`
	Oid    string            `json:"oid"`
	Size   int64             `json:"size"`
	Path   string            `json:"path,omitempty"`
	Action *api.LinkRelation `json:"action"`
}

func NewCustomAdapterUploadRequest(oid string, size int64, path string, action *api.LinkRelation) *customAdapterTransferRequest {
	return &customAdapterTransferRequest{"upload", oid, size, path, action}
}
func NewCustomAdapterDownloadRequest(oid string, size int64, action *api.LinkRelation) *customAdapterTransferRequest {
	return &customAdapterTransferRequest{"download", oid, size, "", action}
}

type customAdapterTerminateRequest struct {
	MessageType string `json:"type"`
}

func NewCustomAdapterTerminateRequest() *customAdapterTerminateRequest {
	return &customAdapterTerminateRequest{"terminate"}
}

// A common struct that allows all types of response to be identified
type customAdapterResponseMessage struct {
	Id             string           `json:"id"`
	Error          *api.ObjectError `json:"error"`
	Oid            string           `json:"oid"`
	Path           string           `json:"path,omitempty"` // always blank for upload
	BytesSoFar     int64            `json:"bytesSoFar"`
	BytesSinceLast int              `json:"bytesSinceLast"`
}

func (a *customAdapter) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
	// If config says not to launch multiple processes, downgrade incoming value
	useConcurrency := maxConcurrency
	if !a.concurrent {
		useConcurrency = 1
	}
	a.originalConcurrency = maxConcurrency

	tracerx.Printf("xfer: Custom transfer adapter %q using concurrency %d", a.name, useConcurrency)

	// Use common workers impl, but downgrade workers to number of processes
	return a.adapterBase.Begin(useConcurrency, cb, completion)
}

func (a *customAdapter) ClearTempStorage() error {
	// no action requred
	return nil
}

func (a *customAdapter) WorkerStarting(workerNum int) (interface{}, error) {

	// Start a process per worker
	// If concurrent = false we have already dialled back workers to 1
	tracerx.Printf("xfer: starting up custom transfer process %q for worker %d", a.name, workerNum)
	cmd := subprocess.ExecCommand(a.path, a.args)
	outp, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdout for custom transfer command %q remote: %v", a.path, err)
	}
	inp, err := cmd.StdinPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdin for custom transfer command %q remote: %v", a.path, err)
	}
	// Capture stderr to trace
	tracer := &traceWriter{}
	tracer.processName = filepath.Base(a.path)
	cmd.Stderr = tracer
	err = cmd.Start()
	if err != nil {
		return nil, fmt.Errorf("Failed to start custom transfer command %q remote: %v", a.path, err)
	}
	// Set up buffered reader/writer since we operate on lines
	ctx := &customAdapterWorkerContext{workerNum, cmd, outp, bufio.NewReader(outp), inp, tracer}

	// send initiate message
	initReq := NewCustomAdapterInitRequest(a.getOperationName(), a.concurrent, a.originalConcurrency)
	resp, err := a.exchangeMessage(ctx, initReq)
	if err != nil {
		a.abortWorkerProcess(ctx)
		return nil, err
	}
	if resp.Error != nil {
		a.abortWorkerProcess(ctx)
		return nil, fmt.Errorf("Error initializing custom adapter %q worker %d: %v", a.name, workerNum, resp.Error)
	}

	tracerx.Printf("xfer: started custom adapter process %q for worker %d OK", a.path, workerNum)

	// Save this process context and use in future callbacks
	return ctx, nil
}

func (a *customAdapter) getOperationName() string {
	if a.direction == Download {
		return "download"
	}
	return "upload"
}

// sendMessage sends a JSON message to the custom adapter process
func (a *customAdapter) sendMessage(ctx *customAdapterWorkerContext, req interface{}) error {
	b, err := json.Marshal(req)
	if err != nil {
		return err
	}
	tracerx.Printf("xfer: Custom adapter worker %d sending message: %v", ctx.workerNum, string(b))
	// Line oriented JSON
	b = append(b, '\n')
	_, err = ctx.stdin.Write(b)
	return err
}

func (a *customAdapter) readResponse(ctx *customAdapterWorkerContext) (*customAdapterResponseMessage, error) {
	line, err := ctx.bufferedOut.ReadString('\n')
	if err != nil {
		return nil, err
	}
	tracerx.Printf("xfer: Custom adapter worker %d received response: %v", ctx.workerNum, line)
	resp := &customAdapterResponseMessage{}
	err = json.Unmarshal([]byte(line), resp)
	return resp, err
}

// exchangeMessage sends a message to a process and reads a response if resp != nil
// Only fatal errors to communicate return an error, errors may be embedded in reply
func (a *customAdapter) exchangeMessage(ctx *customAdapterWorkerContext, req interface{}) (*customAdapterResponseMessage, error) {

	err := a.sendMessage(ctx, req)
	if err != nil {
		return nil, err
	}
	return a.readResponse(ctx)
}

// shutdownWorkerProcess terminates gracefully a custom adapter process
// returns an error if it couldn't shut down gracefully (caller may abortWorkerProcess)
func (a *customAdapter) shutdownWorkerProcess(ctx *customAdapterWorkerContext) error {
	defer ctx.errTracer.Flush()

	tracerx.Printf("xfer: Shutting down adapter worker %d", ctx.workerNum)

	finishChan := make(chan error, 1)
	go func() {
		termReq := NewCustomAdapterTerminateRequest()
		err := a.sendMessage(ctx, termReq)
		if err != nil {
			finishChan <- err
		}
		ctx.stdin.Close()
		ctx.stdout.Close()
		finishChan <- ctx.cmd.Wait()
	}()
	select {
	case err := <-finishChan:
		return err
	case <-time.After(30 * time.Second):
		return fmt.Errorf("Timeout while shutting down worker process %d", ctx.workerNum)
	}
}

// abortWorkerProcess terminates & aborts untidily, most probably breakdown of comms or internal error
func (a *customAdapter) abortWorkerProcess(ctx *customAdapterWorkerContext) {
	tracerx.Printf("xfer: Aborting worker process: %d", ctx.workerNum)
	ctx.stdin.Close()
	ctx.stdout.Close()
	ctx.cmd.Process.Kill()
}
func (a *customAdapter) WorkerEnding(workerNum int, ctx interface{}) {
	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		tracerx.Printf("Context object for custom transfer %q was of the wrong type", a.name)
		return
	}

	err := a.shutdownWorkerProcess(customCtx)
	if err != nil {
		tracerx.Printf("xfer: error finishing up custom transfer process %q worker %d, aborting: %v", a.path, customCtx.workerNum, err)
		a.abortWorkerProcess(customCtx)
	}
}

func (a *customAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
	if ctx == nil {
		return fmt.Errorf("Custom transfer %q was not properly initialized, see previous errors", a.name)
	}

	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		return fmt.Errorf("Context object for custom transfer %q was of the wrong type", a.name)
	}
	var authCalled bool

	rel, ok := t.Object.Rel(a.getOperationName())
	if !ok {
		return errors.New("Object not found on the server.")
	}
	var req *customAdapterTransferRequest
	if a.direction == Upload {
		req = NewCustomAdapterUploadRequest(t.Object.Oid, t.Object.Size, t.Path, rel)
	} else {
		req = NewCustomAdapterDownloadRequest(t.Object.Oid, t.Object.Size, rel)
	}
	err := a.sendMessage(customCtx, req)
	if err != nil {
		return err
	}

	// 1..N replies (including progress & one of download / upload)
	var complete bool
	for !complete {
		resp, err := a.readResponse(customCtx)
		if err != nil {
			return err
		}
		var wasAuthOk bool
		switch resp.Id {
		case "progress":
			// Progress
			if resp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", resp.Oid, t.Object.Oid)
			}
			if cb != nil {
				cb(t.Name, t.Object.Size, resp.BytesSoFar, resp.BytesSinceLast)
			}
			wasAuthOk = resp.BytesSoFar > 0
		case "complete":
			// Download/Upload complete
			if resp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", resp.Oid, t.Object.Oid)
			}
			if resp.Error != nil {
				return fmt.Errorf("Error transferring %q: %v", t.Object.Oid, resp.Error)
			}
			if a.direction == Download {
				// So we don't have to blindly trust external providers, check SHA
				if err = tools.VerifyFileHash(t.Object.Oid, resp.Path); err != nil {
					return fmt.Errorf("Downloaded file failed checks: %v", err)
				}
				// Move file to final location
				if err = tools.RenameFileCopyPermissions(resp.Path, t.Path); err != nil {
					return fmt.Errorf("Failed to copy downloaded file: %v", err)
				}
			} else if a.direction == Upload {
				if err = api.VerifyUpload(t.Object); err != nil {
					return err
				}
			}
			wasAuthOk = true
			complete = true
		}
		// Fall through from both progress and completion messages
		// Call auth on first progress or success to free up other workers
		if wasAuthOk && authOkFunc != nil && !authCalled {
			authOkFunc()
			authCalled = true
		}
	}

	return nil
}

func newCustomAdapter(name string, dir Direction, path, args string, concurrent bool) *customAdapter {
	c := &customAdapter{newAdapterBase(name, dir, nil), path, args, concurrent, 3}
	// self implements impl
	c.transferImpl = c
	return c
}

// Initialise custom adapters based on current config
func ConfigureCustomAdapters() {
	pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
	for k, v := range config.Config.AllGitConfig() {
		match := pathRegex.FindStringSubmatch(k)
		if match == nil {
			continue
		}

		name := match[1]
		path := v
		// retrieve other values
		args, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
		concurrent := config.Config.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
		direction, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
		if len(direction) == 0 {
			direction = "both"
		} else {
			direction = strings.ToLower(direction)
		}

		// Separate closure for each since we need to capture vars above
		newfunc := func(name string, dir Direction) TransferAdapter {
			return newCustomAdapter(name, dir, path, args, concurrent)
		}

		if direction == "download" || direction == "both" {
			RegisterNewTransferAdapterFunc(name, Download, newfunc)
		}
		if direction == "upload" || direction == "both" {
			RegisterNewTransferAdapterFunc(name, Upload, newfunc)
		}
	}
}
<commit_msg>Handle incorrect messages received from custom adapters<commit_after>package transfer

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/github/git-lfs/tools"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/subprocess"
	"github.com/rubyist/tracerx"

	"github.com/github/git-lfs/config"
)

// Adapter for custom transfer via external process
type customAdapter struct {
	*adapterBase
	path                string
	args                string
	concurrent          bool
	originalConcurrency int
}

// Struct to capture stderr and write to trace
type traceWriter struct {
	buf         bytes.Buffer
	processName string
}

func (t *traceWriter) Write(b []byte) (int, error) {
	n, err := t.buf.Write(b)
	t.Flush()
	return n, err
}
func (t *traceWriter) Flush() {
	var err error
	for err == nil {
		var s string
		s, err = t.buf.ReadString('\n')
		if len(s) > 0 {
			tracerx.Printf("xfer[%v]: %v", t.processName, strings.TrimSpace(s))
		}
	}
}

type customAdapterWorkerContext struct {
	workerNum   int
	cmd         *exec.Cmd
	stdout      io.ReadCloser
	bufferedOut *bufio.Reader
	stdin       io.WriteCloser
	errTracer   *traceWriter
}

type customAdapterInitRequest struct {
	Id                  string `json:"id"`
	Operation           string `json:"operation"`
	Concurrent          bool   `json:"concurrent"`
	ConcurrentTransfers int    `json:"concurrenttransfers"`
}

func NewCustomAdapterInitRequest(op string, concurrent bool, concurrentTransfers int) *customAdapterInitRequest {
	return &customAdapterInitRequest{"init", op, concurrent, concurrentTransfers}
}

type customAdapterTransferRequest struct { // common between upload/download
	Id     string            `json:"id"`
	Oid    string            `json:"oid"`
	Size   int64             `json:"size"`
	Path   string            `json:"path,omitempty"`
	Action *api.LinkRelation `json:"action"`
}

func NewCustomAdapterUploadRequest(oid string, size int64, path string, action *api.LinkRelation) *customAdapterTransferRequest {
	return &customAdapterTransferRequest{"upload", oid, size, path, action}
}
func NewCustomAdapterDownloadRequest(oid string, size int64, action *api.LinkRelation) *customAdapterTransferRequest {
	return &customAdapterTransferRequest{"download", oid, size, "", action}
}

type customAdapterTerminateRequest struct {
	MessageType string `json:"type"`
}

func NewCustomAdapterTerminateRequest() *customAdapterTerminateRequest {
	return &customAdapterTerminateRequest{"terminate"}
}

// A common struct that allows all types of response to be identified
type customAdapterResponseMessage struct {
	Id             string           `json:"id"`
	Error          *api.ObjectError `json:"error"`
	Oid            string           `json:"oid"`
	Path           string           `json:"path,omitempty"` // always blank for upload
	BytesSoFar     int64            `json:"bytesSoFar"`
	BytesSinceLast int              `json:"bytesSinceLast"`
}

func (a *customAdapter) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
	// If config says not to launch multiple processes, downgrade incoming value
	useConcurrency := maxConcurrency
	if !a.concurrent {
		useConcurrency = 1
	}
	a.originalConcurrency = maxConcurrency

	tracerx.Printf("xfer: Custom transfer adapter %q using concurrency %d", a.name, useConcurrency)

	// Use common workers impl, but downgrade workers to number of processes
	return a.adapterBase.Begin(useConcurrency, cb, completion)
}

func (a *customAdapter) ClearTempStorage() error {
	// no action requred
	return nil
}

func (a *customAdapter) WorkerStarting(workerNum int) (interface{}, error) {

	// Start a process per worker
	// If concurrent = false we have already dialled back workers to 1
	tracerx.Printf("xfer: starting up custom transfer process %q for worker %d", a.name, workerNum)
	cmd := subprocess.ExecCommand(a.path, a.args)
	outp, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdout for custom transfer command %q remote: %v", a.path, err)
	}
	inp, err := cmd.StdinPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdin for custom transfer command %q remote: %v", a.path, err)
	}
	// Capture stderr to trace
	tracer := &traceWriter{}
	tracer.processName = filepath.Base(a.path)
	cmd.Stderr = tracer
	err = cmd.Start()
	if err != nil {
		return nil, fmt.Errorf("Failed to start custom transfer command %q remote: %v", a.path, err)
	}
	// Set up buffered reader/writer since we operate on lines
	ctx := &customAdapterWorkerContext{workerNum, cmd, outp, bufio.NewReader(outp), inp, tracer}

	// send initiate message
	initReq := NewCustomAdapterInitRequest(a.getOperationName(), a.concurrent, a.originalConcurrency)
	resp, err := a.exchangeMessage(ctx, initReq)
	if err != nil {
		a.abortWorkerProcess(ctx)
		return nil, err
	}
	if resp.Error != nil {
		a.abortWorkerProcess(ctx)
		return nil, fmt.Errorf("Error initializing custom adapter %q worker %d: %v", a.name, workerNum, resp.Error)
	}

	tracerx.Printf("xfer: started custom adapter process %q for worker %d OK", a.path, workerNum)

	// Save this process context and use in future callbacks
	return ctx, nil
}

func (a *customAdapter) getOperationName() string {
	if a.direction == Download {
		return "download"
	}
	return "upload"
}

// sendMessage sends a JSON message to the custom adapter process
func (a *customAdapter) sendMessage(ctx *customAdapterWorkerContext, req interface{}) error {
	b, err := json.Marshal(req)
	if err != nil {
		return err
	}
	tracerx.Printf("xfer: Custom adapter worker %d sending message: %v", ctx.workerNum, string(b))
	// Line oriented JSON
	b = append(b, '\n')
	_, err = ctx.stdin.Write(b)
	return err
}

func (a *customAdapter) readResponse(ctx *customAdapterWorkerContext) (*customAdapterResponseMessage, error) {
	line, err := ctx.bufferedOut.ReadString('\n')
	if err != nil {
		return nil, err
	}
	tracerx.Printf("xfer: Custom adapter worker %d received response: %v", ctx.workerNum, line)
	resp := &customAdapterResponseMessage{}
	err = json.Unmarshal([]byte(line), resp)
	return resp, err
}

// exchangeMessage sends a message to a process and reads a response if resp != nil
// Only fatal errors to communicate return an error, errors may be embedded in reply
func (a *customAdapter) exchangeMessage(ctx *customAdapterWorkerContext, req interface{}) (*customAdapterResponseMessage, error) {

	err := a.sendMessage(ctx, req)
	if err != nil {
		return nil, err
	}
	return a.readResponse(ctx)
}

// shutdownWorkerProcess terminates gracefully a custom adapter process
// returns an error if it couldn't shut down gracefully (caller may abortWorkerProcess)
func (a *customAdapter) shutdownWorkerProcess(ctx *customAdapterWorkerContext) error {
	defer ctx.errTracer.Flush()

	tracerx.Printf("xfer: Shutting down adapter worker %d", ctx.workerNum)

	finishChan := make(chan error, 1)
	go func() {
		termReq := NewCustomAdapterTerminateRequest()
		err := a.sendMessage(ctx, termReq)
		if err != nil {
			finishChan <- err
		}
		ctx.stdin.Close()
		ctx.stdout.Close()
		finishChan <- ctx.cmd.Wait()
	}()
	select {
	case err := <-finishChan:
		return err
	case <-time.After(30 * time.Second):
		return fmt.Errorf("Timeout while shutting down worker process %d", ctx.workerNum)
	}
}

// abortWorkerProcess terminates & aborts untidily, most probably breakdown of comms or internal error
func (a *customAdapter) abortWorkerProcess(ctx *customAdapterWorkerContext) {
	tracerx.Printf("xfer: Aborting worker process: %d", ctx.workerNum)
	ctx.stdin.Close()
	ctx.stdout.Close()
	ctx.cmd.Process.Kill()
}
func (a *customAdapter) WorkerEnding(workerNum int, ctx interface{}) {
	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		tracerx.Printf("Context object for custom transfer %q was of the wrong type", a.name)
		return
	}

	err := a.shutdownWorkerProcess(customCtx)
	if err != nil {
		tracerx.Printf("xfer: error finishing up custom transfer process %q worker %d, aborting: %v", a.path, customCtx.workerNum, err)
		a.abortWorkerProcess(customCtx)
	}
}

func (a *customAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
	if ctx == nil {
		return fmt.Errorf("Custom transfer %q was not properly initialized, see previous errors", a.name)
	}

	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		return fmt.Errorf("Context object for custom transfer %q was of the wrong type", a.name)
	}
	var authCalled bool

	rel, ok := t.Object.Rel(a.getOperationName())
	if !ok {
		return errors.New("Object not found on the server.")
	}
	var req *customAdapterTransferRequest
	if a.direction == Upload {
		req = NewCustomAdapterUploadRequest(t.Object.Oid, t.Object.Size, t.Path, rel)
	} else {
		req = NewCustomAdapterDownloadRequest(t.Object.Oid, t.Object.Size, rel)
	}
	err := a.sendMessage(customCtx, req)
	if err != nil {
		return err
	}

	// 1..N replies (including progress & one of download / upload)
	var complete bool
	for !complete {
		resp, err := a.readResponse(customCtx)
		if err != nil {
			return err
		}
		var wasAuthOk bool
		switch resp.Id {
		case "progress":
			// Progress
			if resp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", resp.Oid, t.Object.Oid)
			}
			if cb != nil {
				cb(t.Name, t.Object.Size, resp.BytesSoFar, resp.BytesSinceLast)
			}
			wasAuthOk = resp.BytesSoFar > 0
		case "complete":
			// Download/Upload complete
			if resp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", resp.Oid, t.Object.Oid)
			}
			if resp.Error != nil {
				return fmt.Errorf("Error transferring %q: %v", t.Object.Oid, resp.Error)
			}
			if a.direction == Download {
				// So we don't have to blindly trust external providers, check SHA
				if err = tools.VerifyFileHash(t.Object.Oid, resp.Path); err != nil {
					return fmt.Errorf("Downloaded file failed checks: %v", err)
				}
				// Move file to final location
				if err = tools.RenameFileCopyPermissions(resp.Path, t.Path); err != nil {
					return fmt.Errorf("Failed to copy downloaded file: %v", err)
				}
			} else if a.direction == Upload {
				if err = api.VerifyUpload(t.Object); err != nil {
					return err
				}
			}
			wasAuthOk = true
			complete = true
		default:
			return fmt.Errorf("Invalid message Id %q from custom adapter %q", resp.Id, a.name)
		}
		// Fall through from both progress and completion messages
		// Call auth on first progress or success to free up other workers
		if wasAuthOk && authOkFunc != nil && !authCalled {
			authOkFunc()
			authCalled = true
		}
	}

	return nil
}

func newCustomAdapter(name string, dir Direction, path, args string, concurrent bool) *customAdapter {
	c := &customAdapter{newAdapterBase(name, dir, nil), path, args, concurrent, 3}
	// self implements impl
	c.transferImpl = c
	return c
}

// Initialise custom adapters based on current config
func ConfigureCustomAdapters() {
	pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
	for k, v := range config.Config.AllGitConfig() {
		match := pathRegex.FindStringSubmatch(k)
		if match == nil {
			continue
		}

		name := match[1]
		path := v
		// retrieve other values
		args, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
		concurrent := config.Config.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
		direction, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
		if len(direction) == 0 {
			direction = "both"
		} else {
			direction = strings.ToLower(direction)
		}

		// Separate closure for each since we need to capture vars above
		newfunc := func(name string, dir Direction) TransferAdapter {
			return newCustomAdapter(name, dir, path, args, concurrent)
		}

		if direction == "download" || direction == "both" {
			RegisterNewTransferAdapterFunc(name, Download, newfunc)
		}
		if direction == "upload" || direction == "both" {
			RegisterNewTransferAdapterFunc(name, Upload, newfunc)
		}
	}
}
<|endoftext|>


<commit_before>package jsonerror

import (
	"fmt"
)

//Struct that contains the Code, Domain, Error and Message.
//Only Code and Domain are exported to encourage usage of the New(...) method to set the message and error.
//JE is shorthand for JSONError.
type JE struct {
	Code    int
	Domain  string
	error   string
	message string
}

//Creates a new JE struct.
//Domain is optional but can be at most 1 string.
func New(code int, error string, message string, domain ...string) *JE {
	if len(domain) == 0 {
		return &JE{Code: code, error: error, message: message}
	} else {
		return &JE{Code: code, error: error, message: message, Domain: domain[0]}
	}
}

//Generates a string that neatly formats the contents of JE struct.
//Useful with panic() because JSONError satisfies error interface.
func (this JE) Error() string {
	finalString := fmt.Sprintf("code: %d", this.Code)

	if this.error != "" {
		finalString = finalString + fmt.Sprintf(" error: %s", this.error)
	}

	if this.message != "" {
		finalString = finalString + fmt.Sprintf(" message: %s", this.message)
	}

	if this.Domain != "" {
		finalString = finalString + fmt.Sprintf(" domain: %s", this.Domain)
	}

	return finalString
}

//For use with package: "gopkg.in/unrolled/render.v1".
//Can easily output properly formatted JSON error messages for REST API services.
func (this JE) Render() map[string]string {

	if this.error == "" {
		if this.message == "" {
			return map[string]string{"code": fmt.Sprintf("%d", this.Code)}
		} else {
			return map[string]string{"code": fmt.Sprintf("%d", this.Code), "message": this.message}
		}
	} else {
		if this.message == "" {
			return map[string]string{"code": fmt.Sprintf("%d", this.Code), "error": this.error}
		} else {
			return map[string]string{"code": fmt.Sprintf("%d", this.Code), "error": this.error, "message": this.message}
		}
	}
}
<commit_msg>Changed from pointer to struct.<commit_after>package jsonerror

import (
	"fmt"
)

//Struct that contains the Code, Domain, Error and Message.
//Only Code and Domain are exported to encourage usage of the New(...) method to set the message and error.
//JE is shorthand for JSONError.
type JE struct {
	Code    int
	Domain  string
	error   string
	message string
}

//Creates a new JE struct.
//Domain is optional but can be at most 1 string.
func New(code int, error string, message string, domain ...string) JE {
	if len(domain) == 0 {
		return &JE{Code: code, error: error, message: message}
	} else {
		return &JE{Code: code, error: error, message: message, Domain: domain[0]}
	}
}

//Generates a string that neatly formats the contents of JE struct.
//Useful with panic() because JSONError satisfies error interface.
func (this JE) Error() string {
	finalString := fmt.Sprintf("code: %d", this.Code)

	if this.error != "" {
		finalString = finalString + fmt.Sprintf(" error: %s", this.error)
	}

	if this.message != "" {
		finalString = finalString + fmt.Sprintf(" message: %s", this.message)
	}

	if this.Domain != "" {
		finalString = finalString + fmt.Sprintf(" domain: %s", this.Domain)
	}

	return finalString
}

//For use with package: "gopkg.in/unrolled/render.v1".
//Can easily output properly formatted JSON error messages for REST API services.
func (this JE) Render() map[string]string {

	if this.error == "" {
		if this.message == "" {
			return map[string]string{"code": fmt.Sprintf("%d", this.Code)}
		} else {
			return map[string]string{"code": fmt.Sprintf("%d", this.Code), "message": this.message}
		}
	} else {
		if this.message == "" {
			return map[string]string{"code": fmt.Sprintf("%d", this.Code), "error": this.error}
		} else {
			return map[string]string{"code": fmt.Sprintf("%d", this.Code), "error": this.error, "message": this.message}
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>support for shards<commit_after><|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"sync"
	"time"

	"github.com/dgraph-io/badger/badger"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
	"github.com/dgraph-io/dgraph/store"
	"github.com/pkg/profile"
)

const mil float64 = 1000000

var (
	which   = flag.String("kv", "both", "Which KV store to use.")
	numKeys = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
)

func fillEntry(e *badger.Entry) {
	var pow uint = 10 // 1KB
	if rand.Intn(2) == 1 {
		pow = 14 // 16KB
	}
	k := rand.Int() % int(*numKeys*mil)
	key := fmt.Sprintf("v=%02d-k=%016d", pow, k)
	if cap(e.Key) < len(key) {
		e.Key = make([]byte, 2*len(key))
	}
	e.Key = e.Key[:len(key)]
	copy(e.Key, key)

	y.AssertTrue(cap(e.Value) == 1<<14)
	vsz := 1 << pow
	e.Value = e.Value[:vsz]
	rand.Read(e.Value)

	e.Meta = 0
	e.Offset = 0
}

var ctx = context.Background()
var bdb *badger.KV
var rdb *store.Store

func writeBatch(entries []*badger.Entry) int {
	rb := rdb.NewWriteBatch()
	defer rb.Destroy()

	for _, e := range entries {
		fillEntry(e)
		rb.Put(e.Key, e.Value)
	}
	if bdb != nil {
		y.Check(bdb.Write(ctx, entries))
	}
	if rdb != nil {
		y.Check(rdb.WriteBatch(rb))
	}
	return len(entries)
}

func main() {
	mode := flag.String("profile.mode", "", "enable profiling mode, one of [cpu, mem, mutex, block]")
	flag.Parse()
	switch *mode {
	case "cpu":
		defer profile.Start(profile.CPUProfile).Stop()
	case "mem":
		defer profile.Start(profile.MemProfile).Stop()
	case "mutex":
		defer profile.Start(profile.MutexProfile).Stop()
	case "block":
		defer profile.Start(profile.BlockProfile).Stop()
	default:
		// do nothing
	}

	nw := *numKeys * mil
	opt := badger.DefaultOptions
	opt.NumMemtables = 3
	opt.MapTablesTo = table.Nothing
	opt.Verbose = true
	opt.Dir = "tmp/badger"
	opt.SyncWrites = false

	var err error

	if *which == "badger" || *which == "both" {
		fmt.Println("Init Badger")
		y.Check(os.RemoveAll("tmp/badger"))
		os.MkdirAll("tmp/badger", 0777)
		bdb = badger.NewKV(&opt)
	}
	if *which == "rocksdb" || *which == "both" {
		fmt.Println("Init Rocks")
		os.RemoveAll("tmp/rocks")
		os.MkdirAll("tmp/rocks", 0777)
		rdb, err = store.NewStore("tmp/rocks")
		// rdb, err = store.NewSyncStore("tmp/rocks")
		y.Check(err)
	}

	go http.ListenAndServe(":8080", nil)

	N := 10
	var wg sync.WaitGroup
	for i := 0; i < N; i++ {
		wg.Add(1)
		go func(proc int) {
			entries := make([]*badger.Entry, 100)
			for i := 0; i < len(entries); i++ {
				e := new(badger.Entry)
				e.Key = make([]byte, 10)
				e.Value = make([]byte, 1<<14)
				entries[i] = e
			}

			var written float64
			for written < nw/float64(N) {
				written += float64(writeBatch(entries))
				if int(written)%100000 == 0 {
					fmt.Printf("[%d] Written %5.2fM key-val pairs\n", proc, written/mil)
				}
			}
			fmt.Printf("[%d] Written %5.2fM key-val pairs\n", proc, written/mil)
			wg.Done()
		}(i)
	}
	// 	wg.Add(1) // Block
	wg.Wait()
	if bdb != nil {
		fmt.Println("closing badger")
		bdb.Close()
	}
	if rdb != nil {
		fmt.Println("closing rocks")
		rdb.Close()
	}
	time.Sleep(10 * time.Second)
}
<commit_msg>Set value size via flag<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"sync"
	"time"

	"github.com/dgraph-io/badger/badger"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
	"github.com/dgraph-io/dgraph/store"
	"github.com/pkg/profile"
)

const mil float64 = 1000000

var (
	which     = flag.String("kv", "both", "Which KV store to use. Options: both, badger, rocksdb")
	numKeys   = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	valueSize = flag.Int("valsz", 128, "Value size in bytes.")
)

func fillEntry(e *badger.Entry) {
	k := rand.Int() % int(*numKeys*mil)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *valueSize, k) // 22 bytes.
	if cap(e.Key) < len(key) {
		e.Key = make([]byte, 2*len(key))
	}
	e.Key = e.Key[:len(key)]
	copy(e.Key, key)

	rand.Read(e.Value)
	e.Meta = 0
	e.Offset = 0
}

var ctx = context.Background()
var bdb *badger.KV
var rdb *store.Store

func writeBatch(entries []*badger.Entry) int {
	rb := rdb.NewWriteBatch()
	defer rb.Destroy()

	for _, e := range entries {
		fillEntry(e)
		rb.Put(e.Key, e.Value)
	}
	if bdb != nil {
		y.Check(bdb.Write(ctx, entries))
	}
	if rdb != nil {
		y.Check(rdb.WriteBatch(rb))
	}
	return len(entries)
}

func main() {
	mode := flag.String("profile.mode", "", "enable profiling mode, one of [cpu, mem, mutex, block]")
	flag.Parse()
	switch *mode {
	case "cpu":
		defer profile.Start(profile.CPUProfile).Stop()
	case "mem":
		defer profile.Start(profile.MemProfile).Stop()
	case "mutex":
		defer profile.Start(profile.MutexProfile).Stop()
	case "block":
		defer profile.Start(profile.BlockProfile).Stop()
	default:
		// do nothing
	}

	nw := *numKeys * mil
	opt := badger.DefaultOptions
	opt.NumMemtables = 3
	opt.MapTablesTo = table.Nothing
	opt.Verbose = true
	opt.Dir = "tmp/badger"
	opt.SyncWrites = false

	var err error

	if *which == "badger" || *which == "both" {
		fmt.Println("Init Badger")
		y.Check(os.RemoveAll("tmp/badger"))
		os.MkdirAll("tmp/badger", 0777)
		bdb = badger.NewKV(&opt)
	}
	if *which == "rocksdb" || *which == "both" {
		fmt.Println("Init Rocks")
		os.RemoveAll("tmp/rocks")
		os.MkdirAll("tmp/rocks", 0777)
		rdb, err = store.NewStore("tmp/rocks")
		// rdb, err = store.NewSyncStore("tmp/rocks")
		y.Check(err)
	}

	go http.ListenAndServe("0.0.0.0:8080", nil)

	N := 10
	var wg sync.WaitGroup
	for i := 0; i < N; i++ {
		wg.Add(1)
		go func(proc int) {
			entries := make([]*badger.Entry, 100)
			for i := 0; i < len(entries); i++ {
				e := new(badger.Entry)
				e.Key = make([]byte, 10)
				e.Value = make([]byte, *valueSize)
				entries[i] = e
			}

			var written float64
			for written < nw/float64(N) {
				written += float64(writeBatch(entries))
				if int(written)%100000 == 0 {
					fmt.Printf("[%d] Written %5.2fM key-val pairs\n", proc, written/mil)
				}
			}
			fmt.Printf("[%d] Written %5.2fM key-val pairs\n", proc, written/mil)
			wg.Done()
		}(i)
	}
	// 	wg.Add(1) // Block
	wg.Wait()
	if bdb != nil {
		fmt.Println("closing badger")
		bdb.Close()
	}
	if rdb != nil {
		fmt.Println("closing rocks")
		rdb.Close()
	}
	time.Sleep(10 * time.Second)
}
<|endoftext|>


<commit_before><commit_msg>Removings unused Values type<commit_after><|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !cgo,!windows,!plan9 android osusergo,!windows,!plan9

package user

import (
	"errors"
	"fmt"
	"os"
	"runtime"
	"strconv"
)

func init() {
	groupImplemented = false
}

func current() (*User, error) {
	uid := currentUID()
	// $USER and /etc/passwd may disagree; prefer the latter if we can get it.
	// See issue 27524 for more information.
	u, err := lookupUserId(uid)
	if err == nil {
		return u, nil
	}

	homeDir, _ := os.UserHomeDir()
	u = &User{
		Uid:      uid,
		Gid:      currentGID(),
		Username: os.Getenv("USER"),
		Name:     "", // ignored
		HomeDir:  homeDir,
	}
	// On NaCL and Android, return a dummy user instead of failing.
	switch runtime.GOOS {
	case "nacl":
		if u.Uid == "" {
			u.Uid = "1"
		}
		if u.Username == "" {
			u.Username = "nacl"
		}
	case "android":
		if u.Uid == "" {
			u.Uid = "1"
		}
		if u.Username == "" {
			u.Username = "android"
		}
	}
	// cgo isn't available, but if we found the minimum information
	// without it, use it:
	if u.Uid != "" && u.Username != "" && u.HomeDir != "" {
		return u, nil
	}
	return u, fmt.Errorf("user: Current not implemented on %s/%s", runtime.GOOS, runtime.GOARCH)
}

func listGroups(*User) ([]string, error) {
	if runtime.GOOS == "android" || runtime.GOOS == "aix" {
		return nil, errors.New(fmt.Sprintf("user: GroupIds not implemented on %s", runtime.GOOS))
	}
	return nil, errors.New("user: GroupIds requires cgo")
}

func currentUID() string {
	if id := os.Getuid(); id >= 0 {
		return strconv.Itoa(id)
	}
	// Note: Windows returns -1, but this file isn't used on
	// Windows anyway, so this empty return path shouldn't be
	// used.
	return ""
}

func currentGID() string {
	if id := os.Getgid(); id >= 0 {
		return strconv.Itoa(id)
	}
	return ""
}
<commit_msg>os/user: make Current return better error w/o cgo & complete environment<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !cgo,!windows,!plan9 android osusergo,!windows,!plan9

package user

import (
	"errors"
	"fmt"
	"os"
	"runtime"
	"strconv"
)

func init() {
	groupImplemented = false
}

func current() (*User, error) {
	uid := currentUID()
	// $USER and /etc/passwd may disagree; prefer the latter if we can get it.
	// See issue 27524 for more information.
	u, err := lookupUserId(uid)
	if err == nil {
		return u, nil
	}

	homeDir, _ := os.UserHomeDir()
	u = &User{
		Uid:      uid,
		Gid:      currentGID(),
		Username: os.Getenv("USER"),
		Name:     "", // ignored
		HomeDir:  homeDir,
	}
	// On NaCL and Android, return a dummy user instead of failing.
	switch runtime.GOOS {
	case "nacl":
		if u.Uid == "" {
			u.Uid = "1"
		}
		if u.Username == "" {
			u.Username = "nacl"
		}
	case "android":
		if u.Uid == "" {
			u.Uid = "1"
		}
		if u.Username == "" {
			u.Username = "android"
		}
	}
	// cgo isn't available, but if we found the minimum information
	// without it, use it:
	if u.Uid != "" && u.Username != "" && u.HomeDir != "" {
		return u, nil
	}
	var missing string
	if u.Username == "" {
		missing = "$USER"
	}
	if u.HomeDir == "" {
		if missing != "" {
			missing += ", "
		}
		missing += "$HOME"
	}
	return u, fmt.Errorf("user: Current requires cgo or %s set in environment", missing)
}

func listGroups(*User) ([]string, error) {
	if runtime.GOOS == "android" || runtime.GOOS == "aix" {
		return nil, errors.New(fmt.Sprintf("user: GroupIds not implemented on %s", runtime.GOOS))
	}
	return nil, errors.New("user: GroupIds requires cgo")
}

func currentUID() string {
	if id := os.Getuid(); id >= 0 {
		return strconv.Itoa(id)
	}
	// Note: Windows returns -1, but this file isn't used on
	// Windows anyway, so this empty return path shouldn't be
	// used.
	return ""
}

func currentGID() string {
	if id := os.Getgid(); id >= 0 {
		return strconv.Itoa(id)
	}
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2016 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types_test

import (
	"reflect"

	. "github.com/containernetworking/cni/pkg/types"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("UnmarshallableBool UnmarshalText", func() {
	DescribeTable("string to bool detection should succeed in all cases",
		func(inputs []string, expected bool) {
			for _, s := range inputs {
				var ub UnmarshallableBool
				err := ub.UnmarshalText([]byte(s))
				Expect(err).ToNot(HaveOccurred())
				Expect(ub).To(Equal(UnmarshallableBool(expected)))
			}
		},
		Entry("parse to true", []string{"True", "true", "1"}, true),
		Entry("parse to false", []string{"False", "false", "0"}, false),
	)

	Context("When passed an invalid value", func() {
		It("should result in an error", func() {
			var ub UnmarshallableBool
			err := ub.UnmarshalText([]byte("invalid"))
			Expect(err).To(HaveOccurred())
		})
	})
})

var _ = Describe("GetKeyField", func() {
	type testcontainer struct {
		Valid string `json:"valid,omitempty"`
	}
	var (
		container          = testcontainer{Valid: "valid"}
		containerInterface = func(i interface{}) interface{} { return i }(&container)
		containerValue     = reflect.ValueOf(containerInterface)
	)
	Context("When a valid field is provided", func() {
		It("should return the correct field", func() {
			field := GetKeyField("Valid", containerValue)
			Expect(field.String()).To(Equal("valid"))
		})
	})
})

var _ = Describe("LoadArgs", func() {
	Context("When no arguments are passed", func() {
		It("LoadArgs should succeed", func() {
			err := LoadArgs("", struct{}{})
			Expect(err).NotTo(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and ignored", func() {
		It("LoadArgs should succeed", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=True;Unk=nown", &ca)
			Expect(err).NotTo(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and not ignored", func() {
		It("LoadArgs should fail", func() {
			ca := CommonArgs{}
			err := LoadArgs("Unk=nown", &ca)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and explicitly not ignored", func() {
		It("LoadArgs should fail", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=0, Unk=nown", &ca)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("When known arguments are passed", func() {
		It("LoadArgs should succeed", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=1", &ca)
			Expect(err).NotTo(HaveOccurred())
		})
	})
})
<commit_msg>pkg/types: cover string for unmarshal tests<commit_after>// Copyright 2016 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types_test

import (
	"reflect"

	. "github.com/containernetworking/cni/pkg/types"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

var _ = Describe("UnmarshallableBool UnmarshalText", func() {
	DescribeTable("string to bool detection should succeed in all cases",
		func(inputs []string, expected bool) {
			for _, s := range inputs {
				var ub UnmarshallableBool
				err := ub.UnmarshalText([]byte(s))
				Expect(err).ToNot(HaveOccurred())
				Expect(ub).To(Equal(UnmarshallableBool(expected)))
			}
		},
		Entry("parse to true", []string{"True", "true", "1"}, true),
		Entry("parse to false", []string{"False", "false", "0"}, false),
	)

	Context("When passed an invalid value", func() {
		It("should result in an error", func() {
			var ub UnmarshallableBool
			err := ub.UnmarshalText([]byte("invalid"))
			Expect(err).To(HaveOccurred())
		})
	})
})

var _ = Describe("UnmarshallableString UnmarshalText", func() {
	DescribeTable("string to string detection should succeed in all cases",
		func(inputs []string, expected string) {
			for _, s := range inputs {
				var us UnmarshallableString
				err := us.UnmarshalText([]byte(s))
				Expect(err).ToNot(HaveOccurred())
				Expect(string(us)).To(Equal(expected))
			}
		},
		Entry("parse empty string", []string{""}, ""),
		Entry("parse non-empty string", []string{"notempty"}, "notempty"),
	)
})

var _ = Describe("GetKeyField", func() {
	type testcontainer struct {
		Valid string `json:"valid,omitempty"`
	}
	var (
		container          = testcontainer{Valid: "valid"}
		containerInterface = func(i interface{}) interface{} { return i }(&container)
		containerValue     = reflect.ValueOf(containerInterface)
	)
	Context("When a valid field is provided", func() {
		It("should return the correct field", func() {
			field := GetKeyField("Valid", containerValue)
			Expect(field.String()).To(Equal("valid"))
		})
	})
})

var _ = Describe("LoadArgs", func() {
	Context("When no arguments are passed", func() {
		It("LoadArgs should succeed", func() {
			err := LoadArgs("", struct{}{})
			Expect(err).NotTo(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and ignored", func() {
		It("LoadArgs should succeed", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=True;Unk=nown", &ca)
			Expect(err).NotTo(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and not ignored", func() {
		It("LoadArgs should fail", func() {
			ca := CommonArgs{}
			err := LoadArgs("Unk=nown", &ca)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("When unknown arguments are passed and explicitly not ignored", func() {
		It("LoadArgs should fail", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=0, Unk=nown", &ca)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("When known arguments are passed", func() {
		It("LoadArgs should succeed", func() {
			ca := CommonArgs{}
			err := LoadArgs("IgnoreUnknown=1", &ca)
			Expect(err).NotTo(HaveOccurred())
		})
	})
})
<|endoftext|>


<commit_before>package mpb_test

import (
	"bytes"
	"context"
	"errors"
	"io"
	"math/rand"
	"strings"
	"testing"
	"time"

	"github.com/vbauerster/mpb/v8"
	"github.com/vbauerster/mpb/v8/decor"
)

const (
	timeout = 200 * time.Millisecond
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func TestBarCount(t *testing.T) {
	p := mpb.New(mpb.WithOutput(io.Discard))

	b := p.AddBar(0, mpb.BarRemoveOnComplete())

	if count := p.BarCount(); count != 1 {
		t.Errorf("BarCount want: %d, got: %d\n", 1, count)
	}

	b.SetTotal(100, true)

	b.Wait()

	if count := p.BarCount(); count != 0 {
		t.Errorf("BarCount want: %d, got: %d\n", 0, count)
	}

	p.Wait()
}

func TestBarAbort(t *testing.T) {
	shutdown := make(chan struct{})
	p := mpb.New(mpb.WithShutdownNotifier(shutdown), mpb.WithOutput(io.Discard))
	n := 2
	bars := make([]*mpb.Bar, n)
	for i := 0; i < n; i++ {
		b := p.AddBar(100)
		switch i {
		case n - 1:
			var abortCalledTimes int
			for j := 0; !b.Aborted(); j++ {
				if j >= 10 {
					b.Abort(true)
					abortCalledTimes++
				} else {
					b.Increment()
				}
			}
			if abortCalledTimes != 1 {
				t.Errorf("Expected abortCalledTimes: %d, got: %d\n", 1, abortCalledTimes)
			}
			b.Wait()
			count := p.BarCount()
			if count != 1 {
				t.Errorf("BarCount want: %d, got: %d\n", 1, count)
			}
		default:
			go func() {
				for !b.Completed() {
					b.Increment()
					time.Sleep(randomDuration(100 * time.Millisecond))
				}
			}()
		}
		bars[i] = b
	}

	go p.Wait()

	bars[0].Abort(false)

	select {
	case <-shutdown:
	case <-time.After(timeout):
		t.Errorf("Progress didn't shutdown after %v", timeout)
	}
}

func TestWithContext(t *testing.T) {
	shutdown := make(chan struct{})
	ctx, cancel := context.WithCancel(context.Background())
	p := mpb.NewWithContext(ctx,
		mpb.WithShutdownNotifier(shutdown),
		mpb.WithOutput(io.Discard),
	)
	_ = p.AddBar(0) // never complete bar
	_ = p.AddBar(0) // never complete bar
	go func() {
		time.Sleep(randomDuration(100 * time.Millisecond))
		cancel()
		p.Wait()
	}()

	select {
	case <-shutdown:
	case <-time.After(timeout):
		t.Errorf("Progress didn't shutdown after %v", timeout)
	}
}

func TestMaxWidthDistributor(t *testing.T) {
	makeWrapper := func(f func([]chan int), start, end chan struct{}) func([]chan int) {
		return func(column []chan int) {
			start <- struct{}{}
			f(column)
			<-end
		}
	}

	ready := make(chan struct{})
	start := make(chan struct{})
	end := make(chan struct{})
	// mpb.MaxWidthDistributor shouldn't stuck in the middle while removing or aborting a bar
	mpb.MaxWidthDistributor = makeWrapper(mpb.MaxWidthDistributor, start, end)

	total := 100
	numBars := 6
	p := mpb.New(mpb.WithOutput(io.Discard))
	for i := 0; i < numBars; i++ {
		bar := p.AddBar(int64(total),
			mpb.BarOptional(mpb.BarRemoveOnComplete(), i == 0),
			mpb.PrependDecorators(decor.EwmaETA(decor.ET_STYLE_GO, 60, decor.WCSyncSpace)),
		)
		go func() {
			<-ready
			for i := 0; i < total; i++ {
				start := time.Now()
				if id := bar.ID(); id > 1 && i >= 32 {
					if id&1 == 1 {
						bar.Abort(true)
					} else {
						bar.Abort(false)
					}
				}
				time.Sleep(randomDuration(100 * time.Millisecond))
				bar.EwmaIncrInt64(rand.Int63n(5)+1, time.Since(start))
			}
		}()
	}

	go func() {
		<-ready
		p.Wait()
		close(start)
	}()

	res := t.Run("maxWidthDistributor", func(t *testing.T) {
		close(ready)
		for v := range start {
			timer := time.NewTimer(100 * time.Millisecond)
			select {
			case end <- v:
				timer.Stop()
			case <-timer.C:
				t.FailNow()
			}
		}
	})

	if !res {
		t.Error("maxWidthDistributor stuck in the middle")
	}
}

func TestProgressShutdownsWithErrFiller(t *testing.T) {
	var debug bytes.Buffer
	shutdown := make(chan struct{})
	p := mpb.New(
		mpb.WithShutdownNotifier(shutdown),
		mpb.WithOutput(io.Discard),
		mpb.WithDebugOutput(&debug),
	)

	testError := errors.New("test error")
	bar := p.AddBar(100,
		mpb.BarFillerMiddleware(func(base mpb.BarFiller) mpb.BarFiller {
			return mpb.BarFillerFunc(func(w io.Writer, st decor.Statistics) error {
				if st.Current >= 42 {
					return testError
				}
				return base.Fill(w, st)
			})
		}),
	)

	go func() {
		for bar.IsRunning() {
			bar.Increment()
		}
	}()

	select {
	case <-shutdown:
		if err := strings.TrimSpace(debug.String()); err != testError.Error() {
			t.Errorf("Expected err: %q, got %q\n", testError.Error(), err)
		}
	case <-time.After(timeout):
		t.Errorf("Progress didn't shutdown after %v", timeout)
	}
	p.Wait()
}

func randomDuration(max time.Duration) time.Duration {
	return time.Duration(rand.Intn(10)+1) * max / 10
}
<commit_msg>test timeout 300 ms<commit_after>package mpb_test

import (
	"bytes"
	"context"
	"errors"
	"io"
	"math/rand"
	"strings"
	"testing"
	"time"

	"github.com/vbauerster/mpb/v8"
	"github.com/vbauerster/mpb/v8/decor"
)

const (
	timeout = 300 * time.Millisecond
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func TestBarCount(t *testing.T) {
	p := mpb.New(mpb.WithOutput(io.Discard))

	b := p.AddBar(0, mpb.BarRemoveOnComplete())

	if count := p.BarCount(); count != 1 {
		t.Errorf("BarCount want: %d, got: %d\n", 1, count)
	}

	b.SetTotal(100, true)

	b.Wait()

	if count := p.BarCount(); count != 0 {
		t.Errorf("BarCount want: %d, got: %d\n", 0, count)
	}

	p.Wait()
}

func TestBarAbort(t *testing.T) {
	shutdown := make(chan struct{})
	p := mpb.New(mpb.WithShutdownNotifier(shutdown), mpb.WithOutput(io.Discard))
	n := 2
	bars := make([]*mpb.Bar, n)
	for i := 0; i < n; i++ {
		b := p.AddBar(100)
		switch i {
		case n - 1:
			var abortCalledTimes int
			for j := 0; !b.Aborted(); j++ {
				if j >= 10 {
					b.Abort(true)
					abortCalledTimes++
				} else {
					b.Increment()
				}
			}
			if abortCalledTimes != 1 {
				t.Errorf("Expected abortCalledTimes: %d, got: %d\n", 1, abortCalledTimes)
			}
			b.Wait()
			count := p.BarCount()
			if count != 1 {
				t.Errorf("BarCount want: %d, got: %d\n", 1, count)
			}
		default:
			go func() {
				for !b.Completed() {
					b.Increment()
					time.Sleep(randomDuration(100 * time.Millisecond))
				}
			}()
		}
		bars[i] = b
	}

	go p.Wait()

	bars[0].Abort(false)

	select {
	case <-shutdown:
	case <-time.After(timeout):
		t.Errorf("Progress didn't shutdown after %v", timeout)
	}
}

func TestWithContext(t *testing.T) {
	shutdown := make(chan struct{})
	ctx, cancel := context.WithCancel(context.Background())
	p := mpb.NewWithContext(ctx,
		mpb.WithShutdownNotifier(shutdown),
		mpb.WithOutput(io.Discard),
	)
	_ = p.AddBar(0) // never complete bar
	_ = p.AddBar(0) // never complete bar
	go func() {
		time.Sleep(randomDuration(100 * time.Millisecond))
		cancel()
		p.Wait()
	}()

	select {
	case <-shutdown:
	case <-time.After(timeout):
		t.Errorf("Progress didn't shutdown after %v", timeout)
	}
}

func TestMaxWidthDistributor(t *testing.T) {
	makeWrapper := func(f func([]chan int), start, end chan struct{}) func([]chan int) {
		return func(column []chan int) {
			start <- struct{}{}
			f(column)
			<-end
		}
	}

	ready := make(chan struct{})
	start := make(chan struct{})
	end := make(chan struct{})
	// mpb.MaxWidthDistributor shouldn't stuck in the middle while removing or aborting a bar
	mpb.MaxWidthDistributor = makeWrapper(mpb.MaxWidthDistributor, start, end)

	total := 100
	numBars := 6
	p := mpb.New(mpb.WithOutput(io.Discard))
	for i := 0; i < numBars; i++ {
		bar := p.AddBar(int64(total),
			mpb.BarOptional(mpb.BarRemoveOnComplete(), i == 0),
			mpb.PrependDecorators(decor.EwmaETA(decor.ET_STYLE_GO, 60, decor.WCSyncSpace)),
		)
		go func() {
			<-ready
			for i := 0; i < total; i++ {
				start := time.Now()
				if id := bar.ID(); id > 1 && i >= 32 {
					if id&1 == 1 {
						bar.Abort(true)
					} else {
						bar.Abort(false)
					}
				}
				time.Sleep(randomDuration(100 * time.Millisecond))
				bar.EwmaIncrInt64(rand.Int63n(5)+1, time.Since(start))
			}
		}()
	}

	go func() {
		<-ready
		p.Wait()
		close(start)
	}()

	res := t.Run("maxWidthDistributor", func(t *testing.T) {
		close(ready)
		for v := range start {
			timer := time.NewTimer(100 * time.Millisecond)
			select {
			case end <- v:
				timer.Stop()
			case <-timer.C:
				t.FailNow()
			}
		}
	})

	if !res {
		t.Error("maxWidthDistributor stuck in the middle")
	}
}

func TestProgressShutdownsWithErrFiller(t *testing.T) {
	var debug bytes.Buffer
	shutdown := make(chan struct{})
	p := mpb.New(
		mpb.WithShutdownNotifier(shutdown),
		mpb.WithOutput(io.Discard),
		mpb.WithDebugOutput(&debug),
	)

	testError := errors.New("test error")
	bar := p.AddBar(100,
		mpb.BarFillerMiddleware(func(base mpb.BarFiller) mpb.BarFiller {
			return mpb.BarFillerFunc(func(w io.Writer, st decor.Statistics) error {
				if st.Current >= 42 {
					return testError
				}
				return base.Fill(w, st)
			})
		}),
	)

	go func() {
		for bar.IsRunning() {
			bar.Increment()
		}
	}()

	select {
	case <-shutdown:
		if err := strings.TrimSpace(debug.String()); err != testError.Error() {
			t.Errorf("Expected err: %q, got %q\n", testError.Error(), err)
		}
	case <-time.After(timeout):
		t.Errorf("Progress didn't shutdown after %v", timeout)
	}
	p.Wait()
}

func randomDuration(max time.Duration) time.Duration {
	return time.Duration(rand.Intn(10)+1) * max / 10
}
<|endoftext|>


<commit_before>package daemon

import (
	"log"
	"os"
	"time"
	"http"
	"flag"
)

type DaemonLogger struct {
	stdoutLog *log.Logger
	fileLog   *log.Logger
	error     int
	name      string
}

var debug bool

func init() {
	flag.BoolVar(&debug, "d", false, "Log debug information")
}

func NewDaemonLogger(logFilePath string, daemonName string) *DaemonLogger {
	daemonLogger := new(DaemonLogger)
	daemonLogger.name = daemonName
	daemonLogger.stdoutLog = log.New(os.Stdout, "", 0)
	logFile, _ := os.OpenFile(logFilePath+daemonName+".log", os.O_WRONLY|os.O_CREATE, 0666)
	daemonLogger.fileLog = log.New(logFile, "", 0)
	daemonLogger.error = 0
	return daemonLogger
}

func (daemonLogger *DaemonLogger) Log(message string) {
	currentTime := time.LocalTime()
	formatTime := currentTime.Format("Jan _2 15:04:05")
	name, _ := os.Hostname()
	pid := os.Getpid()
	daemonLogger.stdoutLog.Printf("%s %s %s[%d]: %s", formatTime, name, daemonLogger.name, pid, message)
	daemonLogger.fileLog.Printf("%s %s %s[%d]: %s", formatTime, name, daemonLogger.name, pid, message)
}

func (daemonLogger *DaemonLogger) LogError(message string, error os.Error) {
	currentTime := time.LocalTime()
	formatTime := currentTime.Format("Jan _2 15:04:05")
	name, _ := os.Hostname()
	pid := os.Getpid()
	if error != nil {
		daemonLogger.stdoutLog.Printf("%s %s %s[%d]: ERROR %s", formatTime, name, daemonLogger.name, pid, message)
		daemonLogger.fileLog.Printf("%s %s %s[%d]: ERROR %s", formatTime, name, daemonLogger.name, pid, message)
		daemonLogger.error++
	}

}

func (daemonLogger *DaemonLogger) LogHttp(request *http.Request) {
	currentTime := time.LocalTime()
	formatTime := currentTime.Format("Jan _2 15:04:05")
	name, _ := os.Hostname()
	pid := os.Getpid()
	daemonLogger.stdoutLog.Printf("%s %s %s[%d]: %s %s Bytes Recieved: %d", formatTime, name, daemonLogger.name, pid, request.Method, request.RawURL, request.ContentLength)
	daemonLogger.fileLog.Printf("%s %s %s[%d]: %s %s Bytes Recieved: %d", formatTime, name, daemonLogger.name, pid, request.Method, request.RawURL, request.ContentLength)
}

func (daemonLogger *DaemonLogger) DebugHttp(request *http.Request) {
	if debug {
		currentTime := time.LocalTime()
		formatTime := currentTime.Format("Jan _2 15:04:05")
		name, _ := os.Hostname()
		pid := os.Getpid()
		daemonLogger.stdoutLog.Printf("%s %s %s[%d]: %s %s Bytes Recieved: %d", formatTime, name, daemonLogger.name, pid, request.Method, request.RawURL, request.ContentLength)
		daemonLogger.fileLog.Printf("%s %s %s[%d]: %s %s Bytes Recieved: %d", formatTime, name, daemonLogger.name, pid, request.Method, request.RawURL, request.ContentLength)
	}
}

func (daemonLogger *DaemonLogger) LogDebug(message string) {
	currentTime := time.LocalTime()
	formatTime := currentTime.Format("Jan _2 15:04:05")
	name, _ := os.Hostname()
	pid := os.Getpid()
	if debug {
		daemonLogger.stdoutLog.Printf("%s %s %s[%d]: DEBUG %s", formatTime, name, daemonLogger.name, pid, message)
		daemonLogger.fileLog.Printf("%s %s %s[%d]: ERROR %s", formatTime, name, daemonLogger.name, pid, message)
	}
}

func (daemonLogger *DaemonLogger) ReturnError() int {
	return daemonLogger.error
}
<commit_msg>Fix logging<commit_after>package daemon

import (
	"fmt"
	"log"
	"os"
	"http"
	"flag"
)

type DaemonLogger struct {
	stdoutLog *log.Logger
	fileLog   *log.Logger
	error     int
}

var debug bool

func init() {
	flag.BoolVar(&debug, "d", false, "Log debug information")
}

func NewDaemonLogger(logFilePath string, daemonName string) *DaemonLogger {
	daemonLogger := new(DaemonLogger)
	hostname, _ := os.Hostname()
	pid := os.Getpid()
	header := fmt.Sprintf("%s %s[%d]: ", hostname, daemonName, pid)
	daemonLogger.stdoutLog = log.New(os.Stdout, header, log.LstdFlags)
	logFile, _ := os.OpenFile(logFilePath+daemonName+".log", os.O_WRONLY|os.O_CREATE, 0666)
	logFile.Seek(0, 2)
	daemonLogger.fileLog = log.New(logFile, header, log.LstdFlags)
	daemonLogger.error = 0
	return daemonLogger
}

func (dLog DaemonLogger) PrintAll(message string) {
	dLog.stdoutLog.Print(message)
	dLog.fileLog.Print(message)
}

func (dLog *DaemonLogger) Log(message string) {
	dLog.PrintAll(message)
}

func (dLog *DaemonLogger) LogError(message string, error os.Error) {
	if error != nil {
		dLog.PrintAll(message)
		dLog.error++
	}
}

func (dLog *DaemonLogger) LogHttp(request *http.Request) {
	msg := fmt.Sprintf("%s %s Bytes Recieved: %d", request.Method, request.RawURL, request.ContentLength)
	dLog.PrintAll(msg)
}

func (dLog *DaemonLogger) DebugHttp(request *http.Request) {
	if debug {
		msg := fmt.Sprintf("%s %s Bytes Recieved: %d", request.Method, request.RawURL, request.ContentLength)
		dLog.PrintAll(msg)
	}
}

func (dLog *DaemonLogger) LogDebug(message string) {
	if debug {
		dLog.PrintAll(message)
	}
}

func (daemonLogger *DaemonLogger) ReturnError() int {
	return daemonLogger.error
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package dwarf provides access to DWARF debugging information loaded from
// executable files, as defined in the DWARF 2.0 Standard at
// http://dwarfstd.org/doc/dwarf-2.0.0.pdf
package dwarf

import "encoding/binary"

// Data represents the DWARF debugging information
// loaded from an executable file (for example, an ELF or Mach-O executable).
type Data struct {
	// raw data
	abbrev   []byte
	aranges  []byte
	frame    []byte
	info     []byte
	line     []byte
	pubnames []byte
	ranges   []byte
	str      []byte

	// parsed data
	abbrevCache map[uint32]abbrevTable
	addrsize    int
	order       binary.ByteOrder
	typeCache   map[Offset]Type
	unit        []unit
}

// New returns a new Data object initialized from the given parameters.
// Rather than calling this function directly, clients should typically use
// the DWARF method of the File type of the appropriate package debug/elf,
// debug/macho, or debug/pe.
//
// The []byte arguments are the data from the corresponding debug section
// in the object file; for example, for an ELF object, abbrev is the contents of
// the ".debug_abbrev" section.
func New(abbrev, aranges, frame, info, line, pubnames, ranges, str []byte) (*Data, error) {
	d := &Data{
		abbrev:      abbrev,
		aranges:     aranges,
		frame:       frame,
		info:        info,
		line:        line,
		pubnames:    pubnames,
		ranges:      ranges,
		str:         str,
		abbrevCache: make(map[uint32]abbrevTable),
		typeCache:   make(map[Offset]Type),
	}

	// Sniff .debug_info to figure out byte order.
	// bytes 4:6 are the version, a tiny 16-bit number (1, 2, 3).
	if len(d.info) < 6 {
		return nil, DecodeError{"info", Offset(len(d.info)), "too short"}
	}
	x, y := d.info[4], d.info[5]
	switch {
	case x == 0 && y == 0:
		return nil, DecodeError{"info", 4, "unsupported version 0"}
	case x == 0:
		d.order = binary.BigEndian
	case y == 0:
		d.order = binary.LittleEndian
	default:
		return nil, DecodeError{"info", 4, "cannot determine byte order"}
	}

	u, err := d.parseUnits()
	if err != nil {
		return nil, err
	}
	d.unit = u
	return d, nil
}
<commit_msg>debug/dwarf: remove unused field addrsize from Data<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package dwarf provides access to DWARF debugging information loaded from
// executable files, as defined in the DWARF 2.0 Standard at
// http://dwarfstd.org/doc/dwarf-2.0.0.pdf
package dwarf

import "encoding/binary"

// Data represents the DWARF debugging information
// loaded from an executable file (for example, an ELF or Mach-O executable).
type Data struct {
	// raw data
	abbrev   []byte
	aranges  []byte
	frame    []byte
	info     []byte
	line     []byte
	pubnames []byte
	ranges   []byte
	str      []byte

	// parsed data
	abbrevCache map[uint32]abbrevTable
	order       binary.ByteOrder
	typeCache   map[Offset]Type
	unit        []unit
}

// New returns a new Data object initialized from the given parameters.
// Rather than calling this function directly, clients should typically use
// the DWARF method of the File type of the appropriate package debug/elf,
// debug/macho, or debug/pe.
//
// The []byte arguments are the data from the corresponding debug section
// in the object file; for example, for an ELF object, abbrev is the contents of
// the ".debug_abbrev" section.
func New(abbrev, aranges, frame, info, line, pubnames, ranges, str []byte) (*Data, error) {
	d := &Data{
		abbrev:      abbrev,
		aranges:     aranges,
		frame:       frame,
		info:        info,
		line:        line,
		pubnames:    pubnames,
		ranges:      ranges,
		str:         str,
		abbrevCache: make(map[uint32]abbrevTable),
		typeCache:   make(map[Offset]Type),
	}

	// Sniff .debug_info to figure out byte order.
	// bytes 4:6 are the version, a tiny 16-bit number (1, 2, 3).
	if len(d.info) < 6 {
		return nil, DecodeError{"info", Offset(len(d.info)), "too short"}
	}
	x, y := d.info[4], d.info[5]
	switch {
	case x == 0 && y == 0:
		return nil, DecodeError{"info", 4, "unsupported version 0"}
	case x == 0:
		d.order = binary.BigEndian
	case y == 0:
		d.order = binary.LittleEndian
	default:
		return nil, DecodeError{"info", 4, "cannot determine byte order"}
	}

	u, err := d.parseUnits()
	if err != nil {
		return nil, err
	}
	d.unit = u
	return d, nil
}
<|endoftext|>