<commit_before>/*
 * The MIT License (MIT)
 *
 *  Copyright (c) 2014 Stephen Parker (withaspark.com)
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/mail"
	"strings"

	"code.google.com/p/go-smtpd/smtpd"
)

// A message structure
type Message struct {
	From     string `json:"from"`
	To       string `json:"to"`
	path     string
	domain   string
	Subject  string `json:"subject"`
	Body     string `json:"body"`
	spath    string
	sdomain  string
	encoding string
	buffer   bytes.Buffer
}

// Getter of path
func (m *Message) GetPath() string {
	return m.spath
}

// Getter of domain
func (m *Message) GetDomain() string {
	return m.sdomain
}

// Add recipient method for Message types
// Required by package
func (m *Message) AddRecipient(to smtpd.MailAddress) error {
	m.To = strings.ToLower(to.Email())

	// Check if valid domain to receive mail for
	if OptDomainCheckingOn {
		_, dom := SplitToAddress(m.To)
		err := DomainCheck(OptValidDomains, dom)
		if err != nil {
			return fmt.Errorf("%v", err)
		}
	}

	return nil
}

// Add BeginData method for Message types
// Required by package
func (m *Message) BeginData() error {
	return nil
}

// Add parse method for Message types
func (m *Message) parse(r io.Reader) error {
	message, _ := mail.ReadMessage(r)

	// Get headers
	m.To = message.Header.Get("To")
	m.encoding = message.Header.Get("Content-Type")

	// Parse to
	m.spath, m.sdomain = SplitToAddress(m.To)

	// Get subject
	m.Subject = message.Header.Get("Subject")

	// Get body
	tempbuf := new(bytes.Buffer)
	tempbuf.ReadFrom(message.Body)
	mediaType, args, _ := mime.ParseMediaType(m.encoding)
	if strings.HasPrefix(mediaType, "multipart") {
		part := multipart.NewReader(tempbuf, args["boundary"])
		for {
			nPart, err := part.NextPart()
			// If reached end of message, stop looping
			if err == io.EOF {
				return nil
			// Pass errors
			} else if err != nil {
				return err
			// Grab the text/plain version
			} else if strings.Contains(nPart.Header.Get("Content-Type"), "text/plain") {
				tempPart, _ := ioutil.ReadAll(nPart)
				m.Body = strings.Replace(strings.Replace(strings.TrimSpace(string(tempPart)), "\r\n", " ", -1), "\n", " ", -1)
			// Message had no text/plain formatting
			//TODO: One day add html parsing to strip tags
			} else {
				return errors.New("Error: No text/plain formatting of message.")
			}
		}
	}

	return nil
}

// Add Close method for Message types
// Required by package
func (m *Message) Close() error {
	// Do some processing here
	m.parse(&m.buffer)

	// Display what we received
	log.Printf("Received message from %s: [%s] %s\n", m.From, m.Subject, m.Body)

	// Send to application for processing
	a, err := NewAction(*m)
	if err != nil {
		return errors.New("Error: Failed to parse action")
	}
	a.Do()

	return nil
}

// Add Write method for Message types
// Required by package
func (m *Message) Write(line []byte) error {
	m.buffer.Write(line)
	return nil
}

// Extract project and domain from email address
func SplitToAddress(sTo string) (string, string) {
	var sMailbox, sDomain string

	// Separate address into mailbox@domain
	sToPieces := strings.Split(sTo, "@")
	sMailbox = sToPieces[0]
	sDomain = sToPieces[1]

	// If + in mailbox, use those to build path to API dest
	sMailbox = OptAPIRoute + strings.Replace(sMailbox, "+", "/", -1)

	return sMailbox, sDomain
}

// Determines if we should handle this domain
func DomainCheck(domains []string, domain string) error {
	var bMatch bool = false

	// Iterate over domains set in config
	for _, el := range domains {
		if strings.HasSuffix(domain, el) {
			bMatch = true
			break
		}
	}
	if !bMatch {
		return fmt.Errorf("Not accepting email for domain: %s", domain)
	}
	return nil
}
<commit_msg>Bug 146: Add time field to post to backend.<commit_after>/*
 * The MIT License (MIT)
 *
 *  Copyright (c) 2014 Stephen Parker (withaspark.com)
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/mail"
	"strings"
	"time"

	"code.google.com/p/go-smtpd/smtpd"
)

// A message structure
type Message struct {
	From     string `json:"from"`
	To       string `json:"to"`
	path     string
	domain   string
	Subject  string `json:"subject"`
	Body     string `json:"body"`
	Time     string `json:"time"`
	spath    string
	sdomain  string
	encoding string
	buffer   bytes.Buffer
}

// Getter of path
func (m *Message) GetPath() string {
	return m.spath
}

// Getter of domain
func (m *Message) GetDomain() string {
	return m.sdomain
}

// Add recipient method for Message types
// Required by package
func (m *Message) AddRecipient(to smtpd.MailAddress) error {
	m.To = strings.ToLower(to.Email())

	// Check if valid domain to receive mail for
	if OptDomainCheckingOn {
		_, dom := SplitToAddress(m.To)
		err := DomainCheck(OptValidDomains, dom)
		if err != nil {
			return fmt.Errorf("%v", err)
		}
	}

	return nil
}

// Add BeginData method for Message types
// Required by package
func (m *Message) BeginData() error {
	return nil
}

// Add parse method for Message types
func (m *Message) parse(r io.Reader) error {
	message, _ := mail.ReadMessage(r)

	// Get headers
	m.To = message.Header.Get("To")
	m.encoding = message.Header.Get("Content-Type")

	// Set time
	m.Time = time.Now().UTC().Format(time.RFC3339)

	// Parse to
	m.spath, m.sdomain = SplitToAddress(m.To)

	// Get subject
	m.Subject = message.Header.Get("Subject")

	// Get body
	tempbuf := new(bytes.Buffer)
	tempbuf.ReadFrom(message.Body)
	mediaType, args, _ := mime.ParseMediaType(m.encoding)
	if strings.HasPrefix(mediaType, "multipart") {
		part := multipart.NewReader(tempbuf, args["boundary"])
		for {
			nPart, err := part.NextPart()
			// If reached end of message, stop looping
			if err == io.EOF {
				return nil
			// Pass errors
			} else if err != nil {
				return err
			// Grab the text/plain version
			} else if strings.Contains(nPart.Header.Get("Content-Type"), "text/plain") {
				tempPart, _ := ioutil.ReadAll(nPart)
				m.Body = strings.Replace(strings.Replace(strings.TrimSpace(string(tempPart)), "\r\n", " ", -1), "\n", " ", -1)
			// Message had no text/plain formatting
			//TODO: One day add html parsing to strip tags
			} else {
				return errors.New("Error: No text/plain formatting of message.")
			}
		}
	}

	return nil
}

// Add Close method for Message types
// Required by package
func (m *Message) Close() error {
	// Do some processing here
	m.parse(&m.buffer)

	// Display what we received
	log.Printf("Received message from %s: [%s] %s\n", m.From, m.Subject, m.Body)

	// Send to application for processing
	a, err := NewAction(*m)
	if err != nil {
		return errors.New("Error: Failed to parse action")
	}
	a.Do()

	return nil
}

// Add Write method for Message types
// Required by package
func (m *Message) Write(line []byte) error {
	m.buffer.Write(line)
	return nil
}

// Extract project and domain from email address
func SplitToAddress(sTo string) (string, string) {
	var sMailbox, sDomain string

	// Separate address into mailbox@domain
	sToPieces := strings.Split(sTo, "@")
	sMailbox = sToPieces[0]
	sDomain = sToPieces[1]

	// If + in mailbox, use those to build path to API dest
	sMailbox = OptAPIRoute + strings.Replace(sMailbox, "+", "/", -1)

	return sMailbox, sDomain
}

// Determines if we should handle this domain
func DomainCheck(domains []string, domain string) error {
	var bMatch bool = false

	// Iterate over domains set in config
	for _, el := range domains {
		if strings.HasSuffix(domain, el) {
			bMatch = true
			break
		}
	}
	if !bMatch {
		return fmt.Errorf("Not accepting email for domain: %s", domain)
	}
	return nil
}
<|endoftext|>


<commit_before>package generator

import (
	"fmt"
	"strings"

	"os"
	"path/filepath"

	"github.com/devimteam/microgen/generator/template"
	"github.com/devimteam/microgen/util"
	"github.com/vetcher/godecl/types"
)

const (
	TagMark         = template.TagMark
	MicrogenMainTag = template.MicrogenMainTag
	ProtobufTag     = "protobuf"
	GRPCRegAddr     = "grpc-addr"

	MiddlewareTag        = template.MiddlewareTag
	LoggingMiddlewareTag = template.LoggingMiddlewareTag
	RecoverMiddlewareTag = template.RecoverMiddlewareTag
	HttpTag              = template.HttpTag
	HttpServerTag        = template.HttpServerTag
	HttpClientTag        = template.HttpClientTag
	GrpcTag              = template.GrpcTag
	GrpcServerTag        = template.GrpcServerTag
	GrpcClientTag        = template.GrpcClientTag
	MainTag              = template.MainTag
)

func ListTemplatesForGen(iface *types.Interface, force bool, importPackageName, absOutPath, sourcePath string) (units []*generationUnit, err error) {
	importPackagePath, err := resolvePackagePath(absOutPath)
	if err != nil {
		return nil, err
	}
	absSourcePath, err := filepath.Abs(sourcePath)
	if err != nil {
		return nil, err
	}
	info := &template.GenerationInfo{
		ServiceImportPackageName: importPackageName,
		ServiceImportPath:        importPackagePath,
		Force:                    force,
		Iface:                    iface,
		AbsOutPath:               absOutPath,
		SourceFilePath:           absSourcePath,
		ProtobufPackage:          fetchMetaInfo(TagMark+ProtobufTag, iface.Docs),
		GRPCRegAddr:              fetchMetaInfo(TagMark+GRPCRegAddr, iface.Docs),
	}
	stubSvc, err := NewGenUnit(template.NewStubInterfaceTemplate(info), absOutPath)
	if err != nil {
		return nil, err
	}
	exch, err := NewGenUnit(template.NewExchangeTemplate(info), absOutPath)
	if err != nil {
		return nil, err
	}
	endp, err := NewGenUnit(template.NewEndpointsTemplate(info), absOutPath)
	if err != nil {
		return nil, err
	}
	units = append(units, stubSvc, exch, endp)

	genTags := util.FetchTags(iface.Docs, TagMark+MicrogenMainTag)
	fmt.Println("Tags:", strings.Join(genTags, ", "))
	for _, tag := range genTags {
		templates := tagToTemplate(tag, info)
		if templates == nil {
			return nil, fmt.Errorf("unexpected tag %s", tag)
		}
		for _, t := range templates {
			unit, err := NewGenUnit(t, absOutPath)
			if err != nil {
				return nil, err
			}
			units = append(units, unit)
		}
	}
	return units, nil
}

// Fetch information from slice of comments (docs).
// Returns appendix of first comment which has tag as prefix.
func fetchMetaInfo(tag string, comments []string) string {
	for _, comment := range comments {
		if len(comment) > len(tag) && strings.HasPrefix(comment, tag) {
			return comment[len(tag)+1:]
		}
	}
	return ""
}

func tagToTemplate(tag string, info *template.GenerationInfo) (tmpls []template.Template) {
	switch tag {
	case MiddlewareTag:
		return append(tmpls, template.NewMiddlewareTemplate(info))
	case LoggingMiddlewareTag:
		return append(tmpls, template.NewLoggingTemplate(info))
	case GrpcTag:
		return append(tmpls,
			template.NewGRPCClientTemplate(info),
			template.NewGRPCServerTemplate(info),
			template.NewGRPCEndpointConverterTemplate(info),
			template.NewStubGRPCTypeConverterTemplate(info),
		)
	case GrpcClientTag:
		return append(tmpls,
			template.NewGRPCClientTemplate(info),
			template.NewGRPCEndpointConverterTemplate(info),
			template.NewStubGRPCTypeConverterTemplate(info),
		)
	case GrpcServerTag:
		return append(tmpls,
			template.NewGRPCServerTemplate(info),
			template.NewGRPCEndpointConverterTemplate(info),
			template.NewStubGRPCTypeConverterTemplate(info),
		)
	case HttpTag:
		return append(tmpls,
			template.NewHttpServerTemplate(info),
			template.NewHttpClientTemplate(info),
			template.NewHttpConverterTemplate(info),
		)
	case HttpServerTag:
		return append(tmpls,
			template.NewHttpServerTemplate(info),
			template.NewHttpConverterTemplate(info),
		)
	case HttpClientTag:
		return append(tmpls,
			template.NewHttpClientTemplate(info),
			template.NewHttpConverterTemplate(info),
		)
	case RecoverMiddlewareTag:
		return append(tmpls, template.NewRecoverTemplate(info))
	case MainTag:
		return append(tmpls, template.NewMainTemplate(info))
	}
	return nil
}

func resolvePackagePath(outPath string) (string, error) {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		return "", fmt.Errorf("GOPATH is empty")
	}

	absOutPath, err := filepath.Abs(outPath)
	if err != nil {
		return "", err
	}

	gopathSrc := filepath.Join(gopath, "src")
	if !strings.HasPrefix(absOutPath, gopathSrc) {
		return "", fmt.Errorf("path not in GOPATH")
	}

	return absOutPath[len(gopathSrc)+1:], nil
}
<commit_msg>fix(generator): remove check for unexpected tag<commit_after>package generator

import (
	"fmt"
	"strings"

	"os"
	"path/filepath"

	"github.com/devimteam/microgen/generator/template"
	"github.com/devimteam/microgen/util"
	"github.com/vetcher/godecl/types"
)

const (
	TagMark         = template.TagMark
	MicrogenMainTag = template.MicrogenMainTag
	ProtobufTag     = "protobuf"
	GRPCRegAddr     = "grpc-addr"

	MiddlewareTag        = template.MiddlewareTag
	LoggingMiddlewareTag = template.LoggingMiddlewareTag
	RecoverMiddlewareTag = template.RecoverMiddlewareTag
	HttpTag              = template.HttpTag
	HttpServerTag        = template.HttpServerTag
	HttpClientTag        = template.HttpClientTag
	GrpcTag              = template.GrpcTag
	GrpcServerTag        = template.GrpcServerTag
	GrpcClientTag        = template.GrpcClientTag
	MainTag              = template.MainTag
)

func ListTemplatesForGen(iface *types.Interface, force bool, importPackageName, absOutPath, sourcePath string) (units []*generationUnit, err error) {
	importPackagePath, err := resolvePackagePath(absOutPath)
	if err != nil {
		return nil, err
	}
	absSourcePath, err := filepath.Abs(sourcePath)
	if err != nil {
		return nil, err
	}
	info := &template.GenerationInfo{
		ServiceImportPackageName: importPackageName,
		ServiceImportPath:        importPackagePath,
		Force:                    force,
		Iface:                    iface,
		AbsOutPath:               absOutPath,
		SourceFilePath:           absSourcePath,
		ProtobufPackage:          fetchMetaInfo(TagMark+ProtobufTag, iface.Docs),
		GRPCRegAddr:              fetchMetaInfo(TagMark+GRPCRegAddr, iface.Docs),
	}
	stubSvc, err := NewGenUnit(template.NewStubInterfaceTemplate(info), absOutPath)
	if err != nil {
		return nil, err
	}
	exch, err := NewGenUnit(template.NewExchangeTemplate(info), absOutPath)
	if err != nil {
		return nil, err
	}
	endp, err := NewGenUnit(template.NewEndpointsTemplate(info), absOutPath)
	if err != nil {
		return nil, err
	}
	units = append(units, stubSvc, exch, endp)

	genTags := util.FetchTags(iface.Docs, TagMark+MicrogenMainTag)
	fmt.Println("Tags:", strings.Join(genTags, ", "))
	for _, tag := range genTags {
		templates := tagToTemplate(tag, info)
		if templates == nil {
			fmt.Printf("Warning! unexpected tag %s\n", tag)
			continue
		}
		for _, t := range templates {
			unit, err := NewGenUnit(t, absOutPath)
			if err != nil {
				return nil, err
			}
			units = append(units, unit)
		}
	}
	return units, nil
}

// Fetch information from slice of comments (docs).
// Returns appendix of first comment which has tag as prefix.
func fetchMetaInfo(tag string, comments []string) string {
	for _, comment := range comments {
		if len(comment) > len(tag) && strings.HasPrefix(comment, tag) {
			return comment[len(tag)+1:]
		}
	}
	return ""
}

func tagToTemplate(tag string, info *template.GenerationInfo) (tmpls []template.Template) {
	switch tag {
	case MiddlewareTag:
		return append(tmpls, template.NewMiddlewareTemplate(info))
	case LoggingMiddlewareTag:
		return append(tmpls, template.NewLoggingTemplate(info))
	case GrpcTag:
		return append(tmpls,
			template.NewGRPCClientTemplate(info),
			template.NewGRPCServerTemplate(info),
			template.NewGRPCEndpointConverterTemplate(info),
			template.NewStubGRPCTypeConverterTemplate(info),
		)
	case GrpcClientTag:
		return append(tmpls,
			template.NewGRPCClientTemplate(info),
			template.NewGRPCEndpointConverterTemplate(info),
			template.NewStubGRPCTypeConverterTemplate(info),
		)
	case GrpcServerTag:
		return append(tmpls,
			template.NewGRPCServerTemplate(info),
			template.NewGRPCEndpointConverterTemplate(info),
			template.NewStubGRPCTypeConverterTemplate(info),
		)
	case HttpTag:
		return append(tmpls,
			template.NewHttpServerTemplate(info),
			template.NewHttpClientTemplate(info),
			template.NewHttpConverterTemplate(info),
		)
	case HttpServerTag:
		return append(tmpls,
			template.NewHttpServerTemplate(info),
			template.NewHttpConverterTemplate(info),
		)
	case HttpClientTag:
		return append(tmpls,
			template.NewHttpClientTemplate(info),
			template.NewHttpConverterTemplate(info),
		)
	case RecoverMiddlewareTag:
		return append(tmpls, template.NewRecoverTemplate(info))
	case MainTag:
		return append(tmpls, template.NewMainTemplate(info))
	}
	return nil
}

func resolvePackagePath(outPath string) (string, error) {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		return "", fmt.Errorf("GOPATH is empty")
	}

	absOutPath, err := filepath.Abs(outPath)
	if err != nil {
		return "", err
	}

	gopathSrc := filepath.Join(gopath, "src")
	if !strings.HasPrefix(absOutPath, gopathSrc) {
		return "", fmt.Errorf("path not in GOPATH")
	}

	return absOutPath[len(gopathSrc)+1:], nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"sync"
	"time"

	"github.com/nyxtom/broadcast/client/go/broadcast"
)

var ip = flag.String("ip", "127.0.0.1", "broadcast-server ip")
var port = flag.Int("port", 7331, "broadcast-server port")
var number = flag.Int("n", 1000, "request number")
var clients = flag.Int("c", 50, "number of clients")

var wg sync.WaitGroup
var client *broadcast.Client

var loop = 0

func waitBenchAsync(cmd string, args ...interface{}) {
	defer wg.Done()

	c := client.Get()
	defer client.CloseConnection(c)
	for i := 0; i < loop; i++ {
		err := c.DoAsync(cmd, args...)
		if err != nil {
			fmt.Printf("do %s error %s", cmd, err.Error())
			return
		}
	}
}

func waitBench(cmd string, args ...interface{}) {
	defer wg.Done()

	c := client.Get()
	defer client.CloseConnection(c)
	for i := 0; i < loop; i++ {
		_, err := c.Do(cmd, args...)
		if err != nil {
			fmt.Printf("do %s error %s", cmd, err.Error())
			return
		}
	}
}

func bench(cmd string, f func()) {
	wg.Add(*clients)

	t1 := time.Now().UnixNano()
	for i := 0; i < *clients; i++ {
		go f()
	}

	wg.Wait()

	t2 := time.Now().UnixNano()
	delta := float64(t2-t1) / float64(time.Second)

	fmt.Printf("%s: %0.2f requests per second\n", cmd, (float64(*number) / delta))
}

func benchSet() {
	n := rand.Int()
	f := func() {
		waitBenchAsync("SET", string(n), n)
	}

	bench("SET", f)
}

func benchPing() {
	f := func() {
		waitBench("PING")
	}

	bench("PING", f)
}

func benchIncr() {
	n := rand.Int()
	f := func() {
		waitBenchAsync("INCR", string(n), 1)
	}

	bench("INCR", f)
}

func benchDecr() {
	n := rand.Int()
	f := func() {
		waitBenchAsync("DECR", string(n), 1)
	}

	bench("DECR", f)
}

func benchGet() {
	f := func() {
		waitBench("GET", "foo")
	}

	bench("GET", f)
}

func benchDel() {
	f := func() {
		waitBenchAsync("DEL", "foo")
	}

	bench("DEL", f)
}

func benchCount() {
	f := func() {
		waitBenchAsync("COUNT", "foo", 1)
	}

	bench("COUNT", f)
}

func benchSetNx() {
	n := rand.Int()
	f := func() {
		waitBenchAsync("SETNX", string(n), n)
	}

	bench("SETNX", f)
}

func main() {
	flag.Parse()

	if *number <= 0 {
		panic("invalid number")
		return
	}

	if *clients <= 0 || *number < *clients {
		panic("invalid client number")
		return
	}

	loop = *number / *clients
	client, _ = broadcast.NewClient(*port, *ip, 1)
	benchSet()
	benchPing()
	benchGet()
	benchDel()
	benchIncr()
	benchDecr()
	benchCount()
	benchSetNx()
}
<commit_msg>Updating benchmark to reflect async/non-async commands<commit_after>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"sync"
	"time"

	"github.com/nyxtom/broadcast/client/go/broadcast"
)

var ip = flag.String("ip", "127.0.0.1", "broadcast-server ip")
var port = flag.Int("port", 7331, "broadcast-server port")
var number = flag.Int("n", 1000, "request number")
var clients = flag.Int("c", 50, "number of clients")

var wg sync.WaitGroup
var client *broadcast.Client

var loop = 0

func waitBenchAsync(cmd string, args ...interface{}) {
	defer wg.Done()

	c := client.Get()
	defer client.CloseConnection(c)
	for i := 0; i < loop; i++ {
		err := c.DoAsync(cmd, args...)
		if err != nil {
			fmt.Printf("do %s error %s", cmd, err.Error())
			return
		}
	}
}

func waitBench(cmd string, args ...interface{}) {
	defer wg.Done()

	c := client.Get()
	defer client.CloseConnection(c)
	for i := 0; i < loop; i++ {
		_, err := c.Do(cmd, args...)
		if err != nil {
			fmt.Printf("do %s error %s", cmd, err.Error())
			return
		}
	}
}

func bench(cmd string, f func()) {
	wg.Add(*clients)

	t1 := time.Now().UnixNano()
	for i := 0; i < *clients; i++ {
		go f()
	}

	wg.Wait()

	t2 := time.Now().UnixNano()
	delta := float64(t2-t1) / float64(time.Second)

	fmt.Printf("%s: %0.2f requests per second\n", cmd, (float64(*number) / delta))
}

func benchSet() {
	n := rand.Int()
	f := func() {
		waitBench("SET", string(n), n)
	}

	bench("SET", f)
}

func benchPing() {
	f := func() {
		waitBench("PING")
	}

	bench("PING", f)
}

func benchIncr() {
	n := rand.Int()
	f := func() {
		waitBench("INCR", string(n), 1)
	}

	bench("INCR", f)
}

func benchDecr() {
	n := rand.Int()
	f := func() {
		waitBench("DECR", string(n), 1)
	}

	bench("DECR", f)
}

func benchGet() {
	f := func() {
		waitBench("GET", "foo")
	}

	bench("GET", f)
}

func benchDel() {
	f := func() {
		waitBench("DEL", "foo")
	}

	bench("DEL", f)
}

func benchCount() {
	f := func() {
		waitBenchAsync("COUNT", "foo", 1)
	}

	bench("COUNT", f)
}

func benchSetNx() {
	n := rand.Int()
	f := func() {
		waitBench("SETNX", string(n), n)
	}

	bench("SETNX", f)
}

func main() {
	flag.Parse()

	if *number <= 0 {
		panic("invalid number")
		return
	}

	if *clients <= 0 || *number < *clients {
		panic("invalid client number")
		return
	}

	loop = *number / *clients
	client, _ = broadcast.NewClient(*port, *ip, 1)
	benchSet()
	benchPing()
	benchGet()
	benchDel()
	benchIncr()
	benchDecr()
	benchCount()
	benchSetNx()
}
<|endoftext|>


<commit_before>package tui

import (
	"image"

	"github.com/gdamore/tcell"
)

var _ UI = &tcellUI{}

type tcellUI struct {
	painter *Painter
	root    Widget

	keybindings []*keybinding

	quit chan struct{}

	screen tcell.Screen

	kbFocus *kbFocusController

	eventQueue chan event
}

func newTcellUI(root Widget) (*tcellUI, error) {
	screen, err := tcell.NewScreen()
	if err != nil {
		return nil, err
	}

	s := &tcellSurface{
		screen: screen,
	}
	p := NewPainter(s, DefaultTheme)

	return &tcellUI{
		painter:     p,
		root:        root,
		keybindings: make([]*keybinding, 0),
		quit:        make(chan struct{}, 1),
		screen:      screen,
		kbFocus:     &kbFocusController{chain: DefaultFocusChain},
		eventQueue:  make(chan event),
	}, nil
}

func (ui *tcellUI) SetWidget(w Widget) {
	ui.root = w
}

func (ui *tcellUI) SetTheme(t *Theme) {
	ui.painter.theme = t
}

func (ui *tcellUI) SetFocusChain(chain FocusChain) {
	if ui.kbFocus.focusedWidget != nil {
		ui.kbFocus.focusedWidget.SetFocused(false)
	}

	ui.kbFocus.chain = chain
	ui.kbFocus.focusedWidget = chain.FocusDefault()

	if ui.kbFocus.focusedWidget != nil {
		ui.kbFocus.focusedWidget.SetFocused(true)
	}
}

func (ui *tcellUI) SetKeybinding(seq string, fn func()) {
	ui.keybindings = append(ui.keybindings, &keybinding{
		sequence: seq,
		handler:  fn,
	})
}

// ClearKeybindings reinitialises ui.keybindings so as to revert to a
// clear/original state
func (ui *tcellUI) ClearKeybindings() {
	ui.keybindings = make([]*keybinding, 0)
}

func (ui *tcellUI) Run() error {
	if err := ui.screen.Init(); err != nil {
		return err
	}

	defer func() {
		if r := recover(); r != nil {
			ui.screen.Fini()
			logger.Printf("Panic: %s", r)
		}
	}()

	if w := ui.kbFocus.chain.FocusDefault(); w != nil {
		w.SetFocused(true)
		ui.kbFocus.focusedWidget = w
	}

	ui.screen.SetStyle(tcell.StyleDefault)
	ui.screen.EnableMouse()
	ui.screen.Clear()

	go func() {
		for {
			switch ev := ui.screen.PollEvent().(type) {
			case *tcell.EventKey:
				ui.handleKeyEvent(ev)
			case *tcell.EventMouse:
				ui.handleMouseEvent(ev)
			case *tcell.EventResize:
				ui.handleResizeEvent(ev)
			}
		}
	}()

	for {
		select {
		case <-ui.quit:
			return nil
		case ev := <-ui.eventQueue:
			ui.handleEvent(ev)
		}
	}
}

func (ui *tcellUI) handleEvent(ev event) {
	switch e := ev.(type) {
	case KeyEvent:
		logger.Printf("Received key event: %s", e.Name())

		for _, b := range ui.keybindings {
			if b.match(e) {
				b.handler()
			}
		}
		ui.kbFocus.OnKeyEvent(e)
		ui.root.OnKeyEvent(e)
		ui.painter.Repaint(ui.root)
	case callbackEvent:
		// Gets stuck in a print loop when the logger is a widget.
		//logger.Printf("Received callback event")
		e.cbFn()
		ui.painter.Repaint(ui.root)
	case paintEvent:
		logger.Printf("Received paint event")
		ui.painter.Repaint(ui.root)
	}
}

func (ui *tcellUI) handleKeyEvent(tev *tcell.EventKey) {
	ui.eventQueue <- KeyEvent{
		Key:       Key(tev.Key()),
		Rune:      tev.Rune(),
		Modifiers: ModMask(tev.Modifiers()),
	}
}

func (ui *tcellUI) handleMouseEvent(ev *tcell.EventMouse) {
	x, y := ev.Position()
	ui.eventQueue <- MouseEvent{Pos: image.Pt(x, y)}
}

func (ui *tcellUI) handleResizeEvent(ev *tcell.EventResize) {
	ui.eventQueue <- paintEvent{}
}

// Quit signals to the UI to start shutting down.
func (ui *tcellUI) Quit() {
	logger.Printf("Quitting")
	ui.screen.Fini()
	ui.quit <- struct{}{}
}

// Schedule an update of the UI, running the given
// function in the UI goroutine.
//
// Use this to update the UI in response to external events,
// like a timer tick.
// This method should be used any time you call methods
// to change UI objects after the first call to `UI.Run()`.
//
// Changes invoked outside of either this callback or the
// other event handler callbacks may appear to work, but
// is likely a race condition.  (Run your program with
// `go run -race` or `go install -race` to detect this!)
//
// Calling Update from within an event handler, or from within an Update call,
// is an error, and will deadlock.
func (ui *tcellUI) Update(fn func()) {
	blk := make(chan struct{})
	ui.eventQueue <- callbackEvent{func() {
		fn()
		close(blk)
	}}
	<-blk
}

var _ Surface = &tcellSurface{}

type tcellSurface struct {
	screen tcell.Screen
}

func (s *tcellSurface) SetCell(x, y int, ch rune, style Style) {
	st := tcell.StyleDefault.Normal().
		Foreground(convertColor(style.Fg, false)).
		Background(convertColor(style.Bg, false)).
		Reverse(style.Reverse == DecorationOn).
		Bold(style.Bold == DecorationOn).
		Underline(style.Underline == DecorationOn)

	s.screen.SetContent(x, y, ch, nil, st)
}

func (s *tcellSurface) SetCursor(x, y int) {
	s.screen.ShowCursor(x, y)
}

func (s *tcellSurface) HideCursor() {
	s.screen.HideCursor()
}

func (s *tcellSurface) Begin() {
	s.screen.Clear()
}

func (s *tcellSurface) End() {
	s.screen.Show()
}

func (s *tcellSurface) Size() image.Point {
	w, h := s.screen.Size()
	return image.Point{w, h}
}

func convertColor(col Color, fg bool) tcell.Color {
	switch col {
	case ColorDefault:
		if fg {
			return tcell.ColorWhite
		}
		return tcell.ColorDefault
	case ColorBlack:
		return tcell.ColorBlack
	case ColorWhite:
		return tcell.ColorWhite
	case ColorRed:
		return tcell.ColorRed
	case ColorGreen:
		return tcell.ColorGreen
	case ColorBlue:
		return tcell.ColorBlue
	case ColorCyan:
		return tcell.ColorDarkCyan
	case ColorMagenta:
		return tcell.ColorDarkMagenta
	case ColorYellow:
		return tcell.ColorYellow
	default:
		if col > 0 {
			return tcell.Color(col)
		}
		return tcell.ColorDefault
	}
}
<commit_msg>disable tcell-mouse until tui-go supports mouse events<commit_after>package tui

import (
	"image"

	"github.com/gdamore/tcell"
)

var _ UI = &tcellUI{}

type tcellUI struct {
	painter *Painter
	root    Widget

	keybindings []*keybinding

	quit chan struct{}

	screen tcell.Screen

	kbFocus *kbFocusController

	eventQueue chan event
}

func newTcellUI(root Widget) (*tcellUI, error) {
	screen, err := tcell.NewScreen()
	if err != nil {
		return nil, err
	}

	s := &tcellSurface{
		screen: screen,
	}
	p := NewPainter(s, DefaultTheme)

	return &tcellUI{
		painter:     p,
		root:        root,
		keybindings: make([]*keybinding, 0),
		quit:        make(chan struct{}, 1),
		screen:      screen,
		kbFocus:     &kbFocusController{chain: DefaultFocusChain},
		eventQueue:  make(chan event),
	}, nil
}

func (ui *tcellUI) SetWidget(w Widget) {
	ui.root = w
}

func (ui *tcellUI) SetTheme(t *Theme) {
	ui.painter.theme = t
}

func (ui *tcellUI) SetFocusChain(chain FocusChain) {
	if ui.kbFocus.focusedWidget != nil {
		ui.kbFocus.focusedWidget.SetFocused(false)
	}

	ui.kbFocus.chain = chain
	ui.kbFocus.focusedWidget = chain.FocusDefault()

	if ui.kbFocus.focusedWidget != nil {
		ui.kbFocus.focusedWidget.SetFocused(true)
	}
}

func (ui *tcellUI) SetKeybinding(seq string, fn func()) {
	ui.keybindings = append(ui.keybindings, &keybinding{
		sequence: seq,
		handler:  fn,
	})
}

// ClearKeybindings reinitialises ui.keybindings so as to revert to a
// clear/original state
func (ui *tcellUI) ClearKeybindings() {
	ui.keybindings = make([]*keybinding, 0)
}

func (ui *tcellUI) Run() error {
	if err := ui.screen.Init(); err != nil {
		return err
	}

	defer func() {
		if r := recover(); r != nil {
			ui.screen.Fini()
			logger.Printf("Panic: %s", r)
		}
	}()

	if w := ui.kbFocus.chain.FocusDefault(); w != nil {
		w.SetFocused(true)
		ui.kbFocus.focusedWidget = w
	}

	ui.screen.SetStyle(tcell.StyleDefault)
	ui.screen.Clear()

	go func() {
		for {
			switch ev := ui.screen.PollEvent().(type) {
			case *tcell.EventKey:
				ui.handleKeyEvent(ev)
			case *tcell.EventMouse:
				ui.handleMouseEvent(ev)
			case *tcell.EventResize:
				ui.handleResizeEvent(ev)
			}
		}
	}()

	for {
		select {
		case <-ui.quit:
			return nil
		case ev := <-ui.eventQueue:
			ui.handleEvent(ev)
		}
	}
}

func (ui *tcellUI) handleEvent(ev event) {
	switch e := ev.(type) {
	case KeyEvent:
		logger.Printf("Received key event: %s", e.Name())

		for _, b := range ui.keybindings {
			if b.match(e) {
				b.handler()
			}
		}
		ui.kbFocus.OnKeyEvent(e)
		ui.root.OnKeyEvent(e)
		ui.painter.Repaint(ui.root)
	case callbackEvent:
		// Gets stuck in a print loop when the logger is a widget.
		//logger.Printf("Received callback event")
		e.cbFn()
		ui.painter.Repaint(ui.root)
	case paintEvent:
		logger.Printf("Received paint event")
		ui.painter.Repaint(ui.root)
	}
}

func (ui *tcellUI) handleKeyEvent(tev *tcell.EventKey) {
	ui.eventQueue <- KeyEvent{
		Key:       Key(tev.Key()),
		Rune:      tev.Rune(),
		Modifiers: ModMask(tev.Modifiers()),
	}
}

func (ui *tcellUI) handleMouseEvent(ev *tcell.EventMouse) {
	x, y := ev.Position()
	ui.eventQueue <- MouseEvent{Pos: image.Pt(x, y)}
}

func (ui *tcellUI) handleResizeEvent(ev *tcell.EventResize) {
	ui.eventQueue <- paintEvent{}
}

// Quit signals to the UI to start shutting down.
func (ui *tcellUI) Quit() {
	logger.Printf("Quitting")
	ui.screen.Fini()
	ui.quit <- struct{}{}
}

// Schedule an update of the UI, running the given
// function in the UI goroutine.
//
// Use this to update the UI in response to external events,
// like a timer tick.
// This method should be used any time you call methods
// to change UI objects after the first call to `UI.Run()`.
//
// Changes invoked outside of either this callback or the
// other event handler callbacks may appear to work, but
// is likely a race condition.  (Run your program with
// `go run -race` or `go install -race` to detect this!)
//
// Calling Update from within an event handler, or from within an Update call,
// is an error, and will deadlock.
func (ui *tcellUI) Update(fn func()) {
	blk := make(chan struct{})
	ui.eventQueue <- callbackEvent{func() {
		fn()
		close(blk)
	}}
	<-blk
}

var _ Surface = &tcellSurface{}

type tcellSurface struct {
	screen tcell.Screen
}

func (s *tcellSurface) SetCell(x, y int, ch rune, style Style) {
	st := tcell.StyleDefault.Normal().
		Foreground(convertColor(style.Fg, false)).
		Background(convertColor(style.Bg, false)).
		Reverse(style.Reverse == DecorationOn).
		Bold(style.Bold == DecorationOn).
		Underline(style.Underline == DecorationOn)

	s.screen.SetContent(x, y, ch, nil, st)
}

func (s *tcellSurface) SetCursor(x, y int) {
	s.screen.ShowCursor(x, y)
}

func (s *tcellSurface) HideCursor() {
	s.screen.HideCursor()
}

func (s *tcellSurface) Begin() {
	s.screen.Clear()
}

func (s *tcellSurface) End() {
	s.screen.Show()
}

func (s *tcellSurface) Size() image.Point {
	w, h := s.screen.Size()
	return image.Point{w, h}
}

func convertColor(col Color, fg bool) tcell.Color {
	switch col {
	case ColorDefault:
		if fg {
			return tcell.ColorWhite
		}
		return tcell.ColorDefault
	case ColorBlack:
		return tcell.ColorBlack
	case ColorWhite:
		return tcell.ColorWhite
	case ColorRed:
		return tcell.ColorRed
	case ColorGreen:
		return tcell.ColorGreen
	case ColorBlue:
		return tcell.ColorBlue
	case ColorCyan:
		return tcell.ColorDarkCyan
	case ColorMagenta:
		return tcell.ColorDarkMagenta
	case ColorYellow:
		return tcell.ColorYellow
	default:
		if col > 0 {
			return tcell.Color(col)
		}
		return tcell.ColorDefault
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"io/ioutil"
	"time"

	"github.com/brocaar/lora-app-server/internal/config"
	"github.com/spf13/viper"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var cfgFile string
var version string

var rootCmd = &cobra.Command{
	Use:   "lora-app-server",
	Short: "LoRa Server project application-server",
	Long: `LoRa App Server is an open-source application-server, part of the LoRa Server project
	> documentation & support: https://www.loraserver.io/lora-app-server
	> source & copyright information: https://github.com/brocaar/lora-app-server`,
	RunE: run,
}

func init() {
	cobra.OnInitialize(initConfig)
	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "path to configuration file (optional)")
	rootCmd.PersistentFlags().Int("log-level", 4, "debug=5, info=4, error=2, fatal=1, panic=0")

	// bind flag to config vars
	viper.BindPFlag("general.log_level", rootCmd.PersistentFlags().Lookup("log-level"))

	// defaults
	viper.SetDefault("general.password_hash_iterations", 100000)
	viper.SetDefault("postgresql.dsn", "postgres://localhost/loraserver_as?sslmode=disable")
	viper.SetDefault("postgresql.automigrate", true)
	viper.SetDefault("redis.url", "redis://localhost:6379")
	viper.SetDefault("redis.max_idle", 10)
	viper.SetDefault("redis.idle_timeout", 5*time.Minute)
	viper.SetDefault("application_server.integration.backend", "mqtt")
	viper.SetDefault("application_server.integration.mqtt.server", "tcp://localhost:1883")
	viper.SetDefault("application_server.api.public_host", "localhost:8001")
	viper.SetDefault("application_server.id", "6d5db27e-4ce2-4b2b-b5d7-91f069397978")
	viper.SetDefault("application_server.api.bind", "0.0.0.0:8001")
	viper.SetDefault("application_server.external_api.bind", "0.0.0.0:8080")
	viper.SetDefault("join_server.bind", "0.0.0.0:8003")
	viper.SetDefault("application_server.integration.mqtt.uplink_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/rx")
	viper.SetDefault("application_server.integration.mqtt.downlink_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/tx")
	viper.SetDefault("application_server.integration.mqtt.join_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/join")
	viper.SetDefault("application_server.integration.mqtt.ack_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/ack")
	viper.SetDefault("application_server.integration.mqtt.error_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/error")
	viper.SetDefault("application_server.integration.mqtt.status_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/status")
	viper.SetDefault("application_server.integration.mqtt.location_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/location")
	viper.SetDefault("application_server.integration.mqtt.clean_session", true)

	rootCmd.AddCommand(versionCmd)
	rootCmd.AddCommand(configCmd)
}

// Execute executes the root command.
func Execute(v string) {
	version = v
	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

func initConfig() {
	if cfgFile != "" {
		b, err := ioutil.ReadFile(cfgFile)
		if err != nil {
			log.WithError(err).WithField("config", cfgFile).Fatal("error loading config file")
		}
		viper.SetConfigType("toml")
		if err := viper.ReadConfig(bytes.NewBuffer(b)); err != nil {
			log.WithError(err).WithField("config", cfgFile).Fatal("error loading config file")
		}
	} else {
		viper.SetConfigName("lora-app-server")
		viper.AddConfigPath(".")
		viper.AddConfigPath("$HOME/.config/lora-app-server")
		viper.AddConfigPath("/etc/lora-app-server")
		if err := viper.ReadInConfig(); err != nil {
			switch err.(type) {
			case viper.ConfigFileNotFoundError:
				log.Warning("No configuration file found, using defaults. See: https://www.loraserver.io/lora-app-server/install/config/")
			default:
				log.WithError(err).Fatal("read configuration file error")
			}
		}
	}

	if err := viper.Unmarshal(&config.C); err != nil {
		log.WithError(err).Fatal("unmarshal config error")
	}
}
<commit_msg>Re-introduce support for environment variables.<commit_after>package cmd

import (
	"bytes"
	"io/ioutil"
	"reflect"
	"strings"
	"time"

	"github.com/brocaar/lora-app-server/internal/config"
	"github.com/spf13/viper"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var cfgFile string
var version string

var rootCmd = &cobra.Command{
	Use:   "lora-app-server",
	Short: "LoRa Server project application-server",
	Long: `LoRa App Server is an open-source application-server, part of the LoRa Server project
	> documentation & support: https://www.loraserver.io/lora-app-server
	> source & copyright information: https://github.com/brocaar/lora-app-server`,
	RunE: run,
}

func init() {
	cobra.OnInitialize(initConfig)
	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "path to configuration file (optional)")
	rootCmd.PersistentFlags().Int("log-level", 4, "debug=5, info=4, error=2, fatal=1, panic=0")

	// bind flag to config vars
	viper.BindPFlag("general.log_level", rootCmd.PersistentFlags().Lookup("log-level"))

	// defaults
	viper.SetDefault("general.password_hash_iterations", 100000)
	viper.SetDefault("postgresql.dsn", "postgres://localhost/loraserver_as?sslmode=disable")
	viper.SetDefault("postgresql.automigrate", true)
	viper.SetDefault("redis.url", "redis://localhost:6379")
	viper.SetDefault("redis.max_idle", 10)
	viper.SetDefault("redis.idle_timeout", 5*time.Minute)
	viper.SetDefault("application_server.integration.backend", "mqtt")
	viper.SetDefault("application_server.integration.mqtt.server", "tcp://localhost:1883")
	viper.SetDefault("application_server.api.public_host", "localhost:8001")
	viper.SetDefault("application_server.id", "6d5db27e-4ce2-4b2b-b5d7-91f069397978")
	viper.SetDefault("application_server.api.bind", "0.0.0.0:8001")
	viper.SetDefault("application_server.external_api.bind", "0.0.0.0:8080")
	viper.SetDefault("join_server.bind", "0.0.0.0:8003")
	viper.SetDefault("application_server.integration.mqtt.uplink_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/rx")
	viper.SetDefault("application_server.integration.mqtt.downlink_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/tx")
	viper.SetDefault("application_server.integration.mqtt.join_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/join")
	viper.SetDefault("application_server.integration.mqtt.ack_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/ack")
	viper.SetDefault("application_server.integration.mqtt.error_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/error")
	viper.SetDefault("application_server.integration.mqtt.status_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/status")
	viper.SetDefault("application_server.integration.mqtt.location_topic_template", "application/{{ .ApplicationID }}/device/{{ .DevEUI }}/location")
	viper.SetDefault("application_server.integration.mqtt.clean_session", true)

	rootCmd.AddCommand(versionCmd)
	rootCmd.AddCommand(configCmd)
}

// Execute executes the root command.
func Execute(v string) {
	version = v
	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

func initConfig() {
	if cfgFile != "" {
		b, err := ioutil.ReadFile(cfgFile)
		if err != nil {
			log.WithError(err).WithField("config", cfgFile).Fatal("error loading config file")
		}
		viper.SetConfigType("toml")
		if err := viper.ReadConfig(bytes.NewBuffer(b)); err != nil {
			log.WithError(err).WithField("config", cfgFile).Fatal("error loading config file")
		}
	} else {
		viper.SetConfigName("lora-app-server")
		viper.AddConfigPath(".")
		viper.AddConfigPath("$HOME/.config/lora-app-server")
		viper.AddConfigPath("/etc/lora-app-server")
		if err := viper.ReadInConfig(); err != nil {
			switch err.(type) {
			case viper.ConfigFileNotFoundError:
				log.Warning("No configuration file found, using defaults. See: https://www.loraserver.io/lora-app-server/install/config/")
			default:
				log.WithError(err).Fatal("read configuration file error")
			}
		}
	}

	viperBindEnvs(config.C)

	if err := viper.Unmarshal(&config.C); err != nil {
		log.WithError(err).Fatal("unmarshal config error")
	}
}

func viperBindEnvs(iface interface{}, parts ...string) {
	ifv := reflect.ValueOf(iface)
	ift := reflect.TypeOf(iface)
	for i := 0; i < ift.NumField(); i++ {
		v := ifv.Field(i)
		t := ift.Field(i)
		tv, ok := t.Tag.Lookup("mapstructure")
		if !ok {
			tv = strings.ToLower(t.Name)
		}
		if tv == "-" {
			continue
		}

		switch v.Kind() {
		case reflect.Struct:
			viperBindEnvs(v.Interface(), append(parts, tv)...)
		default:
			key := strings.Join(append(parts, tv), ".")
			viper.BindEnv(key)
		}
	}
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"os/exec"
	"strings"
	"time"

	receptorrunner "github.com/cloudfoundry-incubator/receptor/cmd/receptor/testrunner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry/gunk/diegonats"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/cloudfoundry/storeadapter"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var _ = Describe("Syncing desired state with CC", func() {
	var (
		gnatsdProcess ifrit.Process
		natsClient    diegonats.NATSClient
		bbs           *Bbs.BBS

		receptorProcess ifrit.Process

		runner  ifrit.Runner
		process ifrit.Process

		etcdAdapter storeadapter.StoreAdapter
	)

	startNATS := func() {
		gnatsdProcess, natsClient = diegonats.StartGnatsd(natsPort)
	}

	stopNATS := func() {
		ginkgomon.Kill(gnatsdProcess)
	}

	startReceptor := func() ifrit.Process {
		return ginkgomon.Invoke(receptorrunner.New(receptorPath, receptorrunner.Args{
			Address:     fmt.Sprintf("127.0.0.1:%d", receptorPort),
			EtcdCluster: strings.Join(etcdRunner.NodeURLS(), ","),
		}))
	}

	newNSyncRunner := func() *ginkgomon.Runner {
		return ginkgomon.New(ginkgomon.Config{
			Name:          "nsync",
			AnsiColorCode: "97m",
			StartCheck:    "nsync.listener.started",
			Command: exec.Command(
				listenerPath,
				"-etcdCluster", strings.Join(etcdRunner.NodeURLS(), ","),
				"-diegoAPIURL", fmt.Sprintf("http://127.0.0.1:%d", receptorPort),
				"-natsAddresses", fmt.Sprintf("127.0.0.1:%d", natsPort),
				"-circuses", `{"some-stack": "some-health-check.tar.gz"}`,
				"-dockerCircusPath", "the/docker/circus/path.tgz",
				"-fileServerURL", "http://file-server.com",
				"-heartbeatInterval", "1s",
				"-logLevel", "debug",
			),
		})
	}

	BeforeEach(func() {
		etcdAdapter = etcdRunner.Adapter()
		bbs = Bbs.NewBBS(etcdAdapter, timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))
		receptorProcess = startReceptor()
		runner = newNSyncRunner()
	})

	AfterEach(func() {
		etcdAdapter.Disconnect()
		ginkgomon.Interrupt(receptorProcess)
	})

	var publishDesireWithInstances = func(nInstances int) {
		err := natsClient.Publish("diego.desire.app", []byte(fmt.Sprintf(`
      {
        "process_guid": "the-guid",
        "droplet_uri": "http://the-droplet.uri.com",
        "start_command": "the-start-command",
        "memory_mb": 128,
        "disk_mb": 512,
        "file_descriptors": 32,
        "num_instances": %d,
        "stack": "some-stack",
        "log_guid": "the-log-guid"
      }
    `, nInstances)))
		Ω(err).ShouldNot(HaveOccurred())
	}

	Context("when NATS is up", func() {
		BeforeEach(func() {
			startNATS()
		})

		AfterEach(func() {
			stopNATS()
		})

		Context("and the nsync listener is started", func() {
			BeforeEach(func() {
				process = ginkgomon.Invoke(runner)
			})

			AfterEach(func() {
				ginkgomon.Interrupt(process)
			})

			Describe("and a 'diego.desire.app' message is recieved", func() {
				BeforeEach(func() {
					publishDesireWithInstances(3)
				})

				It("registers an app desire in etcd", func() {
					Eventually(bbs.DesiredLRPs, 10).Should(HaveLen(1))
				})

				Context("when an app is no longer desired", func() {
					BeforeEach(func() {
						Eventually(bbs.DesiredLRPs).Should(HaveLen(1))

						publishDesireWithInstances(0)
					})

					It("should remove the desired state from etcd", func() {
						Eventually(bbs.DesiredLRPs).Should(HaveLen(0))
					})
				})
			})

			Context("and a second nsync listener is started", func() {
				var (
					secondRunner  *ginkgomon.Runner
					secondProcess ifrit.Process
				)

				BeforeEach(func() {
					secondRunner = newNSyncRunner()
					secondRunner.StartCheck = ""

					secondProcess = ginkgomon.Invoke(secondRunner)
				})

				AfterEach(func() {
					ginkgomon.Interrupt(secondProcess)
				})

				Describe("the second listener", func() {
					It("does not become active", func() {
						Consistently(secondRunner.Buffer, 5*time.Second).ShouldNot(gbytes.Say("nsync.listener.started"))
					})
				})

				Context("and the first listener goes away", func() {
					BeforeEach(func() {
						ginkgomon.Interrupt(process)
					})

					Describe("the second listener", func() {
						It("eventually becomes active", func() {
							Eventually(secondRunner.Buffer, 5*time.Second).Should(gbytes.Say("nsync.listener.started"))
						})
					})
				})
			})
		})
	})

	Describe("when NATS is not up", func() {
		Context("and the nsync listener is started", func() {

			BeforeEach(func() {
				process = ifrit.Background(runner)
			})

			AfterEach(func() {
				defer stopNATS()
				defer ginkgomon.Interrupt(process)
			})

			It("starts only after nats comes up", func() {
				Consistently(process.Ready()).ShouldNot(BeClosed())

				startNATS()
				Eventually(process.Ready(), 5*time.Second).Should(BeClosed())
			})
		})
	})
})
<commit_msg>Bump interrupt timeout in main test<commit_after>package main_test

import (
	"fmt"
	"os/exec"
	"strings"
	"time"

	receptorrunner "github.com/cloudfoundry-incubator/receptor/cmd/receptor/testrunner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry/gunk/diegonats"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/cloudfoundry/storeadapter"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var _ = Describe("Syncing desired state with CC", func() {
	var (
		gnatsdProcess ifrit.Process
		natsClient    diegonats.NATSClient
		bbs           *Bbs.BBS

		receptorProcess ifrit.Process

		runner  ifrit.Runner
		process ifrit.Process

		etcdAdapter storeadapter.StoreAdapter
	)

	startNATS := func() {
		gnatsdProcess, natsClient = diegonats.StartGnatsd(natsPort)
	}

	stopNATS := func() {
		ginkgomon.Kill(gnatsdProcess)
	}

	startReceptor := func() ifrit.Process {
		return ginkgomon.Invoke(receptorrunner.New(receptorPath, receptorrunner.Args{
			Address:     fmt.Sprintf("127.0.0.1:%d", receptorPort),
			EtcdCluster: strings.Join(etcdRunner.NodeURLS(), ","),
		}))
	}

	newNSyncRunner := func() *ginkgomon.Runner {
		return ginkgomon.New(ginkgomon.Config{
			Name:          "nsync",
			AnsiColorCode: "97m",
			StartCheck:    "nsync.listener.started",
			Command: exec.Command(
				listenerPath,
				"-etcdCluster", strings.Join(etcdRunner.NodeURLS(), ","),
				"-diegoAPIURL", fmt.Sprintf("http://127.0.0.1:%d", receptorPort),
				"-natsAddresses", fmt.Sprintf("127.0.0.1:%d", natsPort),
				"-circuses", `{"some-stack": "some-health-check.tar.gz"}`,
				"-dockerCircusPath", "the/docker/circus/path.tgz",
				"-fileServerURL", "http://file-server.com",
				"-heartbeatInterval", "1s",
				"-logLevel", "debug",
			),
		})
	}

	BeforeEach(func() {
		etcdAdapter = etcdRunner.Adapter()
		bbs = Bbs.NewBBS(etcdAdapter, timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))
		receptorProcess = startReceptor()
		runner = newNSyncRunner()
	})

	AfterEach(func() {
		etcdAdapter.Disconnect()
		ginkgomon.Interrupt(receptorProcess, 2*time.Second)
	})

	var publishDesireWithInstances = func(nInstances int) {
		err := natsClient.Publish("diego.desire.app", []byte(fmt.Sprintf(`
      {
        "process_guid": "the-guid",
        "droplet_uri": "http://the-droplet.uri.com",
        "start_command": "the-start-command",
        "memory_mb": 128,
        "disk_mb": 512,
        "file_descriptors": 32,
        "num_instances": %d,
        "stack": "some-stack",
        "log_guid": "the-log-guid"
      }
    `, nInstances)))
		Ω(err).ShouldNot(HaveOccurred())
	}

	Context("when NATS is up", func() {
		BeforeEach(func() {
			startNATS()
		})

		AfterEach(func() {
			stopNATS()
		})

		Context("and the nsync listener is started", func() {
			BeforeEach(func() {
				process = ginkgomon.Invoke(runner)
			})

			AfterEach(func() {
				ginkgomon.Interrupt(process, 2*time.Second)
			})

			Describe("and a 'diego.desire.app' message is recieved", func() {
				BeforeEach(func() {
					publishDesireWithInstances(3)
				})

				It("registers an app desire in etcd", func() {
					Eventually(bbs.DesiredLRPs, 10).Should(HaveLen(1))
				})

				Context("when an app is no longer desired", func() {
					BeforeEach(func() {
						Eventually(bbs.DesiredLRPs).Should(HaveLen(1))

						publishDesireWithInstances(0)
					})

					It("should remove the desired state from etcd", func() {
						Eventually(bbs.DesiredLRPs).Should(HaveLen(0))
					})
				})
			})

			Context("and a second nsync listener is started", func() {
				var (
					secondRunner  *ginkgomon.Runner
					secondProcess ifrit.Process
				)

				BeforeEach(func() {
					secondRunner = newNSyncRunner()
					secondRunner.StartCheck = ""

					secondProcess = ginkgomon.Invoke(secondRunner)
				})

				AfterEach(func() {
					ginkgomon.Interrupt(secondProcess, 2*time.Second)
				})

				Describe("the second listener", func() {
					It("does not become active", func() {
						Consistently(secondRunner.Buffer, 5*time.Second).ShouldNot(gbytes.Say("nsync.listener.started"))
					})
				})

				Context("and the first listener goes away", func() {
					BeforeEach(func() {
						ginkgomon.Interrupt(process, 2*time.Second)
					})

					Describe("the second listener", func() {
						It("eventually becomes active", func() {
							Eventually(secondRunner.Buffer, 5*time.Second).Should(gbytes.Say("nsync.listener.started"))
						})
					})
				})
			})
		})
	})

	Describe("when NATS is not up", func() {
		Context("and the nsync listener is started", func() {

			BeforeEach(func() {
				process = ifrit.Background(runner)
			})

			AfterEach(func() {
				defer stopNATS()
				defer ginkgomon.Interrupt(process, 2*time.Second)
			})

			It("starts only after nats comes up", func() {
				Consistently(process.Ready()).ShouldNot(BeClosed())

				startNATS()
				Eventually(process.Ready(), 5*time.Second).Should(BeClosed())
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary fetches all repos of a user or organization and clones
// them.  It is strongly recommended to get a personal API token from
// https://github.com/settings/tokens, save the token in a file, and
// point the --token option to it.
package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

func main() {
	dest := flag.String("dest", "", "destination directory")
	org := flag.String("org", "", "organization to mirror")
	user := flag.String("user", "", "user to mirror")
	token := flag.String("token", "", "file holding API token.")
	forks := flag.Bool("forks", false, "also mirror forks.")
	flag.Parse()

	if *dest == "" {
		log.Fatal("must set --dest")
	}
	if *org == "" && *user == "" {
		log.Fatal("must set --org or --user")
	}
	destDir := filepath.Join(*dest, "github.com")
	if err := os.MkdirAll(destDir, 0755); err != nil {
		log.Fatal(err)
	}

	client := github.NewClient(nil)
	if *token != "" {
		content, err := ioutil.ReadFile(*token)
		if err != nil {
			log.Fatal(err)
		}

		ts := oauth2.StaticTokenSource(
			&oauth2.Token{
				AccessToken: strings.TrimSpace(string(content)),
			})
		tc := oauth2.NewClient(oauth2.NoContext, ts)
		client = github.NewClient(tc)
	}

	var repos []github.Repository
	var err error
	if *org != "" {
		repos, err = getOrgRepos(client, *org)
	} else if *user != "" {
		repos, err = getUserRepos(client, *user)
	}
	if err != nil {
		log.Fatal(err)
	}

	if !*forks {
		trimmed := repos[:0]
		for _, r := range repos {
			if r.Fork == nil || !*r.Fork {
				trimmed = append(trimmed, r)
			}
		}
		repos = trimmed
	}

	if err := cloneRepos(destDir, repos); err != nil {
		log.Fatal(err)
	}
}

func getOrgRepos(client *github.Client, org string) ([]github.Repository, error) {
	var allRepos []github.Repository
	opt := &github.RepositoryListByOrgOptions{}
	for {
		repos, resp, err := client.Repositories.ListByOrg(org, opt)
		if err != nil {
			return nil, err
		}
		if len(repos) == 0 {
			break
		}
		var names []string
		for _, r := range repos {
			names = append(names, *r.Name)
		}
		log.Println(strings.Join(names, " "))

		opt.Page = resp.NextPage
		allRepos = append(allRepos, repos...)
		if resp.NextPage == 0 {
			break
		}
	}
	return allRepos, nil
}

func getUserRepos(client *github.Client, user string) ([]github.Repository, error) {
	var allRepos []github.Repository
	opt := &github.RepositoryListOptions{}
	for {
		repos, resp, err := client.Repositories.List(user, opt)
		if err != nil {
			return nil, err
		}
		if len(repos) == 0 {
			break
		}

		var names []string
		for _, r := range repos {
			names = append(names, *r.Name)
		}
		log.Println(strings.Join(names, " "))

		opt.Page = resp.NextPage
		allRepos = append(allRepos, repos...)
		if resp.NextPage == 0 {
			break
		}
	}
	return allRepos, nil
}

func cloneRepos(destDir string, repos []github.Repository) error {
	for _, r := range repos {
		parent := filepath.Join(destDir, filepath.Dir(*r.FullName))
		if err := os.MkdirAll(parent, 0755); err != nil {
			return err
		}

		base := *r.Name + ".git"
		if _, err := os.Lstat(filepath.Join(parent, base)); err == nil {
			continue
		}

		cmd := exec.Command("git", "clone", "--bare", *r.CloneURL, base)
		cmd.Dir = parent
		log.Println("running:", cmd.Args)
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>zoekt-mirror-github: use --mirror, and allow filtering by name.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary fetches all repos of a user or organization and clones
// them.  It is strongly recommended to get a personal API token from
// https://github.com/settings/tokens, save the token in a file, and
// point the --token option to it.
package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

func main() {
	dest := flag.String("dest", "", "destination directory")
	org := flag.String("org", "", "organization to mirror")
	user := flag.String("user", "", "user to mirror")
	token := flag.String("token", "", "file holding API token.")
	forks := flag.Bool("forks", false, "also mirror forks.")
	namePattern := flag.String("name", "", "only clone repos whose name contains the given substring.")
	flag.Parse()

	if *dest == "" {
		log.Fatal("must set --dest")
	}
	if *org == "" && *user == "" {
		log.Fatal("must set --org or --user")
	}
	destDir := filepath.Join(*dest, "github.com")
	if err := os.MkdirAll(destDir, 0755); err != nil {
		log.Fatal(err)
	}

	client := github.NewClient(nil)
	if *token != "" {
		content, err := ioutil.ReadFile(*token)
		if err != nil {
			log.Fatal(err)
		}

		ts := oauth2.StaticTokenSource(
			&oauth2.Token{
				AccessToken: strings.TrimSpace(string(content)),
			})
		tc := oauth2.NewClient(oauth2.NoContext, ts)
		client = github.NewClient(tc)
	}

	var repos []github.Repository
	var err error
	if *org != "" {
		repos, err = getOrgRepos(client, *org)
	} else if *user != "" {
		repos, err = getUserRepos(client, *user)
	}
	if err != nil {
		log.Fatal(err)
	}

	if !*forks {
		trimmed := repos[:0]
		for _, r := range repos {
			if r.Fork == nil || !*r.Fork {
				trimmed = append(trimmed, r)
			}
		}
		repos = trimmed
	}

	if *namePattern != "" {
		trimmed := repos[:0]
		for _, r := range repos {
			if r.Name != nil && strings.Contains(*r.Name, *namePattern) {
				trimmed = append(trimmed, r)
			}
		}
		repos = trimmed
	}

	if err := cloneRepos(destDir, repos); err != nil {
		log.Fatal(err)
	}
}

func getOrgRepos(client *github.Client, org string) ([]github.Repository, error) {
	var allRepos []github.Repository
	opt := &github.RepositoryListByOrgOptions{}
	for {
		repos, resp, err := client.Repositories.ListByOrg(org, opt)
		if err != nil {
			return nil, err
		}
		if len(repos) == 0 {
			break
		}
		var names []string
		for _, r := range repos {
			names = append(names, *r.Name)
		}
		log.Println(strings.Join(names, " "))

		opt.Page = resp.NextPage
		allRepos = append(allRepos, repos...)
		if resp.NextPage == 0 {
			break
		}
	}
	return allRepos, nil
}

func getUserRepos(client *github.Client, user string) ([]github.Repository, error) {
	var allRepos []github.Repository
	opt := &github.RepositoryListOptions{}
	for {
		repos, resp, err := client.Repositories.List(user, opt)
		if err != nil {
			return nil, err
		}
		if len(repos) == 0 {
			break
		}

		var names []string
		for _, r := range repos {
			names = append(names, *r.Name)
		}
		log.Println(strings.Join(names, " "))

		opt.Page = resp.NextPage
		allRepos = append(allRepos, repos...)
		if resp.NextPage == 0 {
			break
		}
	}
	return allRepos, nil
}

func cloneRepos(destDir string, repos []github.Repository) error {
	for _, r := range repos {
		parent := filepath.Join(destDir, filepath.Dir(*r.FullName))
		if err := os.MkdirAll(parent, 0755); err != nil {
			return err
		}

		base := *r.Name + ".git"
		if _, err := os.Lstat(filepath.Join(parent, base)); err == nil {
			continue
		}

		cmd := exec.Command("git", "clone", "--mirror", "--recursive", *r.CloneURL, base)
		cmd.Dir = parent
		log.Println("running:", cmd.Args)
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package main

import (
	"flag"
	"fmt"
	"golang.org/x/sys/unix"
	"io"
	"log"
	"os"
	"path/filepath"
	"syscall"
)

var (
	help    = flag.Bool("h", false, "Help")
	version = flag.Bool("V", false, "Version")
)

func usage() string {
	return "switch_root [-h] [-V]\nswitch_root newroot init"
}

// getDev returns the device (as returned by the FSTAT syscall) for the given file descriptor.
func getDev(fd int) (dev uint64, err error) {
	var stat unix.Stat_t

	if err := unix.Fstat(fd, &stat); err != nil {
		return 0, err
	}

	return stat.Dev, nil
}

// recursiveDelete deletes a directory identified by `fd` and everything in it.
//
// This function allows deleting directories no longer referenceable by
// any file name. This function does not descend into mounts.
func recursiveDelete(fd int) error {
	parentDev, err := getDev(fd)
	if err != nil {
		return err
	}

  // The file descriptor is already open, but allocating a os.File
	// here makes reading the files in the dir so much nicer.
	dir := os.NewFile(uintptr(fd), "__ignored__")
	defer dir.Close()
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}

	for _, name := range names {
		// Loop here, but handle loop in separate function to make defer work as expected.
		if err := recusiveDeleteInner(fd, parentDev, name); err != nil {
			return err
		}
	}
	return nil
}

// recusiveDeleteInner is called from recursiveDelete and either deletes
// or recurses into the given file or directory
//
// There should be no need to call this function directly.
func recusiveDeleteInner(parentFd int, parentDev uint64, childName string) error {
	// O_DIRECTORY and O_NOFOLLOW make this open fail for all files and all symlinks (even when pointing to a dir).
	// We need to filter out symlinks because getDev later follows them.
	childFd, err := unix.Openat(parentFd, childName, unix.O_DIRECTORY | unix.O_NOFOLLOW, unix.O_RDWR)
	if err != nil {
		// childName points to either a file or a symlink, delete in any case.
		if err := unix.Unlinkat(parentFd, childName, 0); err != nil {
			return err
		}
	} else {
		// Open succeeded, which means childName points to a real directory.
		defer unix.Close(childFd)

		// Don't descent into other file systems.
		if childFdDev, err := getDev(childFd); err != nil {
			return err
		} else if childFdDev != parentDev {
			// This means continue in recursiveDelete.
			return nil
		}

		if err:= recursiveDelete(childFd); err != nil {
			return err
		}
		// Back from recursion, the directory is now empty, delete.
		if err := unix.Unlinkat(parentFd, childName, unix.AT_REMOVEDIR); err != nil {
			return err
		}
	}
	return nil
}

// execCommand execs into the given command.
//
// In order to preserve whatever PID this program is running with,
// the implementation does an actual EXEC syscall without forking.
func execCommand(path string) error {
	return syscall.Exec(path, []string{path}, []string{})
}

// isEmpty returns true if the directory with the given path is empty.
func isEmpty(name string) (bool, error) {
	f, err := os.Open(name)
	if err != nil {
		return false, err
	}
	defer f.Close()

	_, err = f.Readdirnames(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}

// moveMount moves mount
//
// This function is just a wrapper around the MOUNT syscall with the
// MOVE flag supplied.
func moveMount(oldPath string, newPath string) error {
	return unix.Mount(oldPath, newPath, "", unix.MS_MOVE, "")
}

// specialFS moves the 'special' mounts to the given target path
//
// 'special' in this context refers to the following non-blockdevice backed
// mounts that are almost always used: /dev, /proc, /sys, and /run.
// This function will create the target directories, if necessary.
// If the target directories already exists, they must be empty.
// This function skips missing mounts.
func specialFS(newRoot string) error {
	var mounts = []string{"/dev", "/proc", "/sys", "/run"}

	for _, mount := range mounts {
		path := filepath.Join(newRoot, mount)
		// Skip all mounting if the directory does not exists.
		if _, err := os.Stat(mount); os.IsNotExist(err) {
			fmt.Println("switch_root: Skipping", mount)
			continue
		} else if err != nil {
			return err
		}
		// Make sure the target dir exists and is empty.
		if _, err := os.Stat(path); os.IsNotExist(err) {
			if err := unix.Mkdir(path, 0); err != nil {
				return err
			}
		} else if err != nil {
			return err
		}
		if empty, err := isEmpty(path); err != nil {
			return err
		} else if !empty {
			return fmt.Errorf("%v must be empty", path)
		}
		if err := moveMount(mount, path); err != nil {
			return err
		}
	}
	return nil
}

// switchroot moves special mounts (dev, proc, sys, run) to the new directory,
// then does a chroot, moves the root mount to the new directory and finally
// DELETES EVERYTHING in the old root and execs the given init.
func switchRoot(newRoot string, init string) error {
	log.Printf("switch_root: moving mounts")
	if err := specialFS(newRoot); err != nil {
		return fmt.Errorf("switch_root: moving mounts failed %v", err)
	}

	log.Printf("switch_root: Changing directory")
	if err := unix.Chdir(newRoot); err != nil {
		return fmt.Errorf("switch_root: failed change directory to new_root %v", err)
	}

	// Open "/" now, we need the file descriptor later.
	oldRoot, err := os.Open("/")
	if err != nil {
		return err
	}
	defer oldRoot.Close()

	log.Printf("switch_root: Moving /")
	if err := moveMount(newRoot, "/"); err != nil {
		return err
	}

	log.Printf("switch_root: Changing root!")
	if err := unix.Chroot("."); err != nil {
		return fmt.Errorf("switch_root: fatal chroot error %v", err)
	}

	log.Printf("switch_root: Deleting old /")
	if err := recursiveDelete(int(oldRoot.Fd())); err != nil {
		panic(err)
	}

	log.Printf("switch_root: executing init")
	if err := execCommand(init); err != nil {
		return fmt.Errorf("switch_root: exec failed %v", err)
	}

	return nil
}

func main() {
	flag.Parse()

	if len(flag.Args()) == 0 {
		fmt.Println(usage())
		os.Exit(0)
	}

	if *help {
		fmt.Println(usage())
		os.Exit(0)
	}

	if *version {
		fmt.Println("Version XX")
		os.Exit(0)
	}

	newRoot := flag.Args()[0]
	init := flag.Args()[1]

	if err := switchRoot(newRoot, init); err != nil {
		log.Fatalf("switch_root failed %v\n", err)
	}
}
<commit_msg>cmds/switch_root: fix nit<commit_after>// Copyright 2012-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package main

import (
	"flag"
	"fmt"
	"golang.org/x/sys/unix"
	"io"
	"log"
	"os"
	"path/filepath"
	"syscall"
)

var (
	help    = flag.Bool("h", false, "Help")
	version = flag.Bool("V", false, "Version")
)

func usage() string {
	return "switch_root [-h] [-V]\nswitch_root newroot init"
}

// getDev returns the device (as returned by the FSTAT syscall) for the given file descriptor.
func getDev(fd int) (dev uint64, err error) {
	var stat unix.Stat_t

	if err := unix.Fstat(fd, &stat); err != nil {
		return 0, err
	}

	return stat.Dev, nil
}

// recursiveDelete deletes a directory identified by `fd` and everything in it.
//
// This function allows deleting directories no longer referenceable by
// any file name. This function does not descend into mounts.
func recursiveDelete(fd int) error {
	parentDev, err := getDev(fd)
	if err != nil {
		return err
	}

  // The file descriptor is already open, but allocating a os.File
	// here makes reading the files in the dir so much nicer.
	dir := os.NewFile(uintptr(fd), "__ignored__")
	defer dir.Close()
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}

	for _, name := range names {
		// Loop here, but handle loop in separate function to make defer work as expected.
		if err := recusiveDeleteInner(fd, parentDev, name); err != nil {
			return err
		}
	}
	return nil
}

// recusiveDeleteInner is called from recursiveDelete and either deletes
// or recurses into the given file or directory
//
// There should be no need to call this function directly.
func recusiveDeleteInner(parentFd int, parentDev uint64, childName string) error {
	// O_DIRECTORY and O_NOFOLLOW make this open fail for all files and all symlinks (even when pointing to a dir).
	// We need to filter out symlinks because getDev later follows them.
	childFd, err := unix.Openat(parentFd, childName, unix.O_DIRECTORY | unix.O_NOFOLLOW, unix.O_RDWR)
	if err != nil {
		// childName points to either a file or a symlink, delete in any case.
		if err := unix.Unlinkat(parentFd, childName, 0); err != nil {
			return err
		}
	} else {
		// Open succeeded, which means childName points to a real directory.
		defer unix.Close(childFd)

		// Don't descent into other file systems.
		if childFdDev, err := getDev(childFd); err != nil {
			return err
		} else if childFdDev != parentDev {
			// This means continue in recursiveDelete.
			return nil
		}

		if err:= recursiveDelete(childFd); err != nil {
			return err
		}
		// Back from recursion, the directory is now empty, delete.
		if err := unix.Unlinkat(parentFd, childName, unix.AT_REMOVEDIR); err != nil {
			return err
		}
	}
	return nil
}

// execCommand execs into the given command.
//
// In order to preserve whatever PID this program is running with,
// the implementation does an actual EXEC syscall without forking.
func execCommand(path string) error {
	return syscall.Exec(path, []string{path}, []string{})
}

// isEmpty returns true if the directory with the given path is empty.
func isEmpty(name string) (bool, error) {
	f, err := os.Open(name)
	if err != nil {
		return false, err
	}
	defer f.Close()

	if _, err := f.Readdirnames(1); err == io.EOF {
		return true, nil
	}
	return false, err
}

// moveMount moves mount
//
// This function is just a wrapper around the MOUNT syscall with the
// MOVE flag supplied.
func moveMount(oldPath string, newPath string) error {
	return unix.Mount(oldPath, newPath, "", unix.MS_MOVE, "")
}

// specialFS moves the 'special' mounts to the given target path
//
// 'special' in this context refers to the following non-blockdevice backed
// mounts that are almost always used: /dev, /proc, /sys, and /run.
// This function will create the target directories, if necessary.
// If the target directories already exists, they must be empty.
// This function skips missing mounts.
func specialFS(newRoot string) error {
	var mounts = []string{"/dev", "/proc", "/sys", "/run"}

	for _, mount := range mounts {
		path := filepath.Join(newRoot, mount)
		// Skip all mounting if the directory does not exists.
		if _, err := os.Stat(mount); os.IsNotExist(err) {
			fmt.Println("switch_root: Skipping", mount)
			continue
		} else if err != nil {
			return err
		}
		// Make sure the target dir exists and is empty.
		if _, err := os.Stat(path); os.IsNotExist(err) {
			if err := unix.Mkdir(path, 0); err != nil {
				return err
			}
		} else if err != nil {
			return err
		}
		if empty, err := isEmpty(path); err != nil {
			return err
		} else if !empty {
			return fmt.Errorf("%v must be empty", path)
		}
		if err := moveMount(mount, path); err != nil {
			return err
		}
	}
	return nil
}

// switchroot moves special mounts (dev, proc, sys, run) to the new directory,
// then does a chroot, moves the root mount to the new directory and finally
// DELETES EVERYTHING in the old root and execs the given init.
func switchRoot(newRoot string, init string) error {
	log.Printf("switch_root: moving mounts")
	if err := specialFS(newRoot); err != nil {
		return fmt.Errorf("switch_root: moving mounts failed %v", err)
	}

	log.Printf("switch_root: Changing directory")
	if err := unix.Chdir(newRoot); err != nil {
		return fmt.Errorf("switch_root: failed change directory to new_root %v", err)
	}

	// Open "/" now, we need the file descriptor later.
	oldRoot, err := os.Open("/")
	if err != nil {
		return err
	}
	defer oldRoot.Close()

	log.Printf("switch_root: Moving /")
	if err := moveMount(newRoot, "/"); err != nil {
		return err
	}

	log.Printf("switch_root: Changing root!")
	if err := unix.Chroot("."); err != nil {
		return fmt.Errorf("switch_root: fatal chroot error %v", err)
	}

	log.Printf("switch_root: Deleting old /")
	if err := recursiveDelete(int(oldRoot.Fd())); err != nil {
		panic(err)
	}

	log.Printf("switch_root: executing init")
	if err := execCommand(init); err != nil {
		return fmt.Errorf("switch_root: exec failed %v", err)
	}

	return nil
}

func main() {
	flag.Parse()

	if len(flag.Args()) == 0 {
		fmt.Println(usage())
		os.Exit(0)
	}

	if *help {
		fmt.Println(usage())
		os.Exit(0)
	}

	if *version {
		fmt.Println("Version XX")
		os.Exit(0)
	}

	newRoot := flag.Args()[0]
	init := flag.Args()[1]

	if err := switchRoot(newRoot, init); err != nil {
		log.Fatalf("switch_root failed %v\n", err)
	}
}
<|endoftext|>


<commit_before>package ics

import (
	"bytes"
	"io"
)

var (
	lineWrap = []byte{'\r', '\n', ' '}
	lineEnd  = []byte{'\r', '\n'}
)

type unfolder struct {
	buf bytes.Buffer
}

func NewUnfolder(r io.Reader) (*unfolder, error) {
	var u unfolder
	_, err := u.buf.ReadFrom(r)
	if err != nil {
		return nil, err
	}
	return &u, nil
}

func (u *unfolder) ReadLine() ([]byte, error) {
	var toRet []byte
	for {
		p, err := u.buf.ReadBytes('\r')
		if err != nil {
			return nil, err
		}
		c, err := u.buf.ReadByte()
		if err != nil {
			return nil, err
		}
		if c != '\n' {
			u.buf.UnreadByte()
			toRet = append(toRet, p...)
			continue
		}
		toRet = append(toRet, p[:len(p)-1]...)
		c, _ := u.buf.ReadByte()
		if c != ' ' {
			u.buf.UnreadByte()
			break
		}
	}
	return toRet, nil
}
<commit_msg>Made NewUnfolder unexported<commit_after>package ics

import (
	"bytes"
	"io"
)

var (
	lineWrap = []byte{'\r', '\n', ' '}
	lineEnd  = []byte{'\r', '\n'}
)

type unfolder struct {
	buf bytes.Buffer
}

func newUnfolder(r io.Reader) (*unfolder, error) {
	var u unfolder
	_, err := u.buf.ReadFrom(r)
	if err != nil {
		return nil, err
	}
	return &u, nil
}

func (u *unfolder) ReadLine() ([]byte, error) {
	var toRet []byte
	for {
		p, err := u.buf.ReadBytes('\r')
		if err != nil {
			return nil, err
		}
		c, err := u.buf.ReadByte()
		if err != nil {
			return nil, err
		}
		if c != '\n' {
			u.buf.UnreadByte()
			toRet = append(toRet, p...)
			continue
		}
		toRet = append(toRet, p[:len(p)-1]...)
		c, _ := u.buf.ReadByte()
		if c != ' ' {
			u.buf.UnreadByte()
			break
		}
	}
	return toRet, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/github/git-lfs/lfs"
	"github.com/technoweenie/go-contentaddressable"
	"io"
	"os"
	"path/filepath"
)

func upload(req *lfs.JsonRequest, in io.Reader, out io.Writer, config *Config, path string) *lfs.JsonResponse {
	upreq := lfs.UploadRequest{}
	err := lfs.ExtractStructFromJsonRawMessage(req.Params, &upreq)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	// Build destination path
	filename, err := mediaPath(upreq.Oid, config)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Error determining media path. %v", err))
	}
	// contentaddressable fails hard when file already exists, let's be nicer
	startresult := lfs.UploadResponse{}
	_, staterr := os.Stat(filename)
	var mediaFile *contentaddressable.File
	if staterr != nil && os.IsNotExist(staterr) {
		startresult.OkToSend = true
		// Create file to write to - do this early to allow for fail states
		mediaFile, err = contentaddressable.NewFile(filename)
		if err != nil {
			return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Error opening media file buffer. %v", err))
		}
	}
	// Send start response immediately
	resp, err := lfs.NewJsonResponse(req.Id, startresult)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	err = sendResponse(resp, out)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	if !startresult.OkToSend {
		return nil
	}

	// Next from client should be byte stream of exactly the stated number of bytes
	n, err := io.CopyN(mediaFile, in, upreq.Size)
	if err == nil {
		err = mediaFile.Accept()
	}
	mediaFile.Close()
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Problem uploading data: %v", err.Error()))
	} else if n != upreq.Size {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Received wrong number of bytes %d (expected %d)", n, upreq.Size))
	}

	receivedresult := lfs.UploadCompleteResponse{}
	receivedresult.ReceivedOk = true
	resp, _ = lfs.NewJsonResponse(req.Id, receivedresult)

	return resp

}

func uploadCheck(req *lfs.JsonRequest, in io.Reader, out io.Writer, config *Config, path string) *lfs.JsonResponse {
	upreq := lfs.UploadRequest{}
	err := lfs.ExtractStructFromJsonRawMessage(req.Params, &upreq)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	// Build destination path
	filename, err := mediaPath(upreq.Oid, config)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Error determining media path. %v", err))
	}
	// contentaddressable fails hard when file already exists, let's be nicer
	startresult := lfs.UploadResponse{}
	_, staterr := os.Stat(filename)
	if staterr != nil && os.IsNotExist(staterr) {
		startresult.OkToSend = true
	}
	// Send start response immediately
	resp, err := lfs.NewJsonResponse(req.Id, startresult)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	return resp

}

func downloadCheck(req *lfs.JsonRequest, in io.Reader, out io.Writer, config *Config, path string) *lfs.JsonResponse {
	downreq := lfs.DownloadCheckRequest{}
	err := lfs.ExtractStructFromJsonRawMessage(req.Params, &downreq)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	filename, err := mediaPath(downreq.Oid, config)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Problem determining media path: %v", err))
	}
	result := lfs.DownloadCheckResponse{}
	s, err := os.Stat(filename)
	if err == nil {
		// file exists
		result.Size = s.Size()
	} else {
		result.Size = -1
	}
	resp, err := lfs.NewJsonResponse(req.Id, result)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	return resp
}
func download(req *lfs.JsonRequest, in io.Reader, out io.Writer, config *Config, path string) *lfs.JsonResponse {
	downreq := lfs.DownloadRequest{}
	err := lfs.ExtractStructFromJsonRawMessage(req.Params, &downreq)
	if err != nil {
		// Serve() copes with converting this to stderr rather than JSON response
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	filename, err := mediaPath(downreq.Oid, config)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Problem determining the media path: %v", err))
	}
	// check size
	s, err := os.Stat(filename)
	if err != nil {
		// file doesn't exist, this should not have been called
		return lfs.NewJsonErrorResponse(req.Id, "File doesn't exist")
	}
	if s.Size() != downreq.Size {
		// This won't work!
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("File sizes disagree (client: %d server: %d)", downreq.Size, s.Size()))
	}

	f, err := os.OpenFile(filename, os.O_RDONLY, 0644)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	defer f.Close()

	n, err := io.Copy(out, f)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Error copying data to output: %v", err.Error()))
	}
	if n != s.Size() {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Amount of data copied disagrees (expected: %d actual: %d)", s.Size(), n))
	}
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Error copying data to output: %v", err.Error()))
	}
	if n != s.Size() {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Amount of data copied disagrees (expected: %d actual: %d)", s.Size(), n))
	}

	// Don't return a response, only response is byte stream above except in error cases
	return nil
}

func batch(req *lfs.JsonRequest, in io.Reader, out io.Writer, config *Config, path string) *lfs.JsonResponse {
	batchreq := lfs.BatchRequest{}
	err := lfs.ExtractStructFromJsonRawMessage(req.Params, &batchreq)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	result := lfs.BatchResponse{}
	for _, o := range batchreq.Objects {
		filename, err := mediaPath(o.Oid, config)
		if err != nil {
			return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Problem determining the media path: %v", err))
		}
		resultObj := lfs.BatchResponseObject{Oid: o.Oid}
		s, err := os.Stat(filename)
		if err == nil {
			// file exists
			resultObj.Action = "download"
			resultObj.Size = s.Size()
		} else {
			resultObj.Action = "upload"
			resultObj.Size = o.Size
		}
		result.Results = append(result.Results, resultObj)
	}

	resp, err := lfs.NewJsonResponse(req.Id, result)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	return resp

}

// Store in the same structure as client, just under BasePath
func mediaPath(sha string, config *Config) (string, error) {
	path := filepath.Join(config.BasePath, sha[0:2], sha[2:4])
	if err := os.MkdirAll(path, 0744); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}
	return filepath.Join(path, sha), nil
}
<commit_msg>Remove dependency on contentaddressable<commit_after>package main

import (
	"fmt"
	"github.com/github/git-lfs/lfs"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

func upload(req *lfs.JsonRequest, in io.Reader, out io.Writer, config *Config, path string) *lfs.JsonResponse {
	upreq := lfs.UploadRequest{}
	err := lfs.ExtractStructFromJsonRawMessage(req.Params, &upreq)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	// Build destination path
	filename, err := mediaPath(upreq.Oid, config)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Error determining media path. %v", err))
	}
	startresult := lfs.UploadResponse{}
	_, staterr := os.Stat(filename)
	if staterr != nil && os.IsNotExist(staterr) {
		startresult.OkToSend = true
	}
	// Send start response immediately
	resp, err := lfs.NewJsonResponse(req.Id, startresult)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	err = sendResponse(resp, out)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	if !startresult.OkToSend {
		return nil
	}

	// Next from client should be byte stream of exactly the stated number of bytes
	// Now open temp file to write to
	tempf, err := ioutil.TempFile("", "tempupload")
	defer os.Remove(tempf.Name())
	defer tempf.Close()
	n, err := io.CopyN(tempf, in, upreq.Size)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Unable to read data: %v", err.Error()))
	} else if n != upreq.Size {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Received wrong number of bytes %d (expected %d)", n, upreq.Size))
	}

	receivedresult := lfs.UploadCompleteResponse{}
	receivedresult.ReceivedOk = true
	var receiveerr string
	// force close now before defer so we can copy
	err = tempf.Close()
	if err != nil {
		receivedresult.ReceivedOk = false
		receiveerr = fmt.Sprintf("Error when closing temp file: %v", err.Error())
	} else {
		// ensure final directory exists
		ensureDirExists(filepath.Dir(filename), config)
		// Move temp file to final location
		err = os.Rename(tempf.Name(), filename)
		if err != nil {
			receivedresult.ReceivedOk = false
			receiveerr = fmt.Sprintf("Error when closing temp file: %v", err.Error())
		}

	}

	resp, _ = lfs.NewJsonResponse(req.Id, receivedresult)
	if receiveerr != "" {
		resp.Error = receiveerr
	}

	return resp

}

func ensureDirExists(dir string, cfg *Config) error {
	s, err := os.Stat(dir)
	if err == nil {
		if !s.IsDir() {
			return fmt.Errorf("%v exists but isn't a dir", dir)
		}
	} else {
		// Get permissions from base path & match (or default to user/group write)
		mode := os.FileMode(0775)
		s, err := os.Stat(cfg.BasePath)
		if err == nil {
			mode = s.Mode()
		}
		return os.MkdirAll(dir, mode)
	}
	return nil
}

func uploadCheck(req *lfs.JsonRequest, in io.Reader, out io.Writer, config *Config, path string) *lfs.JsonResponse {
	upreq := lfs.UploadRequest{}
	err := lfs.ExtractStructFromJsonRawMessage(req.Params, &upreq)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	// Build destination path
	filename, err := mediaPath(upreq.Oid, config)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Error determining media path. %v", err))
	}
	startresult := lfs.UploadResponse{}
	_, staterr := os.Stat(filename)
	if staterr != nil && os.IsNotExist(staterr) {
		startresult.OkToSend = true
	}
	// Send start response immediately
	resp, err := lfs.NewJsonResponse(req.Id, startresult)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	return resp

}

func downloadCheck(req *lfs.JsonRequest, in io.Reader, out io.Writer, config *Config, path string) *lfs.JsonResponse {
	downreq := lfs.DownloadCheckRequest{}
	err := lfs.ExtractStructFromJsonRawMessage(req.Params, &downreq)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	filename, err := mediaPath(downreq.Oid, config)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Problem determining media path: %v", err))
	}
	result := lfs.DownloadCheckResponse{}
	s, err := os.Stat(filename)
	if err == nil {
		// file exists
		result.Size = s.Size()
	} else {
		result.Size = -1
	}
	resp, err := lfs.NewJsonResponse(req.Id, result)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	return resp
}
func download(req *lfs.JsonRequest, in io.Reader, out io.Writer, config *Config, path string) *lfs.JsonResponse {
	downreq := lfs.DownloadRequest{}
	err := lfs.ExtractStructFromJsonRawMessage(req.Params, &downreq)
	if err != nil {
		// Serve() copes with converting this to stderr rather than JSON response
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	filename, err := mediaPath(downreq.Oid, config)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Problem determining the media path: %v", err))
	}
	// check size
	s, err := os.Stat(filename)
	if err != nil {
		// file doesn't exist, this should not have been called
		return lfs.NewJsonErrorResponse(req.Id, "File doesn't exist")
	}
	if s.Size() != downreq.Size {
		// This won't work!
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("File sizes disagree (client: %d server: %d)", downreq.Size, s.Size()))
	}

	f, err := os.OpenFile(filename, os.O_RDONLY, 0644)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	defer f.Close()

	n, err := io.Copy(out, f)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Error copying data to output: %v", err.Error()))
	}
	if n != s.Size() {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Amount of data copied disagrees (expected: %d actual: %d)", s.Size(), n))
	}
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Error copying data to output: %v", err.Error()))
	}
	if n != s.Size() {
		return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Amount of data copied disagrees (expected: %d actual: %d)", s.Size(), n))
	}

	// Don't return a response, only response is byte stream above except in error cases
	return nil
}

func batch(req *lfs.JsonRequest, in io.Reader, out io.Writer, config *Config, path string) *lfs.JsonResponse {
	batchreq := lfs.BatchRequest{}
	err := lfs.ExtractStructFromJsonRawMessage(req.Params, &batchreq)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	result := lfs.BatchResponse{}
	for _, o := range batchreq.Objects {
		filename, err := mediaPath(o.Oid, config)
		if err != nil {
			return lfs.NewJsonErrorResponse(req.Id, fmt.Sprintf("Problem determining the media path: %v", err))
		}
		resultObj := lfs.BatchResponseObject{Oid: o.Oid}
		s, err := os.Stat(filename)
		if err == nil {
			// file exists
			resultObj.Action = "download"
			resultObj.Size = s.Size()
		} else {
			resultObj.Action = "upload"
			resultObj.Size = o.Size
		}
		result.Results = append(result.Results, resultObj)
	}

	resp, err := lfs.NewJsonResponse(req.Id, result)
	if err != nil {
		return lfs.NewJsonErrorResponse(req.Id, err.Error())
	}
	return resp

}

// Store in the same structure as client, just under BasePath
func mediaPath(sha string, config *Config) (string, error) {
	path := filepath.Join(config.BasePath, sha[0:2], sha[2:4])
	if err := os.MkdirAll(path, 0744); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}
	return filepath.Join(path, sha), nil
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import "time"

type metricType int8

const (
	Counter metricType = 0
	Gauge metricType = 1
)

type Metric struct {
	Name string
	Type metricType
	Value float64
}

type Metrics struct {
	sourceName string
	quitChannel chan bool
	relayFunc func(string, []Metric)
	relayPeriodInSecs int
	metricChannel chan *Metric
	ticker *time.Ticker
}

func NewMetrics(	sourceName string,
					relayFunc func(string, []Metric),
					relayPeriodInSecs int,
					metricQueueLength int) *Metrics {

	return &Metrics{
		sourceName : sourceName,
		relayFunc : relayFunc,
		relayPeriodInSecs: relayPeriodInSecs,
		quitChannel : make(chan bool),
		metricChannel : make(chan *Metric, metricQueueLength),
	}
}

// Update the gauge value
func (self *Metrics) Gauge(metricName string, value float64) {
	self.metricChannel <- &Metric{
		Name: metricName,
		Type: Gauge,
		Value: value,
	}
}

// Increases the counter by one.
func (self *Metrics) Count(metricName string) {
	self.metricChannel <- &Metric{
		Name: metricName,
		Type: Counter,
		Value: 1,
	}
}

// Increase the counter
func (self *Metrics) CountWithValue(metricName string, value float64) {
	self.metricChannel <- &Metric{
		Name: metricName,
		Type: Counter,
		Value: value,
	}
}

func (self *Metrics) listenForEvents() {

	metrics := make(map[string]*Metric)

    for {
        select {
			case metric := <- self.metricChannel:
				current, found := metrics[metric.Name]
				if !found {
					metrics[metric.Name] = metric
					continue
				}

				if metric.Type == Counter {
					current.Value = current.Value + metric.Value
				} else {
					// This is a gague
					current.Value = metric.Value
				}

			case <- self.ticker.C:
				toRelay := make([]Metric, len(metrics))
				for _, v := range metrics {
					toRelay = append(toRelay, Metric{ Name: v.Name, Type: v.Type, Value: v.Value })
				}

				go self.relayFunc(self.sourceName, toRelay)

			case <- self.quitChannel:
				self.ticker.Stop()
				return
        }
    }
}

func (self *Metrics) Start() error {

	self.ticker = time.NewTicker(time.Duration(self.relayPeriodInSecs) * time.Second)

	go self.listenForEvents()

	return nil
}

func (self *Metrics) Stop() error {
	self.quitChannel <- true
	return nil
}

<commit_msg>added support for a default logging relay function.<commit_after>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import "time"

type metricType int8

const (
	Counter metricType = 0
	Gauge metricType = 1
)

type Metric struct {
	Name string
	Type metricType
	Value float64
}

type Metrics struct {
	sourceName string
	quitChannel chan bool
	relayFunc func(string, []Metric)
	relayPeriodInSecs int
	metricChannel chan *Metric
	ticker *time.Ticker
}

func NewMetrics(	sourceName string,
					relayFunc func(string, []Metric),
					relayPeriodInSecs int,
					metricQueueLength int) *Metrics {

	return &Metrics{
		sourceName : sourceName,
		relayFunc : relayFunc,
		relayPeriodInSecs: relayPeriodInSecs,
		quitChannel : make(chan bool),
		metricChannel : make(chan *Metric, metricQueueLength),
	}
}

// Update the gauge value
func (self *Metrics) Gauge(metricName string, value float64) {
	self.metricChannel <- &Metric{
		Name: metricName,
		Type: Gauge,
		Value: value,
	}
}

// Increases the counter by one.
func (self *Metrics) Count(metricName string) {
	self.metricChannel <- &Metric{
		Name: metricName,
		Type: Counter,
		Value: 1,
	}
}

// Increase the counter
func (self *Metrics) CountWithValue(metricName string, value float64) {
	self.metricChannel <- &Metric{
		Name: metricName,
		Type: Counter,
		Value: value,
	}
}

func (self *Metrics) listenForEvents() {

	metrics := make(map[string]*Metric)

    for {
        select {
			case metric := <- self.metricChannel:
				current, found := metrics[metric.Name]
				if !found {
					metrics[metric.Name] = metric
					continue
				}

				if metric.Type == Counter {
					current.Value = current.Value + metric.Value
				} else {
					// This is a gague
					current.Value = metric.Value
				}

			case <- self.ticker.C:
				toRelay := make([]Metric, len(metrics))
				for _, v := range metrics {
					toRelay = append(toRelay, Metric{ Name: v.Name, Type: v.Type, Value: v.Value })
				}

				go self.relayFunc(self.sourceName, toRelay)

			case <- self.quitChannel:
				self.ticker.Stop()
				return
        }
    }
}

func (self *Metrics) Start() error {

	self.ticker = time.NewTicker(time.Duration(self.relayPeriodInSecs) * time.Second)

	go self.listenForEvents()

	return nil
}

func (self *Metrics) Stop() error {
	self.quitChannel <- true
	return nil
}

// Pass the logger struct to the log metrics.
func NewLogMetrics(logger Logger) *LogMetrics {
	return &LogMetrics{ logger }
}

// This can be used to print metrics out to a configured logger.
type LogMetrics struct {
	Logger
}

// This logs an info message with the following format: [source: %s - metric: %s - value: %f]
func (self *LogMetrics) Log(sourceName string, metrics []Metric) {
	if len(metrics) == 0 {
		return
	}

	for i := range metrics {
		self.Logf(Info, "[source: %s - metric: %s - value: %f]", sourceName, metrics[i].Name, metrics[i].Value)
	}
}

<|endoftext|>