<commit_before>package rules

import (
	"github.com/golang/protobuf/v2/reflect/protoreflect"
	"github.com/jgeewax/api-linter/lint"
	"github.com/jgeewax/api-linter/proto"
)

// Linter defines an operation that checks a lint Request with a Rule,
// and returns a Response and if applicable, an error.
type Linter interface {
	Lint(lint.Request, lint.Rule) (lint.Response, error)
}

type protoLinter struct {
	checker  DescriptorChecker
	info     RuleInfo
	problems []lint.Problem
	source   lint.DescriptorSource
}

func (l *protoLinter) addProblems(p ...lint.Problem) {
	l.problems = append(l.problems, p...)
}

func (l *protoLinter) Consume(d protoreflect.Descriptor) error {
	if l.source.IsRuleDisabled(l.info.Name, d) {
		return nil
	}

	problems, err := l.checker.Check(d)
	if err != nil {
		return err
	}
	l.addProblems(problems...)

	return nil
}

func (l *protoLinter) Lint(req lint.Request, rule lint.Rule) (lint.Response, error) {
	f := req.ProtoFile()
	if err := proto.Walk(f, l); err != nil {
		return lint.Response{}, nil
	}
	return lint.Response{
		Problems: l.problems,
	}, nil
}

func newProtoLinter(info RuleInfo, checker DescriptorChecker) Linter {
	return &protoLinter{
		checker: checker,
		info:    info,
	}
}
<commit_msg>Added the missing source.<commit_after>package rules

import (
	"github.com/golang/protobuf/v2/reflect/protoreflect"
	"github.com/jgeewax/api-linter/lint"
	"github.com/jgeewax/api-linter/proto"
)

// Linter defines an operation that checks a lint Request with a Rule,
// and returns a Response and if applicable, an error.
type Linter interface {
	Lint(lint.Request, lint.Rule) (lint.Response, error)
}

type protoLinter struct {
	checker  DescriptorChecker
	info     RuleInfo
	problems []lint.Problem
	source   lint.DescriptorSource
}

func (l *protoLinter) addProblems(p ...lint.Problem) {
	l.problems = append(l.problems, p...)
}

func (l *protoLinter) ConsumeDescriptor(d protoreflect.Descriptor) error {
	if l.source.IsRuleDisabled(l.info.Name, d) {
		return nil
	}

	problems, err := l.checker.Check(d)
	if err != nil {
		return err
	}
	l.addProblems(problems...)

	return nil
}

func (l *protoLinter) Lint(req lint.Request, rule lint.Rule) (lint.Response, error) {
	f := req.ProtoFile()
	l.source = req.DescriptorSource()
	if err := proto.WalkDescriptor(f, l); err != nil {
		return lint.Response{}, nil
	}
	return lint.Response{
		Problems: l.problems,
	}, nil
}

func newProtoLinter(info RuleInfo, checker DescriptorChecker) Linter {
	return &protoLinter{
		checker: checker,
		info:    info,
	}
}
<|endoftext|>


<commit_before>package listener

import (
	"encoding/json"
	"sort"
	"sync"

	"github.com/jeffpierce/cassabon/config"
	"github.com/jeffpierce/cassabon/pearson"
)

type indexedLine struct {
	peerIndex int
	statLine  string
}

// PeerList contains an ordered list of Cassabon peers.
type PeerList struct {
	wg       *sync.WaitGroup
	target   chan indexedLine  // Channel for forwarding a stat line to a Cassabon peer
	hostPort string            // Host:port on which the local server is listening
	peersMap map[string]string // Peer list as stored in the configuration
	peers    []string          // Host:port information for all Cassabon peers (inclusive)
	conns    map[string]*StubbornTCPConn
	self     sync.RWMutex
}

func (pl *PeerList) Init() {

	pl.conns = make(map[string]*StubbornTCPConn, 0)

	// Create the channel on which stats to forward are received.
	pl.target = make(chan indexedLine, 1)
}

// IsStarted indicates whether the structure has ever been updated.
func (pl *PeerList) IsStarted() bool {
	return pl.hostPort != ""
}

// Start records the current peer list and starts the forwarder goroutine.
func (pl *PeerList) Start(wg *sync.WaitGroup, hostPort string, peersMap map[string]string) {

	// Synchronize access by other goroutines.
	pl.self.Lock()
	defer pl.self.Unlock()

	pl.wg = wg
	pl.hostPort = hostPort
	pl.peersMap = peersMap

	// Dispose of any peer connections that are obsolete.
	peers := sortedMapToArray(pl.peersMap)
	for _, existing := range pl.peers {
		found := false
		for _, incoming := range peers {
			if existing == incoming {
				found = true
				break
			}
		}
		if !found && existing != pl.hostPort {
			pl.conns[existing].Close()
			delete(pl.conns, existing)
		} else {
			config.G.Log.System.LogInfo("Keeping peer connection to %s", existing)
		}
	}

	// Record the current set of peers.
	pl.peers = make([]string, len(peers))
	for i, v := range peers {
		pl.peers[i] = v
		// Set up peer connections for newly added peers.
		if _, found := pl.conns[v]; !found && v != pl.hostPort {
			pl.conns[v] = new(StubbornTCPConn)
			pl.conns[v].Open(v)
		}
	}

	// Start the forwarder goroutine.
	pl.wg.Add(1)
	go pl.run()
}

// IsEqual indicates whether the given new configuration is equal to the current.
func (pl *PeerList) IsEqual(hostPort string, peersMap map[string]string) bool {

	// Synchronize access by other goroutines.
	pl.self.RLock()
	defer pl.self.RUnlock()

	if pl.hostPort != hostPort {
		return false
	}

	peers := sortedMapToArray(peersMap)
	if len(pl.peers) != len(peers) {
		return false
	}
	for i, v := range pl.peers {
		if peers[i] != v {
			return false
		}
	}

	return true
}

// OwnerOf determines which host owns a particular stats path.
func (pl *PeerList) OwnerOf(statPath string) (int, bool) {
	peerIndex := int(pearson.Hash8(statPath)) % len(pl.peers)
	if pl.hostPort == pl.peers[peerIndex] {
		return peerIndex, true
	} else {
		return peerIndex, false
	}
}

// PropagatePeerList sends the current peer list to all known peers.
func (pl *PeerList) PropagatePeerList() {

	// Build the command to be sent.
	var cmd string
	var buf []byte
	buf, _ = json.Marshal(pl.peersMap)
	cmd = "<<peerlist=" + string(buf) + ">>"

	// Send the command to each peer.
	for i, v := range pl.peers {
		if v != pl.hostPort {
			config.G.Log.System.LogInfo("Sending peer list to %s", v)
			pl.target <- indexedLine{i, cmd}
		}
	}
}

// run listens for stat lines on a channel and sends them to the appropriate Cassabon peer.
func (pl *PeerList) run() {

	for {
		select {
		case <-config.G.OnReload2:
			config.G.Log.System.LogDebug("PeerList::run received QUIT message")
			pl.wg.Done()
			return
		case il := <-pl.target:
			if pl.hostPort != pl.peers[il.peerIndex] {
				pl.conns[pl.peers[il.peerIndex]].Send(il.statLine)
			}
		}
	}
}

// sortedMapToArray converts a map to an array of its values, ordered by key.
func sortedMapToArray(m map[string]string) []string {
	var a, t []string
	for k, _ := range m {
		t = append(t, k)
	}
	sort.Strings(t)
	for _, v := range t {
		a = append(a, m[v])
	}
	return a
}
<commit_msg>Do not log "keeping" message for own address<commit_after>package listener

import (
	"encoding/json"
	"sort"
	"sync"

	"github.com/jeffpierce/cassabon/config"
	"github.com/jeffpierce/cassabon/pearson"
)

type indexedLine struct {
	peerIndex int
	statLine  string
}

// PeerList contains an ordered list of Cassabon peers.
type PeerList struct {
	wg       *sync.WaitGroup
	target   chan indexedLine  // Channel for forwarding a stat line to a Cassabon peer
	hostPort string            // Host:port on which the local server is listening
	peersMap map[string]string // Peer list as stored in the configuration
	peers    []string          // Host:port information for all Cassabon peers (inclusive)
	conns    map[string]*StubbornTCPConn
	self     sync.RWMutex
}

func (pl *PeerList) Init() {

	pl.conns = make(map[string]*StubbornTCPConn, 0)

	// Create the channel on which stats to forward are received.
	pl.target = make(chan indexedLine, 1)
}

// IsStarted indicates whether the structure has ever been updated.
func (pl *PeerList) IsStarted() bool {
	return pl.hostPort != ""
}

// Start records the current peer list and starts the forwarder goroutine.
func (pl *PeerList) Start(wg *sync.WaitGroup, hostPort string, peersMap map[string]string) {

	// Synchronize access by other goroutines.
	pl.self.Lock()
	defer pl.self.Unlock()

	pl.wg = wg
	pl.hostPort = hostPort
	pl.peersMap = peersMap

	// Dispose of any peer connections that are obsolete.
	peers := sortedMapToArray(pl.peersMap)
	for _, existing := range pl.peers {
		found := false
		for _, incoming := range peers {
			if existing == incoming {
				found = true
				break
			}
		}
		if existing != pl.hostPort {
			if !found {
				pl.conns[existing].Close()
				delete(pl.conns, existing)
			} else {
				config.G.Log.System.LogInfo("Keeping peer connection to %s", existing)
			}
		}
	}

	// Record the current set of peers.
	pl.peers = make([]string, len(peers))
	for i, v := range peers {
		pl.peers[i] = v
		// Set up peer connections for newly added peers.
		if _, found := pl.conns[v]; !found && v != pl.hostPort {
			pl.conns[v] = new(StubbornTCPConn)
			pl.conns[v].Open(v)
		}
	}

	// Start the forwarder goroutine.
	pl.wg.Add(1)
	go pl.run()
}

// IsEqual indicates whether the given new configuration is equal to the current.
func (pl *PeerList) IsEqual(hostPort string, peersMap map[string]string) bool {

	// Synchronize access by other goroutines.
	pl.self.RLock()
	defer pl.self.RUnlock()

	if pl.hostPort != hostPort {
		return false
	}

	peers := sortedMapToArray(peersMap)
	if len(pl.peers) != len(peers) {
		return false
	}
	for i, v := range pl.peers {
		if peers[i] != v {
			return false
		}
	}

	return true
}

// OwnerOf determines which host owns a particular stats path.
func (pl *PeerList) OwnerOf(statPath string) (int, bool) {
	peerIndex := int(pearson.Hash8(statPath)) % len(pl.peers)
	if pl.hostPort == pl.peers[peerIndex] {
		return peerIndex, true
	} else {
		return peerIndex, false
	}
}

// PropagatePeerList sends the current peer list to all known peers.
func (pl *PeerList) PropagatePeerList() {

	// Build the command to be sent.
	var cmd string
	var buf []byte
	buf, _ = json.Marshal(pl.peersMap)
	cmd = "<<peerlist=" + string(buf) + ">>"

	// Send the command to each peer.
	for i, v := range pl.peers {
		if v != pl.hostPort {
			config.G.Log.System.LogInfo("Sending peer list to %s", v)
			pl.target <- indexedLine{i, cmd}
		}
	}
}

// run listens for stat lines on a channel and sends them to the appropriate Cassabon peer.
func (pl *PeerList) run() {

	for {
		select {
		case <-config.G.OnReload2:
			config.G.Log.System.LogDebug("PeerList::run received QUIT message")
			pl.wg.Done()
			return
		case il := <-pl.target:
			if pl.hostPort != pl.peers[il.peerIndex] {
				pl.conns[pl.peers[il.peerIndex]].Send(il.statLine)
			}
		}
	}
}

// sortedMapToArray converts a map to an array of its values, ordered by key.
func sortedMapToArray(m map[string]string) []string {
	var a, t []string
	for k, _ := range m {
		t = append(t, k)
	}
	sort.Strings(t)
	for _, v := range t {
		a = append(a, m[v])
	}
	return a
}
<|endoftext|>


<commit_before>package lazycache

import (
	"fmt"
	kitlog "github.com/go-kit/kit/log"
	"io"
	"net/http"
	"os"
	"path"
)

type LocalImageStore struct {
	LocalRoot string
	UrlRoot   string
	logger    kitlog.Logger

	Stats struct {
		cacheRequests int
		cacheMisses   int
	}

	cache map[string]int
}

func (store *LocalImageStore) Has(key string) bool {
	filename := store.LocalRoot + key

	_, has := store.cache[filename]
	if has {
		store.cache[filename]++
		return true
	}

	DefaultLogger.Log("level", "debug", "msg", fmt.Sprintf("Checking for \"%s\"", filename))
	_, err := os.Stat(filename)
	if err != nil {
		store.cache[filename] = 1
		return true
	}

	return false
}

func (store *LocalImageStore) Url(key string) (string, bool) {

	store.Stats.cacheRequests++

	if store.Has(key) {
		return store.UrlRoot + key, true
	} else {
		store.Stats.cacheMisses++
		return "", false
	}
}

func RecursiveMkdir(dir string) {
	_, err := os.Stat(dir)
	if err != nil {
		RecursiveMkdir(path.Dir(dir))

		os.Mkdir(dir, 0755)
	}
}

func (store LocalImageStore) Store(key string, data io.Reader) {
	filename := store.LocalRoot + key
	RecursiveMkdir(path.Dir(filename))

	f, err := os.Create(filename)
	if err != nil {
		DefaultLogger.Log("msg", err.Error(), "type", "error")
	}

	io.Copy(f, data)
}

func (store LocalImageStore) Retrieve(key string) (io.Reader, error) {

	f, err := os.Open(store.LocalRoot + key)

	return f, err
}

func (store LocalImageStore) Statistics() interface{} {
	return struct {
		Type          string
		CacheRequests int `json: "cache_requests"`
		CacheMisses   int `json: "cache_misses"`
	}{
		Type:          "local_storage",
		CacheRequests: store.Stats.cacheRequests,
		CacheMisses:   store.Stats.cacheMisses,
	}
}

func (store LocalImageStore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	localPath := path.Join(store.LocalRoot, r.URL.Path)

	if _, err := os.Stat(localPath); err != nil {
		http.Error(w, fmt.Sprintf("Could not find \"%s\"", localPath), 404)
	} else {
		http.ServeFile(w, r, localPath)
	}
}

func CreateLocalStore(localRoot string, addr string) *LocalImageStore {

	// port := 7080
	// addr := fmt.Sprintf("%s:%d", host, port)

	store := &LocalImageStore{
		LocalRoot: localRoot,
		UrlRoot:   addr,
		logger:    kitlog.With(DefaultLogger, "module", "LocalImageStore"),
	}

	DefaultLogger.Log("msg",
		fmt.Sprintf("Creating local image store at \"%s\", exposed at \"%s\"\n", store.LocalRoot, store.UrlRoot))

	s := &http.Server{
		Addr:    addr,
		Handler: store,
	}

	go s.ListenAndServe()

	return store
}
<commit_msg>Trying to improve performance of cache.<commit_after>package lazycache

import (
	"fmt"
	kitlog "github.com/go-kit/kit/log"
	"io"
	"net/http"
	"os"
	"path"
)

type LocalImageStore struct {
	LocalRoot string
	UrlRoot   string
	logger    kitlog.Logger

	Stats struct {
		cacheRequests int
		cacheMisses   int
	}

	cache map[string]int
}

func (store *LocalImageStore) Has(key string) bool {
	filename := store.LocalRoot + key

	_, has := store.cache[filename]
	if has {
		DefaultLogger.Log("level", "debug", "msg", fmt.Sprintf("\"%s\" exists in cache", filename))
		store.cache[filename]++
		return true
	}

	DefaultLogger.Log("level", "debug", "msg", fmt.Sprintf("Checking for \"%s\"", filename))
	_, err := os.Stat(filename)
	if err != nil {
		store.cache[filename] = 1
		return true
	}

	return false
}

func (store *LocalImageStore) Url(key string) (string, bool) {

	store.Stats.cacheRequests++

	if store.Has(key) {
		return store.UrlRoot + key, true
	} else {
		store.Stats.cacheMisses++
		return "", false
	}
}

func RecursiveMkdir(dir string) {
	_, err := os.Stat(dir)
	if err != nil {
		RecursiveMkdir(path.Dir(dir))

		os.Mkdir(dir, 0755)
	}
}

func (store LocalImageStore) Store(key string, data io.Reader) {
	filename := store.LocalRoot + key
	RecursiveMkdir(path.Dir(filename))

	f, err := os.Create(filename)
	if err != nil {
		DefaultLogger.Log("msg", err.Error(), "type", "error")
	}

	io.Copy(f, data)
}

func (store LocalImageStore) Retrieve(key string) (io.Reader, error) {

	f, err := os.Open(store.LocalRoot + key)

	return f, err
}

func (store LocalImageStore) Statistics() interface{} {
	return struct {
		Type          string
		CacheRequests int `json: "cache_requests"`
		CacheMisses   int `json: "cache_misses"`
	}{
		Type:          "local_storage",
		CacheRequests: store.Stats.cacheRequests,
		CacheMisses:   store.Stats.cacheMisses,
	}
}

func (store LocalImageStore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	localPath := path.Join(store.LocalRoot, r.URL.Path)

	if _, err := os.Stat(localPath); err != nil {
		http.Error(w, fmt.Sprintf("Could not find \"%s\"", localPath), 404)
	} else {
		http.ServeFile(w, r, localPath)
	}
}

func CreateLocalStore(localRoot string, addr string) *LocalImageStore {

	// port := 7080
	// addr := fmt.Sprintf("%s:%d", host, port)

	store := &LocalImageStore{
		LocalRoot: localRoot,
		UrlRoot:   addr,
		logger:    kitlog.With(DefaultLogger, "module", "LocalImageStore"),
	}

	DefaultLogger.Log("msg",
		fmt.Sprintf("Creating local image store at \"%s\", exposed at \"%s\"\n", store.LocalRoot, store.UrlRoot))

	s := &http.Server{
		Addr:    addr,
		Handler: store,
	}

	go s.ListenAndServe()

	return store
}
<|endoftext|>


<commit_before>package logsreader

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"sync"

	"github.com/pkg/sftp"
	"golang.org/x/crypto/ssh"
)

// ReadLogs read logs from server
func ReadLogs(connection ConnectionInfo, readerState State, recordProcessor func(*LogRecord)) (*State, error) {
	sftp, err := connectToServer(connection)
	if err != nil {
		return nil, fmt.Errorf("fail to connect to server %s: %v", conn, err)
	}
	defer sftp.Close()

	notZipped := findNotZippedLog(sftp)

	var logOffset int
	if readerState.NotZippedLogFile == notZipped {
		logOffset = readerState.BytesRead
	} else {
		logOffset = 0

		_, err = processRecords(sftp, notZipped, readerState.BytesRead, recordProcessor)
		if err != nil {
			return nil, err
		}
	}

	bytesRead, err := processRecords(sftp, "/var/log/nginx/access.log", logOffset, recordProcessor)
	if err != nil {
		return nil, err
	}

	newState := &State{
		NotZippedLogFile: notZipped,
		BytesRead:        bytesRead + logOffset,
	}

	return newState, nil
}

func connectToServer(connection ConnectionInfo) (*sftp.Client, error) {
	clientConfig := &ssh.ClientConfig{
		User: connection.UserName,
		Auth: []ssh.AuthMethod{
			ssh.Password(connection.Password),
		},
	}

	addressWithPort := fmt.Sprintf("%s:%d", connection.Address, connection.Port)
	client, err := ssh.Dial("tcp", addressWithPort, clientConfig)

	if err != nil {
		return nil, fmt.Errorf("cannot dial remote server: %v", err)
	}

	sftp, err := sftp.NewClient(client)
	if err != nil {
		return nil, fmt.Errorf("fail to create sftp client: %v", err)
	}

	return sftp, nil
}

func findNotZippedLog(sftp *sftp.Client) string {
	w := sftp.Walk("/var/log/nginx/")
	for w.Step() {
		if err := w.Err(); err != nil {
			continue
		}

		fullPath := w.Path()
		fileName := path.Base(fullPath)

		if strings.HasPrefix(fileName, "access.log-") && !strings.HasSuffix(fileName, ".gz") {
			return fullPath
		}
	}

	return ""
}

func processRecords(client *sftp.Client, fileName string, readFrom int, recordProcessor func(*LogRecord)) (int, error) {
	log.Printf("opening file %s\n", fileName)
	file, err := client.Open(fileName)
	if err != nil {
		return 0, fmt.Errorf("cannot open %s: %v", fileName, err)
	}

	defer file.Close()

	_, err = file.Seek(int64(readFrom), os.SEEK_SET)
	if err != nil {
		return 0, fmt.Errorf("cannot seek to %d in %s: %v", readFrom, fileName, err)
	}

	log.Printf("reading file %s from position %d\n", fileName, readFrom)

	bytesRead := 0
	scanner := bufio.NewScanner(file)

	var throttle = make(chan bool, 200)
	var wg sync.WaitGroup
	for scanner.Scan() {
		logLine := scanner.Text()

		throttle <- true
		wg.Add(1)
		go func(logLine string) {
			defer wg.Done()
			logRecord, err := parseLine(logLine)
			if err != nil {
				log.Printf("fail to parse %s\n", logLine)
				return
			}

			recordProcessor(logRecord)
			<-throttle
		}(logLine)

		// 1 is length of line separator (\n)
		bytesRead += len(logLine) + 1
	}
	wg.Wait()

	return bytesRead, nil
}
<commit_msg>fix issue with logs rotation app using different renaming conventions<commit_after>package logsreader

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/pkg/sftp"
	"golang.org/x/crypto/ssh"
)

const (
	logPath = "/var/log/nginx/access.log"
)

// ReadLogs read logs from server
func ReadLogs(conn ConnectionInfo, readerState State, recordProcessor func(*LogRecord)) (*State, error) {
	sftp, err := connectToServer(conn)
	if err != nil {
		return nil, fmt.Errorf("fail to connect to server %s: %v", conn, err)
	}
	defer sftp.Close()

	notZipped := findNotZippedLog(sftp)

	var logOffset int
	if readerState.NotZippedLogFile == notZipped {
		logOffset = readerState.BytesRead
	} else {
		logOffset = 0

		_, err = processRecords(sftp, notZipped, readerState.BytesRead, recordProcessor)
		if err != nil {
			return nil, err
		}
	}

	bytesRead, err := processRecords(sftp, logPath, logOffset, recordProcessor)
	if err != nil {
		return nil, err
	}

	newState := &State{
		NotZippedLogFile: notZipped,
		BytesRead:        bytesRead + logOffset,
	}

	return newState, nil
}

func connectToServer(connection ConnectionInfo) (*sftp.Client, error) {
	clientConfig := &ssh.ClientConfig{
		User: connection.UserName,
		Auth: []ssh.AuthMethod{
			ssh.Password(connection.Password),
		},
	}

	addressWithPort := fmt.Sprintf("%s:%d", connection.Address, connection.Port)
	client, err := ssh.Dial("tcp", addressWithPort, clientConfig)

	if err != nil {
		return nil, fmt.Errorf("cannot dial remote server: %v", err)
	}

	sftp, err := sftp.NewClient(client)
	if err != nil {
		return nil, fmt.Errorf("fail to create sftp client: %v", err)
	}

	return sftp, nil
}

func findNotZippedLog(sftp *sftp.Client) string {
	logDir := filepath.Dir(logPath)
	logName := filepath.Base(logPath)

	w := sftp.Walk(logDir)
	for w.Step() {
		if w.Err() != nil {
			continue
		}

		fullPath := w.Path()
		fileName := path.Base(fullPath)

		if fileName != logName && strings.HasPrefix(fileName, logName) && !strings.HasSuffix(fileName, ".gz") {
			return fullPath
		}
	}

	return ""
}

func processRecords(client *sftp.Client, fileName string, readFrom int, recordProcessor func(*LogRecord)) (int, error) {
	log.Printf("opening file %s\n", fileName)
	file, err := client.Open(fileName)
	if err != nil {
		return 0, fmt.Errorf("cannot open %s: %v", fileName, err)
	}

	defer file.Close()

	_, err = file.Seek(int64(readFrom), os.SEEK_SET)
	if err != nil {
		return 0, fmt.Errorf("cannot seek to %d in %s: %v", readFrom, fileName, err)
	}

	log.Printf("reading file %s from position %d\n", fileName, readFrom)

	bytesRead := 0
	scanner := bufio.NewScanner(file)

	var throttle = make(chan bool, 200)
	var wg sync.WaitGroup
	for scanner.Scan() {
		logLine := scanner.Text()

		throttle <- true
		wg.Add(1)
		go func(logLine string) {
			defer wg.Done()
			logRecord, err := parseLine(logLine)
			if err != nil {
				log.Printf("fail to parse %s\n", logLine)
				return
			}

			recordProcessor(logRecord)
			<-throttle
		}(logLine)

		// 1 is length of line separator (\n)
		bytesRead += len(logLine) + 1
	}
	wg.Wait()

	return bytesRead, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// This test requires Rescheduler to be enabled.
var _ = framework.KubeDescribe("Rescheduler [Serial]", func() {
	f := framework.NewDefaultFramework("rescheduler")
	var ns string
	var totalMillicores int

	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce")
		ns = f.Namespace.Name
		nodes := framework.GetReadySchedulableNodesOrDie(f.Client)
		nodeCount := len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU]
		totalMillicores = int((&cpu).MilliValue()) * nodeCount
	})

	It("should ensure that critical pod is scheduled in case there is no resources available", func() {
		By("reserving all available cpu")
		err := reserveAllCpu(f, "reserve-all-cpu", totalMillicores)
		defer framework.DeleteRCAndPods(f.Client, ns, "reserve-all-cpu")
		framework.ExpectNoError(err)

		By("creating a new instance of DNS and waiting for DNS to be scheduled")
		label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
		listOpts := api.ListOptions{LabelSelector: label}
		rcs, err := f.Client.ReplicationControllers(api.NamespaceSystem).List(listOpts)
		framework.ExpectNoError(err)
		Expect(len(rcs.Items)).Should(Equal(1))

		rc := rcs.Items[0]
		replicas := uint(rc.Spec.Replicas)

		err = framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas+1, true)
		defer framework.ExpectNoError(framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas, true))
		framework.ExpectNoError(err)
	})
})

func reserveAllCpu(f *framework.Framework, id string, millicores int) error {
	timeout := 5 * time.Minute
	replicas := millicores / 100

	ReserveCpu(f, id, 1, 100)
	framework.ExpectNoError(framework.ScaleRC(f.Client, f.Namespace.Name, id, uint(replicas), false))

	for start := time.Now(); time.Since(start) < timeout; time.Sleep(10 * time.Second) {
		pods, err := framework.GetPodsInNamespace(f.Client, f.Namespace.Name, framework.ImagePullerLabels)
		if err != nil {
			return err
		}

		if len(pods) != replicas {
			continue
		}

		allRunningOrUnschedulable := true
		for _, pod := range pods {
			if !podRunningOrUnschedulable(pod) {
				allRunningOrUnschedulable = false
				break
			}
		}
		if allRunningOrUnschedulable {
			return nil
		}
	}
	return fmt.Errorf("Pod name %s: Gave up waiting %v for %d pods to come up", id, timeout, replicas)
}

func podRunningOrUnschedulable(pod *api.Pod) bool {
	_, cond := api.GetPodCondition(&pod.Status, api.PodScheduled)
	if cond != nil && cond.Status == api.ConditionFalse && cond.Reason == "Unschedulable" {
		return true
	}
	running, _ := framework.PodRunningReady(pod)
	return running
}
<commit_msg>Enabled Rescheduler e2e for GKE<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// This test requires Rescheduler to be enabled.
var _ = framework.KubeDescribe("Rescheduler [Serial]", func() {
	f := framework.NewDefaultFramework("rescheduler")
	var ns string
	var totalMillicores int

	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce", "gke")
		ns = f.Namespace.Name
		nodes := framework.GetReadySchedulableNodesOrDie(f.Client)
		nodeCount := len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU]
		totalMillicores = int((&cpu).MilliValue()) * nodeCount
	})

	It("should ensure that critical pod is scheduled in case there is no resources available", func() {
		By("reserving all available cpu")
		err := reserveAllCpu(f, "reserve-all-cpu", totalMillicores)
		defer framework.DeleteRCAndPods(f.Client, ns, "reserve-all-cpu")
		framework.ExpectNoError(err)

		By("creating a new instance of DNS and waiting for DNS to be scheduled")
		label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
		listOpts := api.ListOptions{LabelSelector: label}
		rcs, err := f.Client.ReplicationControllers(api.NamespaceSystem).List(listOpts)
		framework.ExpectNoError(err)
		Expect(len(rcs.Items)).Should(Equal(1))

		rc := rcs.Items[0]
		replicas := uint(rc.Spec.Replicas)

		err = framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas+1, true)
		defer framework.ExpectNoError(framework.ScaleRC(f.Client, api.NamespaceSystem, rc.Name, replicas, true))
		framework.ExpectNoError(err)
	})
})

func reserveAllCpu(f *framework.Framework, id string, millicores int) error {
	timeout := 5 * time.Minute
	replicas := millicores / 100

	ReserveCpu(f, id, 1, 100)
	framework.ExpectNoError(framework.ScaleRC(f.Client, f.Namespace.Name, id, uint(replicas), false))

	for start := time.Now(); time.Since(start) < timeout; time.Sleep(10 * time.Second) {
		pods, err := framework.GetPodsInNamespace(f.Client, f.Namespace.Name, framework.ImagePullerLabels)
		if err != nil {
			return err
		}

		if len(pods) != replicas {
			continue
		}

		allRunningOrUnschedulable := true
		for _, pod := range pods {
			if !podRunningOrUnschedulable(pod) {
				allRunningOrUnschedulable = false
				break
			}
		}
		if allRunningOrUnschedulable {
			return nil
		}
	}
	return fmt.Errorf("Pod name %s: Gave up waiting %v for %d pods to come up", id, timeout, replicas)
}

func podRunningOrUnschedulable(pod *api.Pod) bool {
	_, cond := api.GetPodCondition(&pod.Status, api.PodScheduled)
	if cond != nil && cond.Status == api.ConditionFalse && cond.Reason == "Unschedulable" {
		return true
	}
	running, _ := framework.PodRunningReady(pod)
	return running
}
<|endoftext|>


<commit_before>package gofaker

import (
	"regexp"
	"strconv"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestAddress(t *testing.T) {
	f, err := NewFaker("en")
	assert.NoError(t, err)
	t.Run("City", func(t *testing.T) { assert.NotEmpty(t, f.Address.City()) })
	t.Run("StreetName", func(t *testing.T) { assert.NotEmpty(t, f.Address.StreetName()) })
	t.Run("SteetAddress", func(t *testing.T) { assert.NotEmpty(t, f.Address.SteetAddress()) })
	t.Run("BuildingNumber", func(t *testing.T) { assert.NotEmpty(t, f.Address.BuildingNumber()) })
	t.Run("ZipCode", func(t *testing.T) { assert.NotEmpty(t, f.Address.ZipCode()) })
	t.Run("ZipCodeByState", func(t *testing.T) {
		out := f.Address.ZipCodeByState("AZ")
		assert.Regexp(t, regexp.MustCompile(".*AZ$"), out)
	})
	t.Run("TimeZone", func(t *testing.T) { assert.NotEmpty(t, f.Address.TimeZone()) })

	t.Run("StreetSuffix", func(t *testing.T) { assert.NotEmpty(t, f.Address.StreetSuffix()) })
	t.Run("CitySuffix", func(t *testing.T) { assert.NotEmpty(t, f.Address.CitySuffix()) })
	t.Run("CityPrefix", func(t *testing.T) { assert.NotEmpty(t, f.Address.CityPrefix()) })
	t.Run("StateAbbr", func(t *testing.T) { assert.NotEmpty(t, f.Address.StateAbbr()) })
	t.Run("State", func(t *testing.T) { assert.NotEmpty(t, f.Address.State()) })
	t.Run("Country", func(t *testing.T) { assert.NotEmpty(t, f.Address.Country()) })
	t.Run("CountryCode", func(t *testing.T) { assert.NotEmpty(t, f.Address.CountryCode()) })
	t.Run("Latitude", func(t *testing.T) {
		out := f.Address.Latitude()
		lat, err := strconv.ParseFloat(out, 64)
		assert.NoError(t, err)
		if lat > 90.0 {
			t.Errorf("expected %f to be less than 90.0", lat)
		}
		if lat < -90.0 {
			t.Errorf("expected %f to be greater or equal than -90.0", lat)
		}
	})

	t.Run("Longitude", func(t *testing.T) {
		out := f.Address.Longitude()
		lon, err := strconv.ParseFloat(out, 64)
		assert.NoError(t, err)
		if lon > 180.0 {
			t.Errorf("expected %f to be less than 180.0", lon)
		}
		if lon < -180.0 {
			t.Errorf("expected %f to be greater or equal than -180.0", lon)
		}
	})
}
<commit_msg>failing unit test<commit_after>package gofaker

import (
	"regexp"
	"strconv"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestAddress(t *testing.T) {
	f, err := NewFaker("en")
	assert.NoError(t, err)
	t.Run("City", func(t *testing.T) { assert.NotEmpty(t, f.Address.City()) })
	t.Run("StreetName", func(t *testing.T) { assert.NotEmpty(t, f.Address.StreetName()) })
	t.Run("StreetAddress", func(t *testing.T) { assert.NotEmpty(t, f.Address.StreetAddress()) })
	t.Run("BuildingNumber", func(t *testing.T) { assert.NotEmpty(t, f.Address.BuildingNumber()) })
	t.Run("ZipCode", func(t *testing.T) { assert.NotEmpty(t, f.Address.ZipCode()) })
	t.Run("ZipCodeByState", func(t *testing.T) {
		out := f.Address.ZipCodeByState("AZ")
		assert.Regexp(t, regexp.MustCompile(".*AZ$"), out)
	})
	t.Run("TimeZone", func(t *testing.T) { assert.NotEmpty(t, f.Address.TimeZone()) })

	t.Run("StreetSuffix", func(t *testing.T) { assert.NotEmpty(t, f.Address.StreetSuffix()) })
	t.Run("CitySuffix", func(t *testing.T) { assert.NotEmpty(t, f.Address.CitySuffix()) })
	t.Run("CityPrefix", func(t *testing.T) { assert.NotEmpty(t, f.Address.CityPrefix()) })
	t.Run("StateAbbr", func(t *testing.T) { assert.NotEmpty(t, f.Address.StateAbbr()) })
	t.Run("State", func(t *testing.T) { assert.NotEmpty(t, f.Address.State()) })
	t.Run("Country", func(t *testing.T) { assert.NotEmpty(t, f.Address.Country()) })
	t.Run("CountryCode", func(t *testing.T) { assert.NotEmpty(t, f.Address.CountryCode()) })
	t.Run("Latitude", func(t *testing.T) {
		out := f.Address.Latitude()
		lat, err := strconv.ParseFloat(out, 64)
		assert.NoError(t, err)
		if lat > 90.0 {
			t.Errorf("expected %f to be less than 90.0", lat)
		}
		if lat < -90.0 {
			t.Errorf("expected %f to be greater or equal than -90.0", lat)
		}
	})

	t.Run("Longitude", func(t *testing.T) {
		out := f.Address.Longitude()
		lon, err := strconv.ParseFloat(out, 64)
		assert.NoError(t, err)
		if lon > 180.0 {
			t.Errorf("expected %f to be less than 180.0", lon)
		}
		if lon < -180.0 {
			t.Errorf("expected %f to be greater or equal than -180.0", lon)
		}
	})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/github/git-lfs/lfs"
)

var (
	BuildOS    = flag.String("os", "", "OS to target: darwin, freebsd, linux, windows")
	BuildArch  = flag.String("arch", "", "Arch to target: 386, amd64")
	BuildAll   = flag.Bool("all", false, "Builds all architectures")
	ShowHelp   = flag.Bool("help", false, "Shows help")
	matrixKeys = map[string]string{
		"darwin":  "Mac",
		"freebsd": "FreeBSD",
		"linux":   "Linux",
		"windows": "Windows",
		"amd64":   "AMD64",
	}
	LdFlag string
)

func mainBuild() {
	if *ShowHelp {
		fmt.Println("usage: script/bootstrap [-os] [-arch] [-all]")
		flag.PrintDefaults()
		return
	}

	fmt.Printf("Using %s\n", runtime.Version())

	cmd, _ := exec.Command("git", "rev-parse", "--short", "HEAD").Output()

	if len(cmd) > 0 {
		LdFlag = strings.TrimSpace("-X github.com/github/git-lfs/lfs.GitCommit " + string(cmd))
	}

	buildMatrix := make(map[string]Release)
	errored := false

	if *BuildAll {
		for _, buildos := range []string{"darwin", "freebsd", "linux", "windows"} {
			for _, buildarch := range []string{"386", "amd64"} {
				if err := build(buildos, buildarch, buildMatrix); err != nil {
					errored = true
				}
			}
		}
	} else {
		if err := build(*BuildOS, *BuildArch, buildMatrix); err != nil {
			log.Fatalln(err)
		}
		return // skip build matrix stuff
	}

	if errored {
		os.Exit(1)
	}

	by, err := json.Marshal(buildMatrix)
	if err != nil {
		log.Fatalln("Error encoding build matrix to json:", err)
	}

	file, err := os.Create("bin/releases/build_matrix.json")
	if err != nil {
		log.Fatalln("Error creating build_matrix.json:", err)
	}

	written, err := file.Write(by)
	file.Close()

	if err != nil {
		log.Fatalln("Error writing build_matrix.json", err)
	}

	if jsonSize := len(by); written != jsonSize {
		log.Fatalf("Expected to write %d bytes, actually wrote %d.\n", jsonSize, written)
	}
}

func build(buildos, buildarch string, buildMatrix map[string]Release) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0
	name := "git-lfs-" + lfs.Version
	dir := "bin"

	if addenv {
		fmt.Printf("Building for %s/%s\n", buildos, buildarch)
		dir = filepath.Join(dir, "releases", buildos+"-"+buildarch, name)
	}

	if err := buildCommand(dir, buildos, buildarch); err != nil {
		return err
	}

	if addenv {
		err := os.MkdirAll(dir, 0755)
		if err != nil {
			log.Println("Error setting up installer:\n", err.Error())
			return err
		}

		err = setupInstaller(buildos, buildarch, dir, buildMatrix)
		if err != nil {
			log.Println("Error setting up installer:\n", err.Error())
			return err
		}
	}

	return nil
}

func buildCommand(dir, buildos, buildarch string) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0

	bin := filepath.Join(dir, "git-lfs")

	if buildos == "windows" {
		bin = bin + ".exe"
	}

	args := make([]string, 1, 6)
	args[0] = "build"
	if len(LdFlag) > 0 {
		args = append(args, "-ldflags", LdFlag)
	}
	args = append(args, "-o", bin, ".")

	cmd := exec.Command("go", args...)
	if addenv {
		cmd.Env = []string{
			"GOOS=" + buildos,
			"GOARCH=" + buildarch,
			"GOPATH=" + os.Getenv("GOPATH"),
			"GOROOT=" + os.Getenv("GOROOT"),
		}
	}

	output, err := cmd.CombinedOutput()
	if len(output) > 0 {
		fmt.Println(string(output))
	}
	return err
}

func setupInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	files := []string{
		"README.md", "CHANGELOG.md",
	}

	for _, filename := range files {
		cmd := exec.Command("cp", filename, filepath.Join(dir, filename))
		if err := logAndRun(cmd); err != nil {
			return err
		}
	}

	// Windows installer is uploaded separately. See script/nsis.
	if buildos == "windows" {
		return nil
	}

	fullInstallPath := filepath.Join(dir, "install.sh")
	cmd := exec.Command("cp", "script/install.sh.example", fullInstallPath)
	if err := logAndRun(cmd); err != nil {
		return err
	}

	if err := os.Chmod(fullInstallPath, 0755); err != nil {
		return err
	}

	name := zipName(buildos, buildarch) + ".tar.gz"

	addToMatrix(buildMatrix, buildos, buildarch, name)

	cmd = exec.Command("tar", "czf", "../"+name, filepath.Base(dir))
	cmd.Dir = filepath.Dir(dir)
	return logAndRun(cmd)
}

func addToMatrix(buildMatrix map[string]Release, buildos, buildarch, name string) {
	buildMatrix[fmt.Sprintf("%s-%s", buildos, buildarch)] = Release{
		Label:    releaseLabel(buildos, buildarch),
		Filename: name,
	}
}

func logAndRun(cmd *exec.Cmd) error {
	fmt.Printf(" - %s\n", strings.Join(cmd.Args, " "))
	if len(cmd.Dir) > 0 {
		fmt.Printf("   - in %s\n", cmd.Dir)
	}

	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	return err
}

func zipName(os, arch string) string {
	return fmt.Sprintf("git-lfs-%s-%s-%s", os, arch, lfs.Version)
}

func releaseLabel(buildos, buildarch string) string {
	return fmt.Sprintf("%s %s", key(buildos), key(buildarch))
}

func key(k string) string {
	if s, ok := matrixKeys[k]; ok {
		return s
	}
	return k
}
<commit_msg>prioritize the builds i typically care most<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/github/git-lfs/lfs"
)

var (
	BuildOS    = flag.String("os", "", "OS to target: darwin, freebsd, linux, windows")
	BuildArch  = flag.String("arch", "", "Arch to target: 386, amd64")
	BuildAll   = flag.Bool("all", false, "Builds all architectures")
	ShowHelp   = flag.Bool("help", false, "Shows help")
	matrixKeys = map[string]string{
		"darwin":  "Mac",
		"freebsd": "FreeBSD",
		"linux":   "Linux",
		"windows": "Windows",
		"amd64":   "AMD64",
	}
	LdFlag string
)

func mainBuild() {
	if *ShowHelp {
		fmt.Println("usage: script/bootstrap [-os] [-arch] [-all]")
		flag.PrintDefaults()
		return
	}

	fmt.Printf("Using %s\n", runtime.Version())

	cmd, _ := exec.Command("git", "rev-parse", "--short", "HEAD").Output()

	if len(cmd) > 0 {
		LdFlag = strings.TrimSpace("-X github.com/github/git-lfs/lfs.GitCommit " + string(cmd))
	}

	buildMatrix := make(map[string]Release)
	errored := false

	if *BuildAll {
		for _, buildos := range []string{"linux", "windows", "darwin", "freebsd"} {
			for _, buildarch := range []string{"amd64", "386"} {
				if err := build(buildos, buildarch, buildMatrix); err != nil {
					errored = true
				}
			}
		}
	} else {
		if err := build(*BuildOS, *BuildArch, buildMatrix); err != nil {
			log.Fatalln(err)
		}
		return // skip build matrix stuff
	}

	if errored {
		os.Exit(1)
	}

	by, err := json.Marshal(buildMatrix)
	if err != nil {
		log.Fatalln("Error encoding build matrix to json:", err)
	}

	file, err := os.Create("bin/releases/build_matrix.json")
	if err != nil {
		log.Fatalln("Error creating build_matrix.json:", err)
	}

	written, err := file.Write(by)
	file.Close()

	if err != nil {
		log.Fatalln("Error writing build_matrix.json", err)
	}

	if jsonSize := len(by); written != jsonSize {
		log.Fatalf("Expected to write %d bytes, actually wrote %d.\n", jsonSize, written)
	}
}

func build(buildos, buildarch string, buildMatrix map[string]Release) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0
	name := "git-lfs-" + lfs.Version
	dir := "bin"

	if addenv {
		fmt.Printf("Building for %s/%s\n", buildos, buildarch)
		dir = filepath.Join(dir, "releases", buildos+"-"+buildarch, name)
	}

	if err := buildCommand(dir, buildos, buildarch); err != nil {
		return err
	}

	if addenv {
		err := os.MkdirAll(dir, 0755)
		if err != nil {
			log.Println("Error setting up installer:\n", err.Error())
			return err
		}

		err = setupInstaller(buildos, buildarch, dir, buildMatrix)
		if err != nil {
			log.Println("Error setting up installer:\n", err.Error())
			return err
		}
	}

	return nil
}

func buildCommand(dir, buildos, buildarch string) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0

	bin := filepath.Join(dir, "git-lfs")

	if buildos == "windows" {
		bin = bin + ".exe"
	}

	args := make([]string, 1, 6)
	args[0] = "build"
	if len(LdFlag) > 0 {
		args = append(args, "-ldflags", LdFlag)
	}
	args = append(args, "-o", bin, ".")

	cmd := exec.Command("go", args...)
	if addenv {
		cmd.Env = []string{
			"GOOS=" + buildos,
			"GOARCH=" + buildarch,
			"GOPATH=" + os.Getenv("GOPATH"),
			"GOROOT=" + os.Getenv("GOROOT"),
		}
	}

	output, err := cmd.CombinedOutput()
	if len(output) > 0 {
		fmt.Println(string(output))
	}
	return err
}

func setupInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	files := []string{
		"README.md", "CHANGELOG.md",
	}

	for _, filename := range files {
		cmd := exec.Command("cp", filename, filepath.Join(dir, filename))
		if err := logAndRun(cmd); err != nil {
			return err
		}
	}

	// Windows installer is uploaded separately. See script/nsis.
	if buildos == "windows" {
		return nil
	}

	fullInstallPath := filepath.Join(dir, "install.sh")
	cmd := exec.Command("cp", "script/install.sh.example", fullInstallPath)
	if err := logAndRun(cmd); err != nil {
		return err
	}

	if err := os.Chmod(fullInstallPath, 0755); err != nil {
		return err
	}

	name := zipName(buildos, buildarch) + ".tar.gz"

	addToMatrix(buildMatrix, buildos, buildarch, name)

	cmd = exec.Command("tar", "czf", "../"+name, filepath.Base(dir))
	cmd.Dir = filepath.Dir(dir)
	return logAndRun(cmd)
}

func addToMatrix(buildMatrix map[string]Release, buildos, buildarch, name string) {
	buildMatrix[fmt.Sprintf("%s-%s", buildos, buildarch)] = Release{
		Label:    releaseLabel(buildos, buildarch),
		Filename: name,
	}
}

func logAndRun(cmd *exec.Cmd) error {
	fmt.Printf(" - %s\n", strings.Join(cmd.Args, " "))
	if len(cmd.Dir) > 0 {
		fmt.Printf("   - in %s\n", cmd.Dir)
	}

	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	return err
}

func zipName(os, arch string) string {
	return fmt.Sprintf("git-lfs-%s-%s-%s", os, arch, lfs.Version)
}

func releaseLabel(buildos, buildarch string) string {
	return fmt.Sprintf("%s %s", key(buildos), key(buildarch))
}

func key(k string) string {
	if s, ok := matrixKeys[k]; ok {
		return s
	}
	return k
}
<|endoftext|>


<commit_before>/*
参考：《Go语言编程》
Author: coderzh(http://blog.coderzh.com)
*/

package main

import (
	"fmt"
	"log"
	"os"
	"sync"
	// "errors"
	"math"
	"testing"
)

// 0. 注释
/*
规范：
 1. 命名：骆驼命名法（不要用下划线）

命令:
 go get github.com/coderzh/xxx
 go build calc
 go run xxx.go
 go install calc
*/

// 1. Hello World
func helloWorld() {
	fmt.Println("Hello, 世界")
}

// 2.变量类型
func typeDemo() {
	// 变量申明
	var v1 int
	var (
		v2 int
		v3 string
	)
	//var p *int // 指针类型

	// 变量初始化
	var v4 int = 10
	// 等价于:
	var v5 = 10
	// 一般这样就好
	v6 := 10

	// 赋值，多重赋值
	v1 = 10
	v2, v3 = 20, "test"
	// 匿名变量 _
	_, v4 = v5, v6

	fmt.Println(v1, v2, v3, v4)

	// 常量
	const Pi float64 = 3.1415926
	const MaxPlayer = 10

	// 枚举
	const (
		Sunday = iota // iota从0递增
		Mondy
		Tuesday
		// ...
	)

	// 类型
	// 1. 布尔
	var b1 bool
	b1 = true
	b1 = (1 == 2)

	fmt.Println(b1)

	// 2. 整形
	// int8 uint8 int16 uint16 int32 uint32 int64 uint64 int uint uintptr
	var i32 int32
	// 强制转换
	i32 = int32(64)
	// 运算：+, -, *, /, %（求余）
	// 比较：>, <, ==, >=, <=, !=
	// 位运算：x << y, x >> y, x ^ y, x & y, x | y, ^x （取反）

	fmt.Println(i32)

	// 3. 浮点
	// float32, float64
	var f1 float64 = 1.0001
	var f2 float64 = 1.0002
	// 浮点比较
	isEqual := math.Dim(f1, f2) < 0.0001

	fmt.Println(isEqual)

	// 4. 字符串
	var s1 string
	s1 = "abc"
	// 字符串连接
	s1 = s1 + "ddd"
	// 取长度
	n := len(s1)
	// 取字符
	c1 := s1[0]

	fmt.Println(c1)

	// 字节遍历
	for i := 0; i < n; i++ {
		ch := s1[i]
		fmt.Println(ch)
	}
	// Unicode字符遍历
	for i, ch := range s1 {
		fmt.Println(i, ch)
	}

	// 5. 数组
	var arr1 [32]int
	//var arr2 [3][8]int // 二维数组
	// 初始化
	arr1 = [32]int{0}
	array := [5]int{1, 2, 3, 4, 5}
	// 数组遍历
	for i := 0; i < len(array); i++ {
		fmt.Println(array[i])
	}
	for i, v := range arr1 {
		fmt.Println(i, v)
	}
	// 数组是值类型，每次参数传递都是一份拷贝

	// 数组切片Slice
	var mySlice []int = array[:2]
	mySlice1 := make([]int, 5)
	mySlice2 := make([]int, 5, 10)

	fmt.Println("len(mySlice2:", len(mySlice2)) // 5
	fmt.Println("cap(mySlice2:", cap(mySlice2)) // 10

	mySlice3 := append(mySlice, 2, 3, 4)
	mySlice4 := append(mySlice, mySlice1...)

	copy(mySlice3, mySlice4)

	// 6. Map
	var m map[int]string
	m[1] = "ddd"
	m1 := make(map[int]string)
	m2 := map[int]string{
		1: "a",
		2: "b",
	}

	delete(m2, 1)

	value, ok := m1[1]
	if ok {
		fmt.Println(value)
	}

	for k, v := range m2 {
		fmt.Println(k, v)
	}

}

// 3. 流程控制
func flowDemo() {
	// if else
	a := 10
	if a < 10 {
		// ..
	} else {
		// ..
	}

	// switch
	switch a {
	case 0:
		fmt.Println("0")
	case 10:
		fmt.Println("10")
	default:
		fmt.Println("default")
	}

	switch {
	case a < 10:
		fmt.Println("<10")
	case a < 20:
		fmt.Println("<20")
	}

	// 循环
	for i := 0; i < 10; i++ {
	}
	// 无限循环
	sum := 0
	for {
		sum++
		if sum > 10 {
			break
			// 指定break
			// break JLoop
		}
	}

	goto JLoop

JLoop:
	// break to here

}

// 4. 函数
// func 函数名(参数列表)(返回值列表) {
// }
func sum1(value1 int, value2 int) (result int, err error) {
	// err = errors.New("xxxx")
	return value1 + value2, nil
}

func sum2(value1, value2 int) int {
	return value1 + value2
}

// 不定参数
// myFunc(1, 2, 3, 4, 5)
func myFunc(args ...int) {
	for _, arg := range args {
		fmt.Println(arg)
	}
	// 传递
	// myFunc2(args...)
	// myFunc2(args[1:]...)
}

// 任意类型的不定参数
func myPrintf(args ...interface{}) {
	for _, arg := range args {
		switch arg.(type) {
		case int:
			fmt.Println(arg, " is int")
		case string:
			fmt.Println(arg, " is string")
		default:
			fmt.Println(arg, " is unknown")
		}
	}
}

// 匿名函数
func anonymousFunc() {
	f := func(a, b int) int {
		return a + b
	}

	f(1, 2)
}

// defer
func copyFile(path string) {
	f, err := os.Open(path)
	if err != nil {
		return
	}

	defer f.Close()
	// or
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Runtime error caught: %v", r)
		}
	}()
}

// 5. 结构体
type Rect struct {
	// 小写为private
	x, y float64
	// 大写为public
	Width, Height float64
}

// 大写方法为public，小写为private
func (r *Rect) Area() float64 {
	return r.Width * r.Height
}

func netRect(x, y, width, height float64) *Rect {
	// 实例化结构体
	// rect1 := new(Rect)
	// rect2 := &Rect{}
	// rect3 := &Rect{Width:100, Height:200}
	return &Rect{x, y, width, height}
}

// 匿名组合
type Base struct {
	Name string
}

func (base *Base) Foo() {}
func (base *Base) Bar() {}

type Foo struct {
	Base
	*log.Logger
}

func (foo *Foo) Bar() {
	foo.Base.Bar()
	// ...
}

// 非侵入式接口
type IFile interface {
	Read(buf []byte) (n int, err error)
	Write(buf []byte) (n int, err error)
}

type File struct {
}

func (file *File) Read(buf []byte) (n int, err error) {
	return 0, nil
}

func (file *File) Write(buf []byte) (n int, err error) {
	return 0, nil
}

func interfaceDemo() {
	// 只要实现了Read, Write方法即可
	var file IFile = new(File)

	// 接口查询
	// 是否实现了IFile接口
	if file2, ok := file.(IFile); ok {
		file2.Read([]byte{})
	}
	// 实例类型是否是File
	if file3, ok := file.(*File); ok {
		file3.Read([]byte{})
	}

	// 类型查询
	switch v := file.(type) {
	}
}

// 6. 并发编程
func counting(ch chan int) {
	ch <- 1
	fmt.Println("counting")
}

func channelDemo() {
	chs := make([]chan int, 10)
	for i := 0; i < len(chs); i++ {
		chs[i] = make(chan int)
		// 带缓冲区大小
		// c: = make(chan int, 1024)
		// for i:= range c {
		// }
		go counting(chs[i])
	}

	for _, ch := range chs {
		<-ch
		// channel select
		/*
			select {
			case <-ch:
				// ...
			case ch <- 1:
			}
		*/
	}

	// 单向Channel
	var ch1 chan<- int // 只能写入int
	var ch2 <-chan int // 只能读出int

	// 关闭Channel
	close(ch1)
	_, ok := <-ch2
	if !ok {
		// already closed
	}
}

// 锁
var m sync.Mutex

func lockDemo() {
	m.Lock()
	// do something
	defer m.Unlock()
}

// 全局唯一操作
var once sync.Once

// once.Do(someFunction)

// 7. 网络编程
// import "net"
// net.Dial("tcp", "127.0.0.1:8080")

// 8. json处理
// import "encoding/json"
// json.Marshal(obj) 序列化
// json.Unmarshal() 反序列化

// 9. Web开发
// import "net/http"
// 模板
// import "html/template"

// 10. 常用库
// import "os"
// import "io"
// import "flag"
// import "strconv"
// import "crypto/sha1"
// import "crypto/md5"

// 11. 单元测试
// _test结尾的go文件： xxx_test.go
func testDemo(t *testing.T) {
	r := sum2(2, 3)
	if r != 5 {
		t.Errorf("sum2(2, 3) failed. Got %d, expect 5.", r)
	}
}

// 12. 性能测试
func benchmarkAdd(b *testing.B) {
	b.StopTimer()
	// dosometing
	b.StartTimer()
}

func main() {
	helloWorld()
}
<commit_msg>add some comments<commit_after>/*
GoTips.go:
  Golang速学速查速用代码手册

Source: github.com/coderzh/CodeTips/blob/master/GoTips.go

Author: coderzh(github.com/coderzh)
Blog: http://blog.coderzh.com
参考：《Go语言编程》
*/

package main

import (
	"fmt"
	"log"
	"os"
	"sync"
	// "errors"
	"math"
	"testing"
)

// 0. 注释
/*
规范：
 1. 命名：骆驼命名法（不要用下划线）

命令:
 go get github.com/coderzh/xxx
 go build calc
 go run xxx.go
 go install calc
*/

// 1. Hello World
func helloWorld() {
	fmt.Println("Hello, 世界")
}

// 2.变量类型
func typeDemo() {
	// 变量申明
	var v1 int
	var (
		v2 int
		v3 string
	)
	//var p *int // 指针类型

	// 变量初始化
	var v4 int = 10
	// 等价于:
	var v5 = 10
	// 一般这样就好
	v6 := 10

	// 赋值，多重赋值
	v1 = 10
	v2, v3 = 20, "test"
	// 匿名变量 _
	_, v4 = v5, v6

	fmt.Println(v1, v2, v3, v4)

	// 常量
	const Pi float64 = 3.1415926
	const MaxPlayer = 10

	// 枚举
	const (
		Sunday = iota // iota从0递增
		Mondy
		Tuesday
		// ...
	)

	// 类型
	// 1. 布尔
	var b1 bool
	b1 = true
	b1 = (1 == 2)

	fmt.Println(b1)

	// 2. 整形
	// int8 uint8 int16 uint16 int32 uint32 int64 uint64 int uint uintptr
	var i32 int32
	// 强制转换
	i32 = int32(64)
	// 运算：+, -, *, /, %（求余）
	// 比较：>, <, ==, >=, <=, !=
	// 位运算：x << y, x >> y, x ^ y, x & y, x | y, ^x （取反）

	fmt.Println(i32)

	// 3. 浮点
	// float32, float64
	var f1 float64 = 1.0001
	var f2 float64 = 1.0002
	// 浮点比较
	isEqual := math.Dim(f1, f2) < 0.0001

	fmt.Println(isEqual)

	// 4. 字符串
	var s1 string
	s1 = "abc"
	// 字符串连接
	s1 = s1 + "ddd"
	// 取长度
	n := len(s1)
	// 取字符
	c1 := s1[0]

	fmt.Println(c1)

	// 字节遍历
	for i := 0; i < n; i++ {
		ch := s1[i]
		fmt.Println(ch)
	}
	// Unicode字符遍历
	for i, ch := range s1 {
		fmt.Println(i, ch)
	}

	// 5. 数组
	var arr1 [32]int
	//var arr2 [3][8]int // 二维数组
	// 初始化
	arr1 = [32]int{0}
	array := [5]int{1, 2, 3, 4, 5}
	// 数组遍历
	for i := 0; i < len(array); i++ {
		fmt.Println(array[i])
	}
	for i, v := range arr1 {
		fmt.Println(i, v)
	}
	// 数组是值类型，每次参数传递都是一份拷贝

	// 数组切片Slice
	var mySlice []int = array[:2]
	mySlice1 := make([]int, 5)
	mySlice2 := make([]int, 5, 10)

	fmt.Println("len(mySlice2:", len(mySlice2)) // 5
	fmt.Println("cap(mySlice2:", cap(mySlice2)) // 10

	mySlice3 := append(mySlice, 2, 3, 4)
	mySlice4 := append(mySlice, mySlice1...)

	copy(mySlice3, mySlice4)

	// 6. Map
	var m map[int]string
	m[1] = "ddd"
	m1 := make(map[int]string)
	m2 := map[int]string{
		1: "a",
		2: "b",
	}

	delete(m2, 1)

	value, ok := m1[1]
	if ok {
		fmt.Println(value)
	}

	for k, v := range m2 {
		fmt.Println(k, v)
	}

}

// 3. 流程控制
func flowDemo() {
	// if else
	a := 10
	if a < 10 {
		// ..
	} else {
		// ..
	}

	// switch
	switch a {
	case 0:
		fmt.Println("0")
	case 10:
		fmt.Println("10")
	default:
		fmt.Println("default")
	}

	switch {
	case a < 10:
		fmt.Println("<10")
	case a < 20:
		fmt.Println("<20")
	}

	// 循环
	for i := 0; i < 10; i++ {
	}
	// 无限循环
	sum := 0
	for {
		sum++
		if sum > 10 {
			break
			// 指定break
			// break JLoop
		}
	}

	goto JLoop

JLoop:
	// break to here

}

// 4. 函数
// func 函数名(参数列表)(返回值列表) {
// }
func sum1(value1 int, value2 int) (result int, err error) {
	// err = errors.New("xxxx")
	return value1 + value2, nil
}

func sum2(value1, value2 int) int {
	return value1 + value2
}

// 不定参数
// myFunc(1, 2, 3, 4, 5)
func myFunc(args ...int) {
	for _, arg := range args {
		fmt.Println(arg)
	}
	// 传递
	// myFunc2(args...)
	// myFunc2(args[1:]...)
}

// 任意类型的不定参数
func myPrintf(args ...interface{}) {
	for _, arg := range args {
		switch arg.(type) {
		case int:
			fmt.Println(arg, " is int")
		case string:
			fmt.Println(arg, " is string")
		default:
			fmt.Println(arg, " is unknown")
		}
	}
}

// 匿名函数
func anonymousFunc() {
	f := func(a, b int) int {
		return a + b
	}

	f(1, 2)
}

// defer
func copyFile(path string) {
	f, err := os.Open(path)
	if err != nil {
		return
	}

	defer f.Close()
	// or
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Runtime error caught: %v", r)
		}
	}()
}

// 5. 结构体
type Rect struct {
	// 小写为private
	x, y float64
	// 大写为public
	Width, Height float64
}

// 大写方法为public，小写为private
func (r *Rect) Area() float64 {
	return r.Width * r.Height
}

func netRect(x, y, width, height float64) *Rect {
	// 实例化结构体
	// rect1 := new(Rect)
	// rect2 := &Rect{}
	// rect3 := &Rect{Width:100, Height:200}
	return &Rect{x, y, width, height}
}

// 匿名组合
type Base struct {
	Name string
}

func (base *Base) Foo() {}
func (base *Base) Bar() {}

type Foo struct {
	Base
	*log.Logger
}

func (foo *Foo) Bar() {
	foo.Base.Bar()
	// ...
}

// 非侵入式接口
type IFile interface {
	Read(buf []byte) (n int, err error)
	Write(buf []byte) (n int, err error)
}

type File struct {
}

func (file *File) Read(buf []byte) (n int, err error) {
	return 0, nil
}

func (file *File) Write(buf []byte) (n int, err error) {
	return 0, nil
}

func interfaceDemo() {
	// 只要实现了Read, Write方法即可
	var file IFile = new(File)

	// 接口查询
	// 是否实现了IFile接口
	if file2, ok := file.(IFile); ok {
		file2.Read([]byte{})
	}
	// 实例类型是否是File
	if file3, ok := file.(*File); ok {
		file3.Read([]byte{})
	}

	// 类型查询
	switch v := file.(type) {
	}
}

// 6. 并发编程
func counting(ch chan int) {
	ch <- 1
	fmt.Println("counting")
}

func channelDemo() {
	chs := make([]chan int, 10)
	for i := 0; i < len(chs); i++ {
		chs[i] = make(chan int)
		// 带缓冲区大小
		// c: = make(chan int, 1024)
		// for i:= range c {
		// }
		go counting(chs[i])
	}

	for _, ch := range chs {
		<-ch
		// channel select
		/*
			select {
			case <-ch:
				// ...
			case ch <- 1:
			}
		*/
	}

	// 单向Channel
	var ch1 chan<- int // 只能写入int
	var ch2 <-chan int // 只能读出int

	// 关闭Channel
	close(ch1)
	_, ok := <-ch2
	if !ok {
		// already closed
	}
}

// 锁
var m sync.Mutex

func lockDemo() {
	m.Lock()
	// do something
	defer m.Unlock()
}

// 全局唯一操作
var once sync.Once

// once.Do(someFunction)

// 7. 网络编程
// import "net"
// net.Dial("tcp", "127.0.0.1:8080")

// 8. json处理
// import "encoding/json"
// json.Marshal(obj) 序列化
// json.Unmarshal() 反序列化

// 9. Web开发
// import "net/http"
// 模板
// import "html/template"

// 10. 常用库
// import "os"
// import "io"
// import "flag"
// import "strconv"
// import "crypto/sha1"
// import "crypto/md5"

// 11. 单元测试
// _test结尾的go文件： xxx_test.go
func testDemo(t *testing.T) {
	r := sum2(2, 3)
	if r != 5 {
		t.Errorf("sum2(2, 3) failed. Got %d, expect 5.", r)
	}
}

// 12. 性能测试
func benchmarkAdd(b *testing.B) {
	b.StopTimer()
	// dosometing
	b.StartTimer()
}

func main() {
	helloWorld()
}
<|endoftext|>


<commit_before>package lzma2

import (
	"bytes"
	"fmt"
	"testing"

	"github.com/ulikunitz/xz/lzma"
)

func TestChunkTypeString(t *testing.T) {
	tests := [...]struct {
		c chunkType
		s string
	}{
		{cEOS, "EOS"},
		{cUD, "UD"},
		{cU, "U"},
		{cL, "L"},
		{cLR, "LR"},
		{cLRN, "LRN"},
		{cLRND, "LRND"},
	}
	for _, c := range tests {
		s := fmt.Sprintf("%v", c.c)
		if s != c.s {
			t.Errorf("got %s; want %s", s, c.s)
		}
	}
}

func TestHeaderChunkType(t *testing.T) {
	tests := []struct {
		h byte
		c chunkType
	}{
		{h: 0, c: cEOS},
		{h: 1, c: cUD},
		{h: 2, c: cU},
		{h: 1<<7 | 0x1f, c: cL},
		{h: 1<<7 | 1<<5 | 0x1f, c: cLR},
		{h: 1<<7 | 1<<6 | 0x1f, c: cLRN},
		{h: 1<<7 | 1<<6 | 1<<5 | 0x1f, c: cLRND},
		{h: 1<<7 | 1<<6 | 1<<5, c: cLRND},
	}
	if _, err := headerChunkType(3); err == nil {
		t.Fatalf("headerChunkType(%d) got %v; want %v",
			3, err, errHeaderByte)
	}
	for _, tc := range tests {
		c, err := headerChunkType(tc.h)
		if err != nil {
			t.Fatalf("headerChunkType error %s", err)
		}
		if c != tc.c {
			t.Errorf("got %s; want %s", c, tc.c)
		}
	}
}

func TestHeaderLen(t *testing.T) {
	tests := []struct {
		c chunkType
		n int
	}{
		{cEOS, 1}, {cU, 3}, {cUD, 3}, {cL, 5}, {cLR, 5}, {cLRN, 6},
		{cLRND, 6},
	}
	for _, tc := range tests {
		n := headerLen(tc.c)
		if n != tc.n {
			t.Errorf("header length for %s %d; want %d",
				tc.c, n, tc.n)
		}
	}
}

func chunkHeaderSamples(t *testing.T) []chunkHeader {
	props, err := lzma.NewProperties(3, 0, 2)
	if err != nil {
		t.Fatalf("NewProperties(3, 0, 2) error %s", err)
	}

	headers := make([]chunkHeader, 0, 12)
	for c := cEOS; c <= cLRND; c++ {
		var h chunkHeader
		h.ctype = c
		if c >= cUD {
			h.uncompressed = 0x0304
		}
		if c >= cL {
			h.compressed = 0x0201
		}
		if c >= cLRN {
			h.props = props
		}
		headers = append(headers, h)
	}
	return headers
}

func TestChunkHeaderMarshalling(t *testing.T) {
	for _, h := range chunkHeaderSamples(t) {
		data, err := h.MarshalBinary()
		if err != nil {
			t.Fatalf("MarshalBinary for %v error %s", h, err)
		}
		var g chunkHeader
		if err = g.UnmarshalBinary(data); err != nil {
			t.Fatalf("UnmarshalBinary error %s", err)
		}
		if g != h {
			t.Fatalf("got %v; want %v", g, h)
		}
	}
}

func TestReadChunkHeader(t *testing.T) {
	for _, h := range chunkHeaderSamples(t) {
		data, err := h.MarshalBinary()
		if err != nil {
			t.Fatalf("MarshalBinary for %v error %s", h, err)
		}
		r := bytes.NewReader(data)
		g, err := readChunkHeader(r)
		if err != nil {
			t.Fatalf("readChunkHeader for %v error %s", h, err)
		}
		if *g != h {
			t.Fatalf("got %v; want %v", g, h)
		}
	}
}

func TestWriteEOS(t *testing.T) {
	buf := new(bytes.Buffer)
	err := WriteEOS(buf)
	if err != nil {
		t.Fatalf("WriteEOS: unexpected error %s", err)
	}
	b := buf.Bytes()
	if len(b) != 1 {
		t.Fatalf("len(b) is %d; want %d", len(b), 1)
	}
	if b[0] != 0 {
		t.Fatalf("b[0] is %#4x; want %#4x", b[0], byte(0))
	}
}
<commit_msg>lzma2: added explicit test for reading an EOS chunk header<commit_after>package lzma2

import (
	"bytes"
	"fmt"
	"testing"

	"github.com/ulikunitz/xz/lzma"
)

func TestChunkTypeString(t *testing.T) {
	tests := [...]struct {
		c chunkType
		s string
	}{
		{cEOS, "EOS"},
		{cUD, "UD"},
		{cU, "U"},
		{cL, "L"},
		{cLR, "LR"},
		{cLRN, "LRN"},
		{cLRND, "LRND"},
	}
	for _, c := range tests {
		s := fmt.Sprintf("%v", c.c)
		if s != c.s {
			t.Errorf("got %s; want %s", s, c.s)
		}
	}
}

func TestHeaderChunkType(t *testing.T) {
	tests := []struct {
		h byte
		c chunkType
	}{
		{h: 0, c: cEOS},
		{h: 1, c: cUD},
		{h: 2, c: cU},
		{h: 1<<7 | 0x1f, c: cL},
		{h: 1<<7 | 1<<5 | 0x1f, c: cLR},
		{h: 1<<7 | 1<<6 | 0x1f, c: cLRN},
		{h: 1<<7 | 1<<6 | 1<<5 | 0x1f, c: cLRND},
		{h: 1<<7 | 1<<6 | 1<<5, c: cLRND},
	}
	if _, err := headerChunkType(3); err == nil {
		t.Fatalf("headerChunkType(%d) got %v; want %v",
			3, err, errHeaderByte)
	}
	for _, tc := range tests {
		c, err := headerChunkType(tc.h)
		if err != nil {
			t.Fatalf("headerChunkType error %s", err)
		}
		if c != tc.c {
			t.Errorf("got %s; want %s", c, tc.c)
		}
	}
}

func TestHeaderLen(t *testing.T) {
	tests := []struct {
		c chunkType
		n int
	}{
		{cEOS, 1}, {cU, 3}, {cUD, 3}, {cL, 5}, {cLR, 5}, {cLRN, 6},
		{cLRND, 6},
	}
	for _, tc := range tests {
		n := headerLen(tc.c)
		if n != tc.n {
			t.Errorf("header length for %s %d; want %d",
				tc.c, n, tc.n)
		}
	}
}

func chunkHeaderSamples(t *testing.T) []chunkHeader {
	props, err := lzma.NewProperties(3, 0, 2)
	if err != nil {
		t.Fatalf("NewProperties(3, 0, 2) error %s", err)
	}

	headers := make([]chunkHeader, 0, 12)
	for c := cEOS; c <= cLRND; c++ {
		var h chunkHeader
		h.ctype = c
		if c >= cUD {
			h.uncompressed = 0x0304
		}
		if c >= cL {
			h.compressed = 0x0201
		}
		if c >= cLRN {
			h.props = props
		}
		headers = append(headers, h)
	}
	return headers
}

func TestChunkHeaderMarshalling(t *testing.T) {
	for _, h := range chunkHeaderSamples(t) {
		data, err := h.MarshalBinary()
		if err != nil {
			t.Fatalf("MarshalBinary for %v error %s", h, err)
		}
		var g chunkHeader
		if err = g.UnmarshalBinary(data); err != nil {
			t.Fatalf("UnmarshalBinary error %s", err)
		}
		if g != h {
			t.Fatalf("got %v; want %v", g, h)
		}
	}
}

func TestReadChunkHeader(t *testing.T) {
	for _, h := range chunkHeaderSamples(t) {
		data, err := h.MarshalBinary()
		if err != nil {
			t.Fatalf("MarshalBinary for %v error %s", h, err)
		}
		r := bytes.NewReader(data)
		g, err := readChunkHeader(r)
		if err != nil {
			t.Fatalf("readChunkHeader for %v error %s", h, err)
		}
		if *g != h {
			t.Fatalf("got %v; want %v", g, h)
		}
	}
}

func TestWriteEOS(t *testing.T) {
	buf := new(bytes.Buffer)
	err := WriteEOS(buf)
	if err != nil {
		t.Fatalf("WriteEOS: unexpected error %s", err)
	}
	b := buf.Bytes()
	if len(b) != 1 {
		t.Fatalf("len(b) is %d; want %d", len(b), 1)
	}
	if b[0] != 0 {
		t.Fatalf("b[0] is %#4x; want %#4x", b[0], byte(0))
	}
}

func TestReadEOS(t *testing.T) {
	var b [1]byte
	r := bytes.NewReader(b[:])
	h, err := readChunkHeader(r)
	if err != nil {
		t.Fatalf("readChunkHeader error %s", err)
	}
	if h.ctype != cEOS {
		t.Errorf("ctype got %s; want %s", h.ctype, cEOS)
	}
	if h.compressed != 0 {
		t.Errorf("compressed got %d; want %d", h.compressed, 0)
	}
	if h.uncompressed != 0 {
		t.Errorf("uncompressed got %d; want %d", h.uncompressed, 0)
	}
	if h.props != 0 {
		t.Errorf("props got %d; want %d", h.props, 0)
	}
}
<|endoftext|>


<commit_before>package api

import (
	"context"
	"encoding/json"
	"net/http"

	"github.com/go-chi/chi"
	"github.com/gobuffalo/uuid"
	"github.com/netlify/gotrue/conf"
	"github.com/netlify/gotrue/models"
	"github.com/pkg/errors"
)

func (a *API) loadInstance(w http.ResponseWriter, r *http.Request) (context.Context, error) {
	instanceID, err := uuid.FromString(chi.URLParam(r, "instance_id"))
	if err != nil {
		return nil, badRequestError("Invalid instance ID")
	}
	logEntrySetField(r, "instance_id", instanceID)

	i, err := models.GetInstance(a.db, instanceID)
	if err != nil {
		if models.IsNotFoundError(err) {
			return nil, notFoundError("Instance not found")
		}
		return nil, internalServerError("Database error loading instance").WithInternalError(err)
	}

	return withInstance(r.Context(), i), nil
}

func (a *API) GetAppManifest(w http.ResponseWriter, r *http.Request) error {
	// TODO update to real manifest
	return sendJSON(w, http.StatusOK, map[string]string{
		"version":     a.version,
		"name":        "GoTrue",
		"description": "GoTrue is a user registration and authentication API",
	})
}

type InstanceRequestParams struct {
	UUID       uuid.UUID           `json:"uuid"`
	BaseConfig *conf.Configuration `json:"config"`
}

type InstanceResponse struct {
	models.Instance
	Endpoint string `json:"endpoint"`
	State    string `json:"state"`
}

func (a *API) CreateInstance(w http.ResponseWriter, r *http.Request) error {
	params := InstanceRequestParams{}
	if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
		return badRequestError("Error decoding params: %v", err)
	}

	_, err := models.GetInstanceByUUID(a.db, params.UUID)
	if err != nil {
		if !models.IsNotFoundError(err) {
			return internalServerError("Database error looking up instance").WithInternalError(err)
		}
	} else {
		return badRequestError("An instance with that UUID already exists")
	}

	id, err := uuid.NewV4()
	if err != nil {
		return errors.Wrap(err, "Error generating id")
	}

	i := models.Instance{
		ID:         id,
		UUID:       params.UUID,
		BaseConfig: params.BaseConfig,
	}
	if err = a.db.Create(&i); err != nil {
		return internalServerError("Database error creating instance").WithInternalError(err)
	}

	resp := InstanceResponse{
		Instance: i,
		Endpoint: a.config.API.Endpoint,
		State:    "active",
	}
	return sendJSON(w, http.StatusCreated, resp)
}

func (a *API) GetInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())
	if i.BaseConfig != nil {
		i.BaseConfig.SMTP.Pass = ""
	}
	return sendJSON(w, http.StatusOK, i)
}

func (a *API) UpdateInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())

	params := InstanceRequestParams{}
	if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
		return badRequestError("Error decoding params: %v", err)
	}

	if params.BaseConfig != nil {
		if err := i.UpdateConfig(a.db, params.BaseConfig); err != nil {
			return internalServerError("Database error updating instance").WithInternalError(err)
		}
	}
	return sendJSON(w, http.StatusOK, i)
}

func (a *API) DeleteInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())
	if err := models.DeleteInstance(a.db, i); err != nil {
		return internalServerError("Database error deleting instance").WithInternalError(err)
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}
<commit_msg>Hide pass in other responses<commit_after>package api

import (
	"context"
	"encoding/json"
	"net/http"

	"github.com/go-chi/chi"
	"github.com/gobuffalo/uuid"
	"github.com/netlify/gotrue/conf"
	"github.com/netlify/gotrue/models"
	"github.com/pkg/errors"
)

func (a *API) loadInstance(w http.ResponseWriter, r *http.Request) (context.Context, error) {
	instanceID, err := uuid.FromString(chi.URLParam(r, "instance_id"))
	if err != nil {
		return nil, badRequestError("Invalid instance ID")
	}
	logEntrySetField(r, "instance_id", instanceID)

	i, err := models.GetInstance(a.db, instanceID)
	if err != nil {
		if models.IsNotFoundError(err) {
			return nil, notFoundError("Instance not found")
		}
		return nil, internalServerError("Database error loading instance").WithInternalError(err)
	}

	return withInstance(r.Context(), i), nil
}

func (a *API) GetAppManifest(w http.ResponseWriter, r *http.Request) error {
	// TODO update to real manifest
	return sendJSON(w, http.StatusOK, map[string]string{
		"version":     a.version,
		"name":        "GoTrue",
		"description": "GoTrue is a user registration and authentication API",
	})
}

type InstanceRequestParams struct {
	UUID       uuid.UUID           `json:"uuid"`
	BaseConfig *conf.Configuration `json:"config"`
}

type InstanceResponse struct {
	models.Instance
	Endpoint string `json:"endpoint"`
	State    string `json:"state"`
}

func (a *API) CreateInstance(w http.ResponseWriter, r *http.Request) error {
	params := InstanceRequestParams{}
	if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
		return badRequestError("Error decoding params: %v", err)
	}

	_, err := models.GetInstanceByUUID(a.db, params.UUID)
	if err != nil {
		if !models.IsNotFoundError(err) {
			return internalServerError("Database error looking up instance").WithInternalError(err)
		}
	} else {
		return badRequestError("An instance with that UUID already exists")
	}

	id, err := uuid.NewV4()
	if err != nil {
		return errors.Wrap(err, "Error generating id")
	}

	i := models.Instance{
		ID:         id,
		UUID:       params.UUID,
		BaseConfig: params.BaseConfig,
	}
	if err = a.db.Create(&i); err != nil {
		return internalServerError("Database error creating instance").WithInternalError(err)
	}

	// hide pass in response
	if i.BaseConfig != nil {
		i.BaseConfig.SMTP.Pass = ""
	}

	resp := InstanceResponse{
		Instance: i,
		Endpoint: a.config.API.Endpoint,
		State:    "active",
	}
	return sendJSON(w, http.StatusCreated, resp)
}

func (a *API) GetInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())
	if i.BaseConfig != nil {
		i.BaseConfig.SMTP.Pass = ""
	}
	return sendJSON(w, http.StatusOK, i)
}

func (a *API) UpdateInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())

	params := InstanceRequestParams{}
	if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
		return badRequestError("Error decoding params: %v", err)
	}

	if params.BaseConfig != nil {
		if err := i.UpdateConfig(a.db, params.BaseConfig); err != nil {
			return internalServerError("Database error updating instance").WithInternalError(err)
		}
		i.BaseConfig.SMTP.Pass = "" // hide from response
	}
	return sendJSON(w, http.StatusOK, i)
}

func (a *API) DeleteInstance(w http.ResponseWriter, r *http.Request) error {
	i := getInstance(r.Context())
	if err := models.DeleteInstance(a.db, i); err != nil {
		return internalServerError("Database error deleting instance").WithInternalError(err)
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Cristian Filipov. All Rights Reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Utility for sending push notifications using the Apple's Push
Notification System (APNs) Go library.
*/
package main

import (
	"crypto/tls"
	"encoding/hex"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/cfilipov/apns"
	"net"
	"os"
	"time"
)

var tokens []string
var customGateway = flag.String("apn-gateway", "", "A custom APNs gateway (for testing or proxy)")
var tcpDelay = flag.Bool("tcp-delay", false, "Determines weather to delay TCP packet until it's full")
var verbose = flag.Bool("v", false, "Verbose output")
var expiry = flag.Int("expiry", 0, "UNIX date in seconds (UTC) that identifies when the notification can be discarded")
var repeat = flag.Int("repeat", 1, "Number of times this notification should be sent.")
var keyFile = flag.String("key", "apns-key.pem", "X.509 private key in pem (Privacy Enhanced Mail) format")
var cerFile	= flag.String("cer", "apns-cer.pem", "X.509 certificate in pem (Privacy Enhanced Mail) format")
var pemFile = flag.String("pem", "apns.pem", "X.509 certificate/key pair stored in a pem file")
var sandbox = flag.Bool("sandbox", false, "Indicates the push notification should use the sandbox environment")
var badge = flag.String("badge", "", "Badge value to use in payload")
var sound =	flag.String("sound", "", "Notification sound key")
var text = flag.String("text", "", "Text to send as an APN alert")
var ttl = flag.Int("ttl", 0, "Time-to-live, in seconds. Signifies how long to wait before the notification can be discarded by APNs. Differs from --expiry in that --expiry requires an actual UNIX time stamp. If both flags are provided, expiry takes precedence.")

func init() {
	flag.Parse()

	tokens = make([]string, 0)
	for _, a := range flag.Args() {
		tokens = append(tokens, a)
	}

	flag.Usage = func() {
		fmt.Println("apnsend - Push notification sending utility for Apple's Push Notification system (APNs)\n")
		fmt.Fprintf(os.Stderr, "Usage: apnsend [OPTIONS] token... \n")
		flag.PrintDefaults()
		fmt.Println("\nTo convert a pkcs#12 (.p12) certificate+key pair to pem, use opensll:")
		fmt.Println("\topenssl pkcs12 -in CertificateName.p12 -out CertificateName.pem -nodes")
	}
}

func main() {
	// At least one token is required.
	if len(tokens) == 0 {
		IfErrExitWithUsagePrompt(errors.New("No notifications to send. You must specify at least one push token."))
	}

	// Load certificate.
	var err error
	var cert tls.Certificate
	switch {
	case *pemFile != "":
		cert, err = apns.LoadPemFile(*pemFile)
	case *cerFile != "" && *keyFile != "":
		cert, err = tls.LoadX509KeyPair(*cerFile, *keyFile)
	}

	IfErrExitWithUsagePrompt(err)

	// Setup a secure connection to an APNs server.
	var conn net.Conn
	switch {
	case *sandbox:
		verbosePrintf("Using sandbox environment.\n")
		conn, err = apns.DialAPN(&cert, apns.SANDBOX, false)
	case *customGateway != "":
		verbosePrintf("Using custom gateway: %s\n", *customGateway)
		conn, err = apns.Dial(&cert, *customGateway, false)
	default:
		verbosePrintf("Using production environment.\n")
		conn, err = apns.DialAPN(&cert, apns.DISTRIBUTION, false)
	}

	IfErrExit(err)
	defer conn.Close()

	// Listen for errors.
	go func() {
		for {
			p, err := apns.ReadCommand(conn)
			IfErrExit(err)
			IfErrExit(fmt.Errorf("%s", p))
		}
	}()

	// JSON payload.
	payload := make(map[string]interface{})
	aps := map[string]string{}
	aps["alert"] = *text
	aps["badge"] = *badge
	aps["sound"] = *sound
	payload["aps"] = aps
	jsonPayload, err := json.Marshal(payload)
	IfErrExit(err)

	// Expiry = Specific DateTime, TTL = Length of Time
	var expiryTime uint32
	switch {
	case *expiry != 0:
		expiryTime = uint32(*expiry)
	case *ttl != 0:
		unixTime := uint32(time.Now().Unix())
		expiryTime = unixTime + uint32(*ttl)
	}

	verbosePrintf("%d tokens to send.\n", len(tokens))

	var notification apns.Notification
	for i := *repeat; i > 0; i-- {
		for _, t := range tokens {
			token, err := hex.DecodeString(t)
			IfErrExit(err)

			// Expiry is only available in enhanced notifications.
			if expiryTime != 0 {
				notification = &apns.EnhancedNotification{
					Identifier:    1,
					Expiry:        expiryTime,
					TokenLength:   uint16(len(token)),
					DeviceToken:   token,
					PayloadLength: uint16(len(jsonPayload)),
					Payload:       jsonPayload,
				}
			} else {
				notification = &apns.SimpleNotification{
					TokenLength:   uint16(len(token)),
					DeviceToken:   token,
					PayloadLength: uint16(len(jsonPayload)),
					Payload:       jsonPayload,
				}
			}

			// Send the notification.
			verbosePrintf("Sending %d of %d: %s\n", (*repeat - i + 1), *repeat, notification)
			notification.WriteTo(conn)
		}
	}

	// Wait for a short time before quitting to give APNs a chance to
	// return error responses, if any.
	time.Sleep(5000 * time.Millisecond)
	return
}

func IfErrExit(err error) {
	if err != nil {
		fmt.Printf("\nERROR: %s\n", err)
		os.Exit(1)
	}
}

func IfErrExitWithUsagePrompt(err error) {
	if err != nil {
		fmt.Printf("ERROR: %s\n\nUsage: apnsend [OPTIONS] token... \n", err)
		flag.PrintDefaults()
		os.Exit(1)
	}
}

func verbosePrintf(format string, a ...interface{}) (n int, err error) {
	if *verbose {
		return fmt.Printf(format, a...)
	}
	return 0, nil
}
<commit_msg>Support raw JSON payload option.<commit_after>// Copyright (c) 2013 Cristian Filipov. All Rights Reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Utility for sending push notifications using the Apple's Push
Notification System (APNs) Go library.
*/
package main

import (
	"crypto/tls"
	"encoding/hex"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/cfilipov/apns"
	"net"
	"os"
	"time"
)

var tokens []string
var customGateway = flag.String("apn-gateway", "", "A custom APNs gateway (for testing or proxy)")
var tcpDelay = flag.Bool("tcp-delay", false, "Determines weather to delay TCP packet until it's full")
var verbose = flag.Bool("v", false, "Verbose output")
var expiry = flag.Int("expiry", 0, "UNIX date in seconds (UTC) that identifies when the notification can be discarded")
var repeat = flag.Int("repeat", 1, "Number of times this notification should be sent.")
var keyFile = flag.String("key", "apns-key.pem", "X.509 private key in pem (Privacy Enhanced Mail) format")
var cerFile	= flag.String("cer", "apns-cer.pem", "X.509 certificate in pem (Privacy Enhanced Mail) format")
var pemFile = flag.String("pem", "apns.pem", "X.509 certificate/key pair stored in a pem file")
var sandbox = flag.Bool("sandbox", false, "Indicates the push notification should use the sandbox environment")
var badge = flag.String("badge", "", "Badge value to use in payload")
var sound =	flag.String("sound", "", "Notification sound key")
var text = flag.String("text", "", "Text to send as an APN alert")
var payloadStr = flag.String("payload", "", "Raw (JSON) payload to send. This option overrides -text -badge and -sound options.")
var ttl = flag.Int("ttl", 0, "Time-to-live, in seconds. Signifies how long to wait before the notification can be discarded by APNs. Differs from --expiry in that --expiry requires an actual UNIX time stamp. If both flags are provided, expiry takes precedence.")

func init() {
	flag.Parse()

	tokens = make([]string, 0)
	for _, a := range flag.Args() {
		tokens = append(tokens, a)
	}

	flag.Usage = func() {
		fmt.Println("apnsend - Push notification sending utility for Apple's Push Notification system (APNs)\n")
		fmt.Fprintf(os.Stderr, "Usage: apnsend [OPTIONS] token... \n")
		flag.PrintDefaults()
		fmt.Println("\nTo convert a pkcs#12 (.p12) certificate+key pair to pem, use opensll:")
		fmt.Println("\topenssl pkcs12 -in CertificateName.p12 -out CertificateName.pem -nodes")
	}
}

func main() {
	// At least one token is required.
	if len(tokens) == 0 {
		IfErrExitWithUsagePrompt(errors.New("No notifications to send. You must specify at least one push token."))
	}

	// Load certificate.
	var err error
	var cert tls.Certificate
	switch {
	case *pemFile != "":
		cert, err = apns.LoadPemFile(*pemFile)
	case *cerFile != "" && *keyFile != "":
		cert, err = tls.LoadX509KeyPair(*cerFile, *keyFile)
	}

	IfErrExitWithUsagePrompt(err)

	// Setup a secure connection to an APNs server.
	var conn net.Conn
	switch {
	case *sandbox:
		verbosePrintf("Using sandbox environment.\n")
		conn, err = apns.DialAPN(&cert, apns.SANDBOX, false)
	case *customGateway != "":
		verbosePrintf("Using custom gateway: %s\n", *customGateway)
		conn, err = apns.Dial(&cert, *customGateway, false)
	default:
		verbosePrintf("Using production environment.\n")
		conn, err = apns.DialAPN(&cert, apns.DISTRIBUTION, false)
	}

	IfErrExit(err)
	defer conn.Close()

	// Listen for errors.
	go func() {
		for {
			p, err := apns.ReadCommand(conn)
			IfErrExit(err)
			IfErrExit(fmt.Errorf("%s", p))
		}
	}()

	var jsonPayload []byte

	if len(*payloadStr) == 0 {
		payload := make(map[string]interface{})
		aps := map[string]string{}
		aps["alert"] = *text
		aps["badge"] = *badge
		aps["sound"] = *sound
		payload["aps"] = aps
		jsonPayload, err = json.Marshal(payload)
		IfErrExit(err)
	} else {
		jsonPayload = []byte(*payloadStr)
	}

	// Expiry = Specific DateTime, TTL = Length of Time
	var expiryTime uint32
	switch {
	case *expiry != 0:
		expiryTime = uint32(*expiry)
	case *ttl != 0:
		unixTime := uint32(time.Now().Unix())
		expiryTime = unixTime + uint32(*ttl)
	}

	verbosePrintf("%d tokens to send.\n", len(tokens))

	var notification apns.Notification
	for i := *repeat; i > 0; i-- {
		for _, t := range tokens {
			token, err := hex.DecodeString(t)
			IfErrExit(err)

			// Expiry is only available in enhanced notifications.
			if expiryTime != 0 {
				notification = &apns.EnhancedNotification{
					Identifier:    1,
					Expiry:        expiryTime,
					TokenLength:   uint16(len(token)),
					DeviceToken:   token,
					PayloadLength: uint16(len(jsonPayload)),
					Payload:       jsonPayload,
				}
			} else {
				notification = &apns.SimpleNotification{
					TokenLength:   uint16(len(token)),
					DeviceToken:   token,
					PayloadLength: uint16(len(jsonPayload)),
					Payload:       jsonPayload,
				}
			}

			// Send the notification.
			verbosePrintf("Sending %d of %d: %s\n", (*repeat - i + 1), *repeat, notification)
			notification.WriteTo(conn)
		}
	}

	// Wait for a short time before quitting to give APNs a chance to
	// return error responses, if any.
	time.Sleep(5000 * time.Millisecond)
	return
}

func IfErrExit(err error) {
	if err != nil {
		fmt.Printf("\nERROR: %s\n", err)
		os.Exit(1)
	}
}

func IfErrExitWithUsagePrompt(err error) {
	if err != nil {
		fmt.Printf("ERROR: %s\n\nUsage: apnsend [OPTIONS] token... \n", err)
		flag.PrintDefaults()
		os.Exit(1)
	}
}

func verbosePrintf(format string, a ...interface{}) (n int, err error) {
	if *verbose {
		return fmt.Printf(format, a...)
	}
	return 0, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package search

import (
	"encoding/json"
	"fmt"
	//"log"
	"strings"
)

// QueryDsl creates a new Query Dsl
func Query() *QueryDsl {
	return &QueryDsl{}
}

/*

some ways to serialize
"query": {
	"filtered": {
	  "query": {
	    "query_string": {
	      "default_operator": "OR",
	      "default_field": "_all",
	      "query": " actor:\"bob\"  AND type:\"EventType\""
	    }
	  },
	  "filter": {
	    "range": {
	      "@timestamp": {
	        "from": "2012-12-29T16:52:48+00:00",
	        "to": "2012-12-29T17:52:48+00:00"
	      }
	    }
	  }
	}
},

"query" : {
    "term" : { "user" : "kimchy" }
}

"query" : {
    "match_all" : {}
},
*/
type QueryDsl struct {
	QueryEmbed
	FilterVal *FilterOp `json:"filter,omitempty"`
}

// The core Query Syntax can be embedded as a child of a variety of different parents
type QueryEmbed struct {
	MatchAll *MatchAll         `json:"match_all,omitempty"`
	Terms    map[string]string `json:"term,omitempty"`
	Qs       *QueryString      `json:"query_string,omitempty"`
	//Exist    string            `json:"_exists_,omitempty"`
}

// MarshalJSON provides custom marshalling to support the query dsl which is a conditional
// json format, not always the same parent/children
func (qd *QueryDsl) MarshalJSON() ([]byte, error) {
	q := qd.QueryEmbed
	hasQuery := false
	if q.Qs != nil || len(q.Terms) > 0 || q.MatchAll != nil {
		hasQuery = true
	}
	// If a query has a
	if qd.FilterVal != nil && hasQuery {
		queryB, err := json.Marshal(q)
		if err != nil {
			return queryB, err
		}
		filterB, err := json.Marshal(qd.FilterVal)
		if err != nil {
			return filterB, err
		}
		return []byte(fmt.Sprintf(`{"filtered":{"query":%s,"filter":%s}}`, queryB, filterB)), nil
	}
	return json.Marshal(q)
}

// get all
func (q *QueryDsl) All() *QueryDsl {
	q.MatchAll = &MatchAll{""}
	return q
}

// Limit the query to this range
func (q *QueryDsl) Range(fop *FilterOp) *QueryDsl {
	if q.FilterVal == nil {
		q.FilterVal = fop
		return q
	}
	// TODO:  this is not valid, refactor
	q.FilterVal.Add(fop)
	return q
}

// Add a term search for a specific field
//    Term("user","kimchy")
func (q *QueryDsl) Term(name, value string) *QueryDsl {
	if len(q.Terms) == 0 {
		q.Terms = make(map[string]string)
	}
	q.Terms[name] = value
	return q
}

// The raw search strings (lucene valid)
func (q *QueryDsl) Search(searchFor string) *QueryDsl {
	//I don't think this is right, it is not a filter.query, it should be q query?
	qs := NewQueryString("", "")
	q.QueryEmbed.Qs = &qs
	q.QueryEmbed.Qs.Query = searchFor
	return q
}

// Querystring operations
func (q *QueryDsl) Qs(qs *QueryString) *QueryDsl {
	q.QueryEmbed.Qs = qs
	return q
}

// Fields in query_string search
//     Fields("fieldname","search_for","","")
//
//     Fields("fieldname,field2,field3","search_for","","")
//
//     Fields("fieldname,field2,field3","search_for","field_exists","")
func (q *QueryDsl) Fields(fields, search, exists, missing string) *QueryDsl {
	fieldList := strings.Split(fields, ",")
	qs := NewQueryString("", "")
	q.QueryEmbed.Qs = &qs
	q.QueryEmbed.Qs.Query = search
	if len(fieldList) == 1 {
		q.QueryEmbed.Qs.DefaultField = fields
	} else {
		q.QueryEmbed.Qs.Fields = fieldList
	}
	q.QueryEmbed.Qs.Exists = exists
	q.QueryEmbed.Qs.Missing = missing
	return q
}

// Filter this query
func (q *QueryDsl) Filter(f *FilterOp) *QueryDsl {
	q.FilterVal = f
	return q
}

type MatchAll struct {
	All string `json:"-"`
}

// should we reuse QueryDsl here?
type QueryWrap struct {
	Qs QueryString `json:"query_string,omitempty"`
}

// QueryString based search
func NewQueryString(field, query string) QueryString {
	return QueryString{"", field, query, "", "", nil}
}

type QueryString struct {
	DefaultOperator string   `json:"default_operator,omitempty"`
	DefaultField    string   `json:"default_field,omitempty"`
	Query           string   `json:"query,omitempty"`
	Exists          string   `json:"_exists_,omitempty"`
	Missing         string   `json:"_missing_,omitempty"`
	Fields          []string `json:"fields,omitempty"`
	//_exists_:field1,
	//_missing_:field1,
}

// Generic Term based (used in query, facet, filter)
type Term struct {
	Terms     Terms       `json:"terms,omitempty"`
	FilterVal *FilterWrap `json:"facet_filter,omitempty"`
}

type Terms struct {
	Fields []string `json:"field,omitempty"`
	Size   string   `json:"size,omitempty"`
	Regex  string   `json:"regex,omitempty"`
}

func NewTerm(fields ...string) *Term {
	m := &Term{Terms{Fields: fields}, nil}
	return m
}

func (s *Term) Filter(fl ...interface{}) *Term {
	if s.FilterVal == nil {
		s.FilterVal = NewFilterWrap()
	}

	s.FilterVal.addFilters(fl)
	return s
}

// Custom marshalling
func (t *Terms) MarshalJSON() ([]byte, error) {
	m := make(map[string]interface{})
	// TODO:  this isn't getting called!?
	if len(t.Fields) == 1 {
		m["field"] = t.Fields[0]
	} else if len(t.Fields) > 1 {
		m["fields"] = t.Fields
	}
	if len(t.Regex) > 0 {
		m["regex"] = t.Regex
	}
	if len(t.Size) > 0 {
		m["size"] = t.Size
	}
	return json.Marshal(m)
}
<commit_msg>Add support for multi match to query dsl<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package search

import (
	"encoding/json"
	"fmt"
	//"log"
	"strings"
)

// QueryDsl creates a new Query Dsl
func Query() *QueryDsl {
	return &QueryDsl{}
}

/*

some ways to serialize
"query": {
	"filtered": {
	  "query": {
	    "query_string": {
	      "default_operator": "OR",
	      "default_field": "_all",
	      "query": " actor:\"bob\"  AND type:\"EventType\""
	    }
	  },
	  "filter": {
	    "range": {
	      "@timestamp": {
	        "from": "2012-12-29T16:52:48+00:00",
	        "to": "2012-12-29T17:52:48+00:00"
	      }
	    }
	  }
	}
},

"query" : {
    "term" : { "user" : "kimchy" }
}

"query" : {
    "match_all" : {}
},
*/
type QueryDsl struct {
	QueryEmbed
	FilterVal *FilterOp `json:"filter,omitempty"`
}

// The core Query Syntax can be embedded as a child of a variety of different parents
type QueryEmbed struct {
	MatchAll   *MatchAll         `json:"match_all,omitempty"`
	Terms      map[string]string `json:"term,omitempty"`
	Qs         *QueryString      `json:"query_string,omitempty"`
	MultiMatch *MultiMatch       `json:"multi_match,omitempty"`
	//Exist    string            `json:"_exists_,omitempty"`
}

// MarshalJSON provides custom marshalling to support the query dsl which is a conditional
// json format, not always the same parent/children
func (qd *QueryDsl) MarshalJSON() ([]byte, error) {
	q := qd.QueryEmbed
	hasQuery := false
	if q.Qs != nil || len(q.Terms) > 0 || q.MatchAll != nil || q.MultiMatch != nil {
		hasQuery = true
	}
	// If a query has a
	if qd.FilterVal != nil && hasQuery {
		queryB, err := json.Marshal(q)
		if err != nil {
			return queryB, err
		}
		filterB, err := json.Marshal(qd.FilterVal)
		if err != nil {
			return filterB, err
		}
		return []byte(fmt.Sprintf(`{"filtered":{"query":%s,"filter":%s}}`, queryB, filterB)), nil
	}
	return json.Marshal(q)
}

// get all
func (q *QueryDsl) All() *QueryDsl {
	q.MatchAll = &MatchAll{""}
	return q
}

// Limit the query to this range
func (q *QueryDsl) Range(fop *FilterOp) *QueryDsl {
	if q.FilterVal == nil {
		q.FilterVal = fop
		return q
	}
	// TODO:  this is not valid, refactor
	q.FilterVal.Add(fop)
	return q
}

// Add a term search for a specific field
//    Term("user","kimchy")
func (q *QueryDsl) Term(name, value string) *QueryDsl {
	if len(q.Terms) == 0 {
		q.Terms = make(map[string]string)
	}
	q.Terms[name] = value
	return q
}

// The raw search strings (lucene valid)
func (q *QueryDsl) Search(searchFor string) *QueryDsl {
	//I don't think this is right, it is not a filter.query, it should be q query?
	qs := NewQueryString("", "")
	q.QueryEmbed.Qs = &qs
	q.QueryEmbed.Qs.Query = searchFor
	return q
}

// Querystring operations
func (q *QueryDsl) Qs(qs *QueryString) *QueryDsl {
	q.QueryEmbed.Qs = qs
	return q
}

// Fields in query_string search
//     Fields("fieldname","search_for","","")
//
//     Fields("fieldname,field2,field3","search_for","","")
//
//     Fields("fieldname,field2,field3","search_for","field_exists","")
func (q *QueryDsl) Fields(fields, search, exists, missing string) *QueryDsl {
	fieldList := strings.Split(fields, ",")
	qs := NewQueryString("", "")
	q.QueryEmbed.Qs = &qs
	q.QueryEmbed.Qs.Query = search
	if len(fieldList) == 1 {
		q.QueryEmbed.Qs.DefaultField = fields
	} else {
		q.QueryEmbed.Qs.Fields = fieldList
	}
	q.QueryEmbed.Qs.Exists = exists
	q.QueryEmbed.Qs.Missing = missing
	return q
}

// Filter this query
func (q *QueryDsl) Filter(f *FilterOp) *QueryDsl {
	q.FilterVal = f
	return q
}

func (q *QueryDsl) MultiMatch(s string, fields []string) *QueryDsl {
	mm := MultiMatch{Query: s, Fields: fields}
	q.QueryEmbed.MultiMatch = &mm
	return q
}

type MatchAll struct {
	All string `json:"-"`
}

type MultiMatch struct {
	Query  string   `json:"query"`
	Fields []string `json:"fields"`
}

// should we reuse QueryDsl here?
type QueryWrap struct {
	Qs QueryString `json:"query_string,omitempty"`
}

// QueryString based search
func NewQueryString(field, query string) QueryString {
	return QueryString{"", field, query, "", "", nil}
}

type QueryString struct {
	DefaultOperator string   `json:"default_operator,omitempty"`
	DefaultField    string   `json:"default_field,omitempty"`
	Query           string   `json:"query,omitempty"`
	Exists          string   `json:"_exists_,omitempty"`
	Missing         string   `json:"_missing_,omitempty"`
	Fields          []string `json:"fields,omitempty"`
	//_exists_:field1,
	//_missing_:field1,
}

// Generic Term based (used in query, facet, filter)
type Term struct {
	Terms     Terms       `json:"terms,omitempty"`
	FilterVal *FilterWrap `json:"facet_filter,omitempty"`
}

type Terms struct {
	Fields []string `json:"field,omitempty"`
	Size   string   `json:"size,omitempty"`
	Regex  string   `json:"regex,omitempty"`
}

func NewTerm(fields ...string) *Term {
	m := &Term{Terms{Fields: fields}, nil}
	return m
}

func (s *Term) Filter(fl ...interface{}) *Term {
	if s.FilterVal == nil {
		s.FilterVal = NewFilterWrap()
	}

	s.FilterVal.addFilters(fl)
	return s
}

// Custom marshalling
func (t *Terms) MarshalJSON() ([]byte, error) {
	m := make(map[string]interface{})
	// TODO:  this isn't getting called!?
	if len(t.Fields) == 1 {
		m["field"] = t.Fields[0]
	} else if len(t.Fields) > 1 {
		m["fields"] = t.Fields
	}
	if len(t.Regex) > 0 {
		m["regex"] = t.Regex
	}
	if len(t.Size) > 0 {
		m["size"] = t.Size
	}
	return json.Marshal(m)
}
<|endoftext|>