<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pzse

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"time"

	"github.com/venicegeo/pzsvc-lib"
)

type inpStruct struct {
	Command    string   `json:"cmd"`
	InPzFiles  []string `json:"inPzFiles"`    // slice: Pz dataIds
	InExtFiles []string `json:"inExtFiles"`   // slice: external URL
	InPzNames  []string `json:"inPzNames"`    // slice: name for the InPzFile of the same index
	InExtNames []string `json:"inExtNames"`   // slice: name for the InExtFile of the same index
	OutTiffs   []string `json:"outTiffs"`     // slice: filenames of GeoTIFFs to be ingested
	OutTxts    []string `json:"outTxts"`      // slice: filenames of text files to be ingested
	OutGeoJs   []string `json:"outGeoJson"`   // slice: filenames of GeoJSON files to be ingested
	ExtAuth    string   `json:"inExtAuthKey"` // string: auth key for accessing external files
	PzAuth     string   `json:"pzAuthKey"`    // string: suth key for accessing Piazza
}

// ParseConfig parses the config file on starting up
func ParseConfig(configObj *ConfigType) ConfigParseOut {

	canReg, canFile, hasAuth := CheckConfig(configObj)

	var authKey string
	if hasAuth {
		authKey = os.Getenv(configObj.AuthEnVar)
		if authKey == "" {
			fmt.Println("Error: no auth key at AuthEnVar.  Registration disabled, and client will have to provide authKey.")
			hasAuth = false
			canReg = false
		}
	}

	if configObj.Port <= 0 {
		configObj.Port = 8080
	}
	portStr := ":" + strconv.Itoa(configObj.Port)

	version := GetVersion(configObj)

	if canReg {
		fmt.Println("About to manage registration.")
		err := pzsvc.ManageRegistration(configObj.SvcName,
			configObj.Description,
			configObj.URL+"/execute",
			configObj.PzAddr,
			version,
			authKey,
			configObj.Attributes)
		if err != nil {
			fmt.Println("pzsvc-exec error in managing registration: ", err.Error())
		}
		fmt.Println("Registration managed.")
	}

	var procPool = pzsvc.Semaphore(nil)
	if configObj.NumProcs > 0 {
		procPool = make(pzsvc.Semaphore, configObj.NumProcs)
	}

	return ConfigParseOut{authKey, portStr, version, canFile, procPool}
}

// Execute does the primary work for pzsvc-exec.  Given a request and various
// blocks of config data, it creates a temporary folder to work in, downloads
// any files indicated in the request (if the configs support it), executes
// the command indicated by the combination of request and configs, uploads
// any files indicated by the request (if the configs support it) and cleans
// up after itself
func Execute(w http.ResponseWriter, r *http.Request, configObj ConfigType, pzAuth, version string, canFile bool, procPool pzsvc.Semaphore) OutStruct {

	// Makes sure that you only have a certain number of execution tasks firing at once.
	// pzsvc-exec calls can get pretty resource-intensive, and this keeps them from
	// trampling each other into messy deadlock
	procPool.Lock()
	defer procPool.Unlock()

	var output OutStruct
	output.InFiles = make(map[string]string)
	output.OutFiles = make(map[string]string)
	output.HTTPStatus = http.StatusOK

	if r.Method != "POST" {
		handleError(&output, "", fmt.Errorf(r.Method+" not supported.  Please us POST."), w, http.StatusMethodNotAllowed)
		return output
	}

	var inpObj inpStruct

	if _, err := pzsvc.ReadBodyJSON(&inpObj, r.Body); err != nil {
		inpObj.Command = r.FormValue("cmd")

		inpObj.InPzFiles = splitOrNil(r.FormValue("inFiles"), ",")
		inpObj.InExtFiles = splitOrNil(r.FormValue("inFileURLs"), ",")
		inpObj.InPzNames = splitOrNil(r.FormValue("inPzFileNames"), ",")
		inpObj.InExtNames = splitOrNil(r.FormValue("inExtFileNames"), ",")
		inpObj.OutTiffs = splitOrNil(r.FormValue("outTiffs"), ",")
		inpObj.OutTxts = splitOrNil(r.FormValue("outTxts"), ",")
		inpObj.OutGeoJs = splitOrNil(r.FormValue("outGeoJson"), ",")
		inpObj.ExtAuth = r.FormValue("inUrlAuthKey")
		inpObj.PzAuth = r.FormValue("authKey")
	}

	cmdParamSlice := splitOrNil(inpObj.Command, " ")
	cmdConfigSlice := splitOrNil(configObj.CliCmd, " ")
	cmdSlice := append(cmdConfigSlice, cmdParamSlice...)

	if inpObj.PzAuth != "" {
		pzAuth = inpObj.PzAuth
	}

	if !canFile && (len(inpObj.InPzFiles)+len(inpObj.OutTiffs)+len(inpObj.OutTxts)+len(inpObj.OutGeoJs) != 0) {
		handleError(&output, "", fmt.Errorf("Cannot complete.  File up/download not enabled in config file."), w, http.StatusForbidden)
		return output
	}

	if pzAuth == "" && (len(inpObj.InPzFiles)+len(inpObj.OutTiffs)+len(inpObj.OutTxts)+len(inpObj.OutGeoJs) != 0) {
		handleError(&output, "", fmt.Errorf("Cannot complete.  Auth Key not available."), w, http.StatusForbidden)
		return output
	}

	runID, err := pzsvc.PsuUUID()
	handleError(&output, "psuUUID error: ", err, w, http.StatusInternalServerError)

	err = os.Mkdir("./"+runID, 0777)
	handleError(&output, "os.Mkdir error: ", err, w, http.StatusInternalServerError)
	defer os.RemoveAll("./" + runID)

	err = os.Chmod("./"+runID, 0777)
	handleError(&output, "os.Chmod error: ", err, w, http.StatusInternalServerError)

	// this is done to enable use of handleFList, which lets us
	// reduce a fair bit of code duplication in plowing through
	// our upload/download lists.  handleFList gets used a fair
	// bit more after the execute call.
	pzDownlFunc := func(dataID, fname, fType string) (string, error) {
		return pzsvc.DownloadByID(dataID, fname, runID, configObj.PzAddr, pzAuth)
	}
	handleFList(inpObj.InPzFiles, inpObj.InPzNames, pzDownlFunc, "", &output, output.InFiles, w)

	extDownlFunc := func(url, fname, fType string) (string, error) {
		return pzsvc.DownloadByURL(url, fname, runID, inpObj.ExtAuth)
	}
	handleFList(inpObj.InExtFiles, inpObj.InExtNames, extDownlFunc, "", &output, output.InFiles, w)

	if len(cmdSlice) == 0 {
		handleError(&output, "", errors.New(`No cmd or CliCmd.  Please provide "cmd" param.`), w, http.StatusBadRequest)
		return output
	}

	fmt.Println(`Executing "` + configObj.CliCmd + ` ` + inpObj.Command + `".`)

	// we're calling this from inside a temporary subfolder.  If the
	// program called exists inside the initial pzsvc-exec folder, that's
	// probably where it's called from, and we need to acccess it directly.
	_, err = os.Stat(fmt.Sprintf("./%s", cmdSlice[0]))
	if err == nil || !(os.IsNotExist(err)) {
		// ie, if there's a file in the start folder named the same thing
		// as the base command
		cmdSlice[0] = ("../" + cmdSlice[0])
	}

	clc := exec.Command(cmdSlice[0], cmdSlice[1:]...)
	clc.Dir = runID

	var stdout bytes.Buffer
	var stderr bytes.Buffer
	clc.Stdout = &stdout
	clc.Stderr = &stderr

	err = clc.Run()
	handleError(&output, "clc.Run error: ", err, w, http.StatusBadRequest)

	output.ProgStdOut = stdout.String()
	output.ProgStdErr = stderr.String()

	fmt.Println(`Program stdout: ` + output.ProgStdOut)
	fmt.Println(`Program stderr: ` + output.ProgStdErr)

	attMap := make(map[string]string)
	attMap["algoName"] = configObj.SvcName
	attMap["algoVersion"] = version
	attMap["algoCmd"] = configObj.CliCmd + " " + inpObj.Command
	attMap["algoProcTime"] = time.Now().UTC().Format("20060102.150405.99999")

	// this is the other spot that handleFlist gets used, and works on the
	// same principles.

	ingFunc := func(fName, dummy, fType string) (string, error) {
		return pzsvc.IngestFile(fName, runID, fType, configObj.PzAddr, configObj.SvcName, version, pzAuth, attMap)
	}

	handleFList(inpObj.OutTiffs, nil, ingFunc, "raster", &output, output.OutFiles, w)
	handleFList(inpObj.OutTxts, nil, ingFunc, "text", &output, output.OutFiles, w)
	handleFList(inpObj.OutGeoJs, nil, ingFunc, "geojson", &output, output.OutFiles, w)

	return output
}
<commit_msg>Added a hopefully useful bit of debug<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pzse

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"time"

	"github.com/venicegeo/pzsvc-lib"
)

type inpStruct struct {
	Command    string   `json:"cmd"`
	InPzFiles  []string `json:"inPzFiles"`    // slice: Pz dataIds
	InExtFiles []string `json:"inExtFiles"`   // slice: external URL
	InPzNames  []string `json:"inPzNames"`    // slice: name for the InPzFile of the same index
	InExtNames []string `json:"inExtNames"`   // slice: name for the InExtFile of the same index
	OutTiffs   []string `json:"outTiffs"`     // slice: filenames of GeoTIFFs to be ingested
	OutTxts    []string `json:"outTxts"`      // slice: filenames of text files to be ingested
	OutGeoJs   []string `json:"outGeoJson"`   // slice: filenames of GeoJSON files to be ingested
	ExtAuth    string   `json:"inExtAuthKey"` // string: auth key for accessing external files
	PzAuth     string   `json:"pzAuthKey"`    // string: suth key for accessing Piazza
}

// ParseConfig parses the config file on starting up
func ParseConfig(configObj *ConfigType) ConfigParseOut {

	canReg, canFile, hasAuth := CheckConfig(configObj)

	var authKey string
	if hasAuth {
		authKey = os.Getenv(configObj.AuthEnVar)
		if authKey == "" {
			fmt.Println("Error: no auth key at AuthEnVar.  Registration disabled, and client will have to provide authKey.")
			hasAuth = false
			canReg = false
		}
	}

	if configObj.Port <= 0 {
		configObj.Port = 8080
	}
	portStr := ":" + strconv.Itoa(configObj.Port)

	version := GetVersion(configObj)

	if canReg {
		fmt.Println("About to manage registration.")
		err := pzsvc.ManageRegistration(configObj.SvcName,
			configObj.Description,
			configObj.URL+"/execute",
			configObj.PzAddr,
			version,
			authKey,
			configObj.Attributes)
		if err != nil {
			fmt.Println("pzsvc-exec error in managing registration: ", err.Error())
		}
		fmt.Println("Registration managed.")
	}

	var procPool = pzsvc.Semaphore(nil)
	if configObj.NumProcs > 0 {
		procPool = make(pzsvc.Semaphore, configObj.NumProcs)
	}

	return ConfigParseOut{authKey, portStr, version, canFile, procPool}
}

// Execute does the primary work for pzsvc-exec.  Given a request and various
// blocks of config data, it creates a temporary folder to work in, downloads
// any files indicated in the request (if the configs support it), executes
// the command indicated by the combination of request and configs, uploads
// any files indicated by the request (if the configs support it) and cleans
// up after itself
func Execute(w http.ResponseWriter, r *http.Request, configObj ConfigType, pzAuth, version string, canFile bool, procPool pzsvc.Semaphore) OutStruct {

	// Makes sure that you only have a certain number of execution tasks firing at once.
	// pzsvc-exec calls can get pretty resource-intensive, and this keeps them from
	// trampling each other into messy deadlock
	procPool.Lock()
	defer procPool.Unlock()

	var output OutStruct
	output.InFiles = make(map[string]string)
	output.OutFiles = make(map[string]string)
	output.HTTPStatus = http.StatusOK

	if r.Method != "POST" {
		handleError(&output, "", fmt.Errorf(r.Method+" not supported.  Please us POST."), w, http.StatusMethodNotAllowed)
		return output
	}

	var inpObj inpStruct

	if _, err := pzsvc.ReadBodyJSON(&inpObj, r.Body); err != nil {
		handleError(&output, "could not interpret body as json: ", err, w, 200)
		inpObj.Command = r.FormValue("cmd")
		inpObj.InPzFiles = splitOrNil(r.FormValue("inFiles"), ",")
		inpObj.InExtFiles = splitOrNil(r.FormValue("inFileURLs"), ",")
		inpObj.InPzNames = splitOrNil(r.FormValue("inPzFileNames"), ",")
		inpObj.InExtNames = splitOrNil(r.FormValue("inExtFileNames"), ",")
		inpObj.OutTiffs = splitOrNil(r.FormValue("outTiffs"), ",")
		inpObj.OutTxts = splitOrNil(r.FormValue("outTxts"), ",")
		inpObj.OutGeoJs = splitOrNil(r.FormValue("outGeoJson"), ",")
		inpObj.ExtAuth = r.FormValue("inUrlAuthKey")
		inpObj.PzAuth = r.FormValue("authKey")
	}

	cmdParamSlice := splitOrNil(inpObj.Command, " ")
	cmdConfigSlice := splitOrNil(configObj.CliCmd, " ")
	cmdSlice := append(cmdConfigSlice, cmdParamSlice...)

	if inpObj.PzAuth != "" {
		pzAuth = inpObj.PzAuth
	}

	if !canFile && (len(inpObj.InPzFiles)+len(inpObj.OutTiffs)+len(inpObj.OutTxts)+len(inpObj.OutGeoJs) != 0) {
		handleError(&output, "", fmt.Errorf("Cannot complete.  File up/download not enabled in config file."), w, http.StatusForbidden)
		return output
	}

	if pzAuth == "" && (len(inpObj.InPzFiles)+len(inpObj.OutTiffs)+len(inpObj.OutTxts)+len(inpObj.OutGeoJs) != 0) {
		handleError(&output, "", fmt.Errorf("Cannot complete.  Auth Key not available."), w, http.StatusForbidden)
		return output
	}

	runID, err := pzsvc.PsuUUID()
	handleError(&output, "psuUUID error: ", err, w, http.StatusInternalServerError)

	err = os.Mkdir("./"+runID, 0777)
	handleError(&output, "os.Mkdir error: ", err, w, http.StatusInternalServerError)
	defer os.RemoveAll("./" + runID)

	err = os.Chmod("./"+runID, 0777)
	handleError(&output, "os.Chmod error: ", err, w, http.StatusInternalServerError)

	// this is done to enable use of handleFList, which lets us
	// reduce a fair bit of code duplication in plowing through
	// our upload/download lists.  handleFList gets used a fair
	// bit more after the execute call.
	pzDownlFunc := func(dataID, fname, fType string) (string, error) {
		return pzsvc.DownloadByID(dataID, fname, runID, configObj.PzAddr, pzAuth)
	}
	handleFList(inpObj.InPzFiles, inpObj.InPzNames, pzDownlFunc, "", &output, output.InFiles, w)

	extDownlFunc := func(url, fname, fType string) (string, error) {
		return pzsvc.DownloadByURL(url, fname, runID, inpObj.ExtAuth)
	}
	handleFList(inpObj.InExtFiles, inpObj.InExtNames, extDownlFunc, "", &output, output.InFiles, w)

	if len(cmdSlice) == 0 {
		handleError(&output, "", errors.New(`No cmd or CliCmd.  Please provide "cmd" param.`), w, http.StatusBadRequest)
		return output
	}

	fmt.Println(`Executing "` + configObj.CliCmd + ` ` + inpObj.Command + `".`)

	// we're calling this from inside a temporary subfolder.  If the
	// program called exists inside the initial pzsvc-exec folder, that's
	// probably where it's called from, and we need to acccess it directly.
	_, err = os.Stat(fmt.Sprintf("./%s", cmdSlice[0]))
	if err == nil || !(os.IsNotExist(err)) {
		// ie, if there's a file in the start folder named the same thing
		// as the base command
		cmdSlice[0] = ("../" + cmdSlice[0])
	}

	clc := exec.Command(cmdSlice[0], cmdSlice[1:]...)
	clc.Dir = runID

	var stdout bytes.Buffer
	var stderr bytes.Buffer
	clc.Stdout = &stdout
	clc.Stderr = &stderr

	err = clc.Run()
	handleError(&output, "clc.Run error: ", err, w, http.StatusBadRequest)

	output.ProgStdOut = stdout.String()
	output.ProgStdErr = stderr.String()

	fmt.Println(`Program stdout: ` + output.ProgStdOut)
	fmt.Println(`Program stderr: ` + output.ProgStdErr)

	attMap := make(map[string]string)
	attMap["algoName"] = configObj.SvcName
	attMap["algoVersion"] = version
	attMap["algoCmd"] = configObj.CliCmd + " " + inpObj.Command
	attMap["algoProcTime"] = time.Now().UTC().Format("20060102.150405.99999")

	// this is the other spot that handleFlist gets used, and works on the
	// same principles.

	ingFunc := func(fName, dummy, fType string) (string, error) {
		return pzsvc.IngestFile(fName, runID, fType, configObj.PzAddr, configObj.SvcName, version, pzAuth, attMap)
	}

	handleFList(inpObj.OutTiffs, nil, ingFunc, "raster", &output, output.OutFiles, w)
	handleFList(inpObj.OutTxts, nil, ingFunc, "text", &output, output.OutFiles, w)
	handleFList(inpObj.OutGeoJs, nil, ingFunc, "geojson", &output, output.OutFiles, w)

	return output
}
<|endoftext|>


<commit_before>package gomgen

import (
	"bitbucket.org/pkg/inflect"
	"bytes"
	"database/sql"
	"go/format"
	"strings"
	"text/template"
)

// Gomgen generator is the primary interface for scanning,
// analyzing and generating models with gomgen
type Generator struct {
	Db      *sql.DB
	Schema  string
	Tables  []*Table
	Imports map[string]bool
	Output  *bytes.Buffer
}

// create and initialize new Gomgen object
func NewGenerator(db *sql.DB, schema string) *Generator {
	return &Generator{
		Db:     db,
		Schema: schema,
		Tables: nil,
		Imports: map[string]bool{
			"database/sql": true,
		},
		Output: &bytes.Buffer{},
	}
}

// Investigate the database
func (this *Generator) Analyse() error {
	mysql := &Mysql{}
	return mysql.Analyze(this)
}

// Generate the model source code
func (this *Generator) Generate() error {
	var t = template.Must(template.New("headerTpl").Parse(headerTpl))
	if err := t.Execute(this.Output, this); err != nil {
		return err
	}

	// entities
	for _, table := range this.Tables {
		this.genStruct(table)
		this.genScanFn(table)
	}

	// format the code
	c, err := format.Source(this.Output.Bytes())
	if err != nil {
		// return err
	}
	this.Output.Reset()
	this.Output.Write(c)

	// done :)
	return nil
}

// generate the table entity
func (this *Generator) genStruct(table *Table) error {
	var t = template.Must(template.New("entityStructTpl").Parse(entityStructTpl))
	return t.Execute(this.Output, table)
}

// Generate scan function
func (this *Generator) genScanFn(table *Table) error {
	// the template
	var t = template.Must(template.New("scanEntity").Parse(scanEntityTpl))

	// template params
	type templateParams struct {
		*Table
		Vars   map[string]string // declared extra variables
		Params string // params for the Scan method
		Inits  []string // value loads for the variables
	}
	p := &templateParams{}
	p.Table = table
	p.Vars = make(map[string]string)

	// process fields
	var params []string
	for _, field := range table.Fields {
		if field.Type == GoTime {
			if _, ok := p.Vars["string"]; ok {
				p.Vars["string"] += ", " + field.Name
			} else {
				p.Vars["string"] = field.Name
			}
			params = append(params, "&" + field.Name)
			init := "if t, err := time.Parse(\"" + field.Format + "\", " + field.Name + "); err != nil {\n"
			init += "	return err\n"
			init += "} else {\n"
			init += "	this." + field.Name + " = t\n"
			init += "}"
			p.Inits = append(p.Inits, init)
		} else {
			params = append(params, "&this." + field.Name)
		}
	}
	p.Params = strings.Join(params, ", ")

	// process
	return t.Execute(this.Output, p)
}

// represent a database table
type Table struct {
	Name           string
	EntitySingular string
	EntityPlural   string
	Comment        string
	Fields         []*Field
	Identity       []string
}

// create new table
func NewTable(sqlName, comment string) *Table {
	name := strings.ToLower(sqlName)
	singular := inflect.Singularize(name)
	plural := inflect.Pluralize(name)
	return &Table{
		Name:           sqlName,
		Comment:        comment,
		EntitySingular: strings.Title(singular),
		EntityPlural:   strings.Title(plural),
	}
}

// Field data type mapping to Go
type GoType int

const (
	GoInt GoType = iota
	GoFloat64
	GoBool
	GoString
	GoTime
	GoNullInt
	GoNullFloat64
	GoNullBool
	GoNullString
)

// map GoType constants to strings of actual types
var GoTypeMap = map[GoType]string{
	GoInt:         "int",
	GoFloat64:     "float64",
	GoBool:        "bool",
	GoString:      "string",
	GoTime:        "time.Time",
	GoNullInt:     "sql.NullInt64",
	GoNullFloat64: "sql.NullFloat64",
	GoNullBool:    "sql.NullBool",
	GoNullString:  "sql.NullString",
}

// represent individual field in the table
type Field struct {
	Name     string
	SqlName	string
	Default  sql.NullString
	Nullable bool
	Type     GoType
	GoType   string
	Primary  bool
	Comment  string
	Format 	 string
}

// the name of the field
func NewField(rawName string) *Field {
	parts := strings.Split(strings.ToLower(rawName), "_")
	for i := 0; i < len(parts); i++ {
		parts[i] = strings.Title(parts[i])
	}
	return &Field{
		Name: strings.Join(parts, ""),
		SqlName: rawName,
	}
}
<commit_msg>Simplify date parsing for now<commit_after>package gomgen

import (
	"bitbucket.org/pkg/inflect"
	"bytes"
	"database/sql"
	"go/format"
	"strings"
	"text/template"
)

// Gomgen generator is the primary interface for scanning,
// analyzing and generating models with gomgen
type Generator struct {
	Db      *sql.DB
	Schema  string
	Tables  []*Table
	Imports map[string]bool
	Output  *bytes.Buffer
}

// create and initialize new Gomgen object
func NewGenerator(db *sql.DB, schema string) *Generator {
	return &Generator{
		Db:     db,
		Schema: schema,
		Tables: nil,
		Imports: map[string]bool{
			"database/sql": true,
		},
		Output: &bytes.Buffer{},
	}
}

// Investigate the database
func (this *Generator) Analyse() error {
	mysql := &Mysql{}
	return mysql.Analyze(this)
}

// Generate the model source code
func (this *Generator) Generate() error {
	var t = template.Must(template.New("headerTpl").Parse(headerTpl))
	if err := t.Execute(this.Output, this); err != nil {
		return err
	}

	// entities
	for _, table := range this.Tables {
		this.genStruct(table)
		this.genScanFn(table)
	}

	// format the code
	c, err := format.Source(this.Output.Bytes())
	if err != nil {
		// return err
	}
	this.Output.Reset()
	this.Output.Write(c)

	// done :)
	return nil
}

// generate the table entity
func (this *Generator) genStruct(table *Table) error {
	var t = template.Must(template.New("entityStructTpl").Parse(entityStructTpl))
	return t.Execute(this.Output, table)
}

// Generate scan function
func (this *Generator) genScanFn(table *Table) error {
	// the template
	var t = template.Must(template.New("scanEntity").Parse(scanEntityTpl))

	// template params
	type templateParams struct {
		*Table
		Vars   map[string]string // declared extra variables
		Params string // params for the Scan method
		Inits  []string // value loads for the variables
	}
	p := &templateParams{}
	p.Table = table
	p.Vars = make(map[string]string)

	// process fields
	var params []string
	for _, field := range table.Fields {
		if field.Type == GoTime {
			if _, ok := p.Vars["string"]; ok {
				p.Vars["string"] += ", " + field.Name
			} else {
				p.Vars["string"] = field.Name
			}
			params = append(params, "&" + field.Name)
			init := "this." + field.Name + ", _ = time.Parse(\"" + field.Format + "\", " + field.Name + ")"
			p.Inits = append(p.Inits, init)
		} else {
			params = append(params, "&this." + field.Name)
		}
	}
	p.Params = strings.Join(params, ", ")

	// process
	return t.Execute(this.Output, p)
}

// represent a database table
type Table struct {
	Name           string
	EntitySingular string
	EntityPlural   string
	Comment        string
	Fields         []*Field
	Identity       []string
}

// create new table
func NewTable(sqlName, comment string) *Table {
	name := strings.ToLower(sqlName)
	singular := inflect.Singularize(name)
	plural := inflect.Pluralize(name)
	return &Table{
		Name:           sqlName,
		Comment:        comment,
		EntitySingular: strings.Title(singular),
		EntityPlural:   strings.Title(plural),
	}
}

// Field data type mapping to Go
type GoType int

const (
	GoInt GoType = iota
	GoFloat64
	GoBool
	GoString
	GoTime
	GoNullInt
	GoNullFloat64
	GoNullBool
	GoNullString
)

// map GoType constants to strings of actual types
var GoTypeMap = map[GoType]string{
	GoInt:         "int",
	GoFloat64:     "float64",
	GoBool:        "bool",
	GoString:      "string",
	GoTime:        "time.Time",
	GoNullInt:     "sql.NullInt64",
	GoNullFloat64: "sql.NullFloat64",
	GoNullBool:    "sql.NullBool",
	GoNullString:  "sql.NullString",
}

// represent individual field in the table
type Field struct {
	Name     string
	SqlName	string
	Default  sql.NullString
	Nullable bool
	Type     GoType
	GoType   string
	Primary  bool
	Comment  string
	Format 	 string
}

// the name of the field
func NewField(rawName string) *Field {
	parts := strings.Split(strings.ToLower(rawName), "_")
	for i := 0; i < len(parts); i++ {
		parts[i] = strings.Title(parts[i])
	}
	return &Field{
		Name: strings.Join(parts, ""),
		SqlName: rawName,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/imdario/mergo"
	uuid "github.com/satori/go.uuid"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

var jobMutex sync.Mutex
var jobs = make(map[string]*Job)

type Service struct {
	EndPoint    string            `yaml:"endPoint" json:"endPoint"`
	CommandLine []string          `yaml:"commandLine" json:"commandLine"`
	Description string            `json:"description"`
	Defaults    map[string]string `yaml:defaults json:defaults`
}

type Job struct {
	sync.Mutex        `json:ignore`
	UUID              string            `json:"uuid"`
	CommandLine       []string          `yaml:"commandLine" json:"commandLine"`
	ParsedCommandLine []string          `json:"-"`
	FileMap           map[string]string `json:"-"`
	StartTime         time.Time         `json:"start_time"`
	EndTime           time.Time         `json:"end_time"`
	Status            string            `json:"status"`
	Address           []string          `json:"address"`
	Endpoint          string            `json:"endpoint"`

	// Registered channels
	waiters []chan bool

	// Running process
	cmd    *exec.Cmd
	Output bytes.Buffer `json:"output"`
}

func Template(name string, data map[string]interface{}, w http.ResponseWriter, request *http.Request) {
	var templateData = map[string]interface{}{
		"jobs":       jobs,
		"services":   config.Services,
		"serviceMap": config.ServiceMap,
	}
	// merge in our extra data
	mergo.Map(&templateData, data)
	contents, _ := Asset("template/" + name + ".html")
	t, _ := template.New(name).Parse(string(contents))
	t.Execute(w, templateData)
}
func Help(w http.ResponseWriter, request *http.Request) {
	Template("help", nil, w, request)
}
func Jobs(w http.ResponseWriter, request *http.Request) {
	Template("jobs", nil, w, request)
}
func Submit(w http.ResponseWriter, request *http.Request) {
	Template("submit", nil, w, request)
}
func Services(w http.ResponseWriter, request *http.Request) {
	Template("services", nil, w, request)
}
func JobDetail(w http.ResponseWriter, request *http.Request) {
	key := mux.Vars(request)["id"]
	job := jobs[key]
	if job == nil {
		http.Error(w, "could not find job", http.StatusNotFound)
		return
	}
	var data = map[string]interface{}{
		"job": job}
	Template("job", data, w, request)
}

func GetServices(w http.ResponseWriter, request *http.Request) {
	json.NewEncoder(w).Encode(config)
}

func GetService(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	service := config.ServiceMap[vars["id"]]
	json.NewEncoder(w).Encode(service)
}

func StartService(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	service := config.ServiceMap[vars["id"]]
	log.Printf("Found service %v:%v", vars["id"], service)
	// Pull out our arguments
	err := request.ParseMultipartForm(10 * 1024 * 1024)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	job := Job{
		UUID:        uuid.NewV4().String(),
		CommandLine: service.CommandLine,
		FileMap:     make(map[string]string),
		Endpoint:    service.EndPoint,
	}

	// do we have an email address?
	if request.MultipartForm.Value["mail"] != nil {
		job.Address = request.MultipartForm.Value["mail"]
	}

	cl := make([]string, 0)
	// Make a temp directory
	dir, err := ioutil.TempDir("", job.UUID)
	for _, arg := range service.CommandLine {
		log.Printf("Parsing %v", arg)
		// Do we start with an @?
		key := arg[1:]
		prefix := arg[0]
		if prefix == '@' {
			// Lookup first in form
			if request.MultipartForm.Value[key] != nil {
				cl = append(cl, request.MultipartForm.Value[key][0])
			} else {
				// Look up in defaults
				cl = append(cl, service.Defaults[key])
			}
		} else if prefix == '<' {
			// Do we have an < to indicate an uploaded file?
			v := request.MultipartForm.File[key]
			if v == nil {
				http.Error(w, fmt.Sprintf("Could not find %v in form data", key), http.StatusInternalServerError)
				return
			}
			header := v[0]
			// Save a temp file
			fout, err := os.Create(filepath.Join(dir, filepath.Base(header.Filename)))
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			f, err := header.Open()
			count, err := io.Copy(fout, f)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			log.Printf("Wrote %v bytes to %v", count, header.Filename)
			fout.Close()
			cl = append(cl, fout.Name())
		} else if prefix == '>' {
			// Write a file...
			// Save a temp file
			if request.MultipartForm.Value[key] == nil {
				http.Error(w, fmt.Sprintf("filename must be specified for %v", key), http.StatusInternalServerError)
				return
			}
			tmp := filepath.Join(dir, filepath.Base(request.MultipartForm.Value[key][0]))
			job.FileMap[key] = tmp
			cl = append(cl, tmp)
		} else {
			cl = append(cl, arg)
		}
	}
	log.Printf("Final command line: %v", cl)
	job.ParsedCommandLine = cl
	cmd := exec.Command(cl[0], cl[1:]...)
	job.StartTime = time.Now()
	cmd.Stdout = &job.Output
	cmd.Stderr = &job.Output
	job.cmd = cmd
	job.Status = "pending"

	// Launch a go routine to wait
	go func() {
		jobMutex.Lock()
		defer jobMutex.Unlock()
		job.Status = "running"
		job.cmd.Start()
		err := job.cmd.Wait()
		job.EndTime = time.Now()
		if err != nil {
			job.Status = "error"
		} else {
			if job.cmd.ProcessState.Success() {
				job.Status = "success"
			} else {
				job.Status = "failed"
			}
		}
		// Notify waiters
		for _, c := range job.waiters {
			c <- true
		}

		// Send email here
		Email(&job)

		// Cleanup after 120 minutes
		<-time.After(time.Minute * 120)
		Cleanup(&job)
	}()

	json.NewEncoder(w).Encode(job)
	jobs[job.UUID] = &job
}

func GetJob(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	job := jobs[vars["id"]]
	json.NewEncoder(w).Encode(job)
}

func WaitForJob(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	job := jobs[vars["id"]]
	c := make(chan bool)
	job.Lock()
	job.waiters = append(job.waiters, c)
	job.Unlock()
	<-c
	close(c)
	json.NewEncoder(w).Encode(job)
}

func GetJobFile(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	key := vars["id"]
	job := jobs[key]
	if job == nil {
		http.Error(w, fmt.Sprintf("job %v does not exist", key), http.StatusInternalServerError)
		return
	}
	file := job.FileMap[vars["filename"]]
	w.Header().Set("Content-Disposition", "attachment;filename="+filepath.Base(file))
	http.ServeFile(w, request, file)
}
<commit_msg>Adds a bit more logging<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/imdario/mergo"
	uuid "github.com/satori/go.uuid"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

var jobMutex sync.Mutex
var jobs = make(map[string]*Job)

type Service struct {
	EndPoint    string            `yaml:"endPoint" json:"endPoint"`
	CommandLine []string          `yaml:"commandLine" json:"commandLine"`
	Description string            `json:"description"`
	Defaults    map[string]string `yaml:defaults json:defaults`
}

type Job struct {
	sync.Mutex        `json:ignore`
	UUID              string            `json:"uuid"`
	CommandLine       []string          `yaml:"commandLine" json:"commandLine"`
	ParsedCommandLine []string          `json:"-"`
	FileMap           map[string]string `json:"-"`
	StartTime         time.Time         `json:"start_time"`
	EndTime           time.Time         `json:"end_time"`
	Status            string            `json:"status"`
	Address           []string          `json:"address"`
	Endpoint          string            `json:"endpoint"`

	// Registered channels
	waiters []chan bool

	// Running process
	cmd    *exec.Cmd
	Output bytes.Buffer `json:"output"`
}

func Template(name string, data map[string]interface{}, w http.ResponseWriter, request *http.Request) {
	var templateData = map[string]interface{}{
		"jobs":       jobs,
		"services":   config.Services,
		"serviceMap": config.ServiceMap,
	}
	// merge in our extra data
	mergo.Map(&templateData, data)
	contents, _ := Asset("template/" + name + ".html")
	t, _ := template.New(name).Parse(string(contents))
	t.Execute(w, templateData)
}
func Help(w http.ResponseWriter, request *http.Request) {
	Template("help", nil, w, request)
}
func Jobs(w http.ResponseWriter, request *http.Request) {
	Template("jobs", nil, w, request)
}
func Submit(w http.ResponseWriter, request *http.Request) {
	Template("submit", nil, w, request)
}
func Services(w http.ResponseWriter, request *http.Request) {
	Template("services", nil, w, request)
}
func JobDetail(w http.ResponseWriter, request *http.Request) {
	key := mux.Vars(request)["id"]
	job := jobs[key]
	if job == nil {
		http.Error(w, "could not find job", http.StatusNotFound)
		return
	}
	var data = map[string]interface{}{
		"job": job}
	Template("job", data, w, request)
}

func GetServices(w http.ResponseWriter, request *http.Request) {
	json.NewEncoder(w).Encode(config)
}

func GetService(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	service := config.ServiceMap[vars["id"]]
	json.NewEncoder(w).Encode(service)
}

func StartService(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	service := config.ServiceMap[vars["id"]]
	log.Printf("Found service %v:%v", vars["id"], service)
	// Pull out our arguments
	err := request.ParseMultipartForm(10 * 1024 * 1024)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	job := Job{
		UUID:        uuid.NewV4().String(),
		CommandLine: service.CommandLine,
		FileMap:     make(map[string]string),
		Endpoint:    service.EndPoint,
	}

	// do we have an email address?
	if request.MultipartForm.Value["mail"] != nil {
		job.Address = request.MultipartForm.Value["mail"]
	}

	cl := make([]string, 0)
	// Make a temp directory
	dir, err := ioutil.TempDir("", job.UUID)
	for _, arg := range service.CommandLine {
		log.Printf("Parsing %v", arg)
		// Do we start with an @?
		key := arg[1:]
		prefix := arg[0]
		if prefix == '@' {
			// Lookup first in form
			if request.MultipartForm.Value[key] != nil {
				cl = append(cl, request.MultipartForm.Value[key][0])
			} else {
				// Look up in defaults
				cl = append(cl, service.Defaults[key])
			}
		} else if prefix == '<' {
			// Do we have an < to indicate an uploaded file?
			v := request.MultipartForm.File[key]
			if v == nil {
				http.Error(w, fmt.Sprintf("Could not find %v in form data", key), http.StatusInternalServerError)
				return
			}
			header := v[0]
			// Save a temp file
			fout, err := os.Create(filepath.Join(dir, filepath.Base(header.Filename)))
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			f, err := header.Open()
			count, err := io.Copy(fout, f)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			log.Printf("Wrote %v bytes to %v", count, header.Filename)
			fout.Close()
			cl = append(cl, fout.Name())
		} else if prefix == '>' {
			// Write a file...
			// Save a temp file
			if request.MultipartForm.Value[key] == nil {
				http.Error(w, fmt.Sprintf("filename must be specified for %v", key), http.StatusInternalServerError)
				return
			}
			tmp := filepath.Join(dir, filepath.Base(request.MultipartForm.Value[key][0]))
			job.FileMap[key] = tmp
			cl = append(cl, tmp)
		} else {
			cl = append(cl, arg)
		}
	}
	log.Printf("Final command line: %v", cl)
	job.ParsedCommandLine = cl
	cmd := exec.Command(cl[0], cl[1:]...)
	job.StartTime = time.Now()
	cmd.Stdout = &job.Output
	cmd.Stderr = &job.Output
	job.cmd = cmd
	job.Status = "pending"

	// Launch a go routine to wait
	go func() {
		jobMutex.Lock()
		job.Status = "running"
		job.cmd.Start()
		err := job.cmd.Wait()
		job.EndTime = time.Now()
		if err != nil {
			job.Status = "error"
		} else {
			if job.cmd.ProcessState.Success() {
				job.Status = "success"
			} else {
				job.Status = "failed"
			}
		}
		jobMutex.Unlock()
		// Notify waiters
		log.Printf("%v completed with status %v", job.UUID, job.Status)
		for _, c := range job.waiters {
			c <- true
		}

		// Send email here
		Email(&job)

		// Cleanup after 120 minutes
		<-time.After(time.Minute * 120)
		Cleanup(&job)
	}()

	json.NewEncoder(w).Encode(job)
	jobs[job.UUID] = &job
}

func GetJob(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	job := jobs[vars["id"]]
	json.NewEncoder(w).Encode(job)
}

func WaitForJob(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	job := jobs[vars["id"]]
	c := make(chan bool)
	job.Lock()
	job.waiters = append(job.waiters, c)
	job.Unlock()
	<-c
	close(c)
	json.NewEncoder(w).Encode(job)
}

func GetJobFile(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	key := vars["id"]
	job := jobs[key]
	if job == nil {
		http.Error(w, fmt.Sprintf("job %v does not exist", key), http.StatusInternalServerError)
		return
	}
	file := job.FileMap[vars["filename"]]
	w.Header().Set("Content-Disposition", "attachment;filename="+filepath.Base(file))
	http.ServeFile(w, request, file)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package conversion_test

import (
	"io/ioutil"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta1"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta2"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta3"
)

func BenchmarkPodConversion(b *testing.B) {
	data, err := ioutil.ReadFile("pod_example.json")
	if err != nil {
		b.Fatalf("unexpected error while reading file: %v", err)
	}
	var pod api.Pod
	if err := api.Scheme.DecodeInto(data, &pod); err != nil {
		b.Fatalf("unexpected error decoding pod: %v", err)
	}

	scheme := api.Scheme.Raw()
	for i := 0; i < b.N; i++ {
		versionedObj, err := scheme.ConvertToVersion(&pod, testapi.Version())
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
		_, err = scheme.ConvertToVersion(versionedObj, scheme.InternalVersion)
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
	}
}

func BenchmarkNodeConversion(b *testing.B) {
	data, err := ioutil.ReadFile("node_example.json")
	if err != nil {
		b.Fatalf("unexpected error while reading file: %v", err)
	}
	var node api.Node
	if err := api.Scheme.DecodeInto(data, &node); err != nil {
		b.Fatalf("unexpected error decoding node: %v", err)
	}

	scheme := api.Scheme.Raw()
	for i := 0; i < b.N; i++ {
		versionedObj, err := scheme.ConvertToVersion(&node, testapi.Version())
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
		_, err = scheme.ConvertToVersion(versionedObj, scheme.InternalVersion)
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
	}
}

func BenchmarkReplicationControllerConversion(b *testing.B) {
	data, err := ioutil.ReadFile("replication_controller_example.json")
	if err != nil {
		b.Fatalf("unexpected error while reading file: %v", err)
	}
	var replicationController api.ReplicationController
	if err := api.Scheme.DecodeInto(data, &replicationController); err != nil {
		b.Fatalf("unexpected error decoding node: %v", err)
	}

	scheme := api.Scheme.Raw()
	for i := 0; i < b.N; i++ {
		versionedObj, err := scheme.ConvertToVersion(&replicationController, testapi.Version())
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
		_, err = scheme.ConvertToVersion(versionedObj, scheme.InternalVersion)
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
	}
}
<commit_msg>Avoid benchmarks from being optimized<commit_after>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package conversion_test

import (
	"io/ioutil"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta1"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta2"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta3"
)

func BenchmarkPodConversion(b *testing.B) {
	data, err := ioutil.ReadFile("pod_example.json")
	if err != nil {
		b.Fatalf("unexpected error while reading file: %v", err)
	}
	var pod api.Pod
	if err := api.Scheme.DecodeInto(data, &pod); err != nil {
		b.Fatalf("unexpected error decoding pod: %v", err)
	}

	scheme := api.Scheme.Raw()
	var result *api.Pod
	for i := 0; i < b.N; i++ {
		versionedObj, err := scheme.ConvertToVersion(&pod, testapi.Version())
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
		obj, err := scheme.ConvertToVersion(versionedObj, scheme.InternalVersion)
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
		result = obj.(*api.Pod)
	}
	if !api.Semantic.DeepDerivative(pod, *result) {
		b.Fatalf("Incorrect conversion: expected %v, got %v", pod, result)
	}
}

func BenchmarkNodeConversion(b *testing.B) {
	data, err := ioutil.ReadFile("node_example.json")
	if err != nil {
		b.Fatalf("unexpected error while reading file: %v", err)
	}
	var node api.Node
	if err := api.Scheme.DecodeInto(data, &node); err != nil {
		b.Fatalf("unexpected error decoding node: %v", err)
	}

	scheme := api.Scheme.Raw()
	var result *api.Node
	for i := 0; i < b.N; i++ {
		versionedObj, err := scheme.ConvertToVersion(&node, testapi.Version())
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
		obj, err := scheme.ConvertToVersion(versionedObj, scheme.InternalVersion)
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
		result = obj.(*api.Node)
	}
	if !api.Semantic.DeepDerivative(node, *result) {
		b.Fatalf("Incorrect conversion: expected %v, got %v", node, result)
	}
}

func BenchmarkReplicationControllerConversion(b *testing.B) {
	data, err := ioutil.ReadFile("replication_controller_example.json")
	if err != nil {
		b.Fatalf("unexpected error while reading file: %v", err)
	}
	var replicationController api.ReplicationController
	if err := api.Scheme.DecodeInto(data, &replicationController); err != nil {
		b.Fatalf("unexpected error decoding node: %v", err)
	}

	scheme := api.Scheme.Raw()
	var result *api.ReplicationController
	for i := 0; i < b.N; i++ {
		versionedObj, err := scheme.ConvertToVersion(&replicationController, testapi.Version())
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
		obj, err := scheme.ConvertToVersion(versionedObj, scheme.InternalVersion)
		if err != nil {
			b.Fatalf("Conversion error: %v", err)
		}
		result = obj.(*api.ReplicationController)
	}
	if !api.Semantic.DeepDerivative(replicationController, *result) {
		b.Fatalf("Incorrect conversion: expected %v, got %v", replicationController, result)
	}
}
<|endoftext|>


<commit_before>package lexer

import (
	"testing"

	"github.com/BenchR267/lbd/lexer/token"
)

func TestStreamFromString(t *testing.T) {
	count := 0

	const input = "abc"

	stream := StreamFromString(input)

	for b := range stream {
		if b != input[count] {
			t.Errorf("Expected to receive %c over channel at index %d from input %s", b, count, input)
		}
		count++
	}

	if count != len(input) {
		t.Errorf("Expected count (%d) to be equal to %d.", count, len(input))
	}
}

func TestIsWhitespace(t *testing.T) {
	testCases := []struct {
		input    byte
		expected bool
	}{
		{'a', false},
		{'b', false},
		{'\n', true},
		{' ', true},
	}

	for _, c := range testCases {
		got := isWhitespace(c.input)
		if got != c.expected {
			t.Errorf("Whitespace test fail. Input: %c - Expected: %#v - Got: %#v", c.input, c.expected, got)
		}
	}
}

func TestIsLetter(t *testing.T) {
	testCases := []struct {
		input    byte
		expected bool
	}{
		{'a', true},
		{'b', true},
		{'\n', false},
		{' ', false},
		{'3', false},
		{'$', false},
	}

	for _, c := range testCases {
		got := isLetter(c.input)
		if got != c.expected {
			t.Errorf("Letter test fail. Input: %c - Expected: %#v - Got: %#v", c.input, c.expected, got)
		}
	}
}

func TestTokenizer(t *testing.T) {
	testCases := []struct {
		input    string
		expected token.Type
	}{
		{"abc", token.Identifier},
		{"a", token.Identifier},

		{"5", token.Integer},

		{"(", token.Parenthesis},
		{"{", token.CurlyBracket},
		{"[", token.SquareBracket},

		{"=", token.Assign},
		{"+", token.Plus},
		{"-", token.Minus},
		{"/", token.Slash},
		{"%", token.Percent},

		{"==", token.Equal},
		{"!=", token.NotEqual},
		{">", token.Greater},
		{"<", token.Less},

		{"#", token.Illegal},
	}

	for _, c := range testCases {
		tokenizer := Tokenizer{
			content: []byte{},
		}
		for i := 0; i < len(c.input); i++ {
			got := tokenizer.append(c.input[i], token.Position{})
			if got != nil {
				t.Errorf("Expected token to be nil, but got %#v instead. input: %s", got, c.input)
			}
		}

		got := tokenizer.append(' ', token.Position{})
		if got.Type != c.expected {
			t.Errorf("Expected to get tokentype %#v, but got %#v instead.", c.expected.String(), got.Type.String())
		}
	}
}
<commit_msg>Added lexer test for complete example (one line)<commit_after>package lexer

import (
	"testing"

	"github.com/BenchR267/lbd/lexer/token"
)

func TestStreamFromString(t *testing.T) {
	count := 0

	const input = "abc"

	stream := StreamFromString(input)

	for b := range stream {
		if b != input[count] {
			t.Errorf("Expected to receive %c over channel at index %d from input %s", b, count, input)
		}
		count++
	}

	if count != len(input) {
		t.Errorf("Expected count (%d) to be equal to %d.", count, len(input))
	}
}

func TestIsWhitespace(t *testing.T) {
	testCases := []struct {
		input    byte
		expected bool
	}{
		{'a', false},
		{'b', false},
		{'\n', true},
		{' ', true},
	}

	for _, c := range testCases {
		got := isWhitespace(c.input)
		if got != c.expected {
			t.Errorf("Whitespace test fail. Input: %c - Expected: %#v - Got: %#v", c.input, c.expected, got)
		}
	}
}

func TestIsLetter(t *testing.T) {
	testCases := []struct {
		input    byte
		expected bool
	}{
		{'a', true},
		{'b', true},
		{'\n', false},
		{' ', false},
		{'3', false},
		{'$', false},
	}

	for _, c := range testCases {
		got := isLetter(c.input)
		if got != c.expected {
			t.Errorf("Letter test fail. Input: %c - Expected: %#v - Got: %#v", c.input, c.expected, got)
		}
	}
}

func TestTokenizer(t *testing.T) {
	testCases := []struct {
		input    string
		expected token.Type
	}{
		{"abc", token.Identifier},
		{"a", token.Identifier},

		{"5", token.Integer},

		{"(", token.Parenthesis},
		{"{", token.CurlyBracket},
		{"[", token.SquareBracket},

		{"=", token.Assign},
		{"+", token.Plus},
		{"-", token.Minus},
		{"/", token.Slash},
		{"%", token.Percent},

		{"==", token.Equal},
		{"!=", token.NotEqual},
		{">", token.Greater},
		{"<", token.Less},

		{"#", token.Illegal},
	}

	for _, c := range testCases {
		tokenizer := Tokenizer{
			content: []byte{},
		}
		for i := 0; i < len(c.input); i++ {
			got := tokenizer.append(c.input[i], token.Position{})
			if got != nil {
				t.Errorf("Expected token to be nil, but got %#v instead. input: %s", got, c.input)
			}
		}

		got := tokenizer.append(' ', token.Position{})
		if got.Type != c.expected {
			t.Errorf("Expected to get tokentype %#v, but got %#v instead.", c.expected.String(), got.Type.String())
		}
	}
}

func TestStart_OneLine(t *testing.T) {
	stream := StreamFromString("abc = dfe + 3")
	l := NewLexer(stream)
	l.Start()

	expectedValues := []token.Token{
		token.Token{
			Pos:  token.Position{Line: 0, Column: 0, Len: 3},
			Type: token.Identifier,
			Raw:  "abc",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 4, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 6, Len: 3},
			Type: token.Identifier,
			Raw:  "dfe",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 10, Len: 1},
			Type: token.Plus,
			Raw:  "+",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 12, Len: 1},
			Type: token.Integer,
			Raw:  "3",
		},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected != token {
			t.Errorf("Expected: %#v, got: %#v.", expected, token)
		}

		i++
	}
}
<|endoftext|>


<commit_before>package lexer

import (
	"testing"

	"github.com/BenchR267/lbd/lexer/token"
)

func TestIgnoreWhitespace(t *testing.T) {
	stream := StreamFromString("a=    5")
	l := NewLexer(stream)
	l.Start()

	expectedValues := []token.Token{
		token.Token{
			Pos:  token.Position{Line: 0, Column: 0, Len: 1},
			Type: token.Identifier,
			Raw:  "a",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 1, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 6, Len: 1},
			Type: token.Integer,
			Raw:  "5",
		},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected != token {
			t.Errorf("Expected: %#v, got: %#v.", expected, token)
		}

		i++
	}
}

func TestStart_OneLine(t *testing.T) {
	stream := StreamFromString("abc = dfe + 3")
	l := NewLexer(stream)
	l.Start()

	expectedValues := []token.Token{
		token.Token{
			Pos:  token.Position{Line: 0, Column: 0, Len: 3},
			Type: token.Identifier,
			Raw:  "abc",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 4, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 6, Len: 3},
			Type: token.Identifier,
			Raw:  "dfe",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 10, Len: 1},
			Type: token.Plus,
			Raw:  "+",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 12, Len: 1},
			Type: token.Integer,
			Raw:  "3",
		},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected != token {
			t.Errorf("Expected: %#v, got: %#v.", expected, token)
		}

		i++
	}
}

func TestStart_MultipleLines(t *testing.T) {
	stream := StreamFromString(`a = 5
b = 4
c = a + b`)
	l := NewLexer(stream)
	l.Start()

	expectedValues := []token.Token{
		token.Token{
			Pos:  token.Position{Line: 0, Column: 0, Len: 1},
			Type: token.Identifier,
			Raw:  "a",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 2, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 4, Len: 1},
			Type: token.Integer,
			Raw:  "5",
		},
		token.Token{
			Pos:  token.Position{Line: 1, Column: 0, Len: 1},
			Type: token.Identifier,
			Raw:  "b",
		},
		token.Token{
			Pos:  token.Position{Line: 1, Column: 2, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		token.Token{
			Pos:  token.Position{Line: 1, Column: 4, Len: 1},
			Type: token.Integer,
			Raw:  "4",
		},
		token.Token{
			Pos:  token.Position{Line: 2, Column: 0, Len: 1},
			Type: token.Identifier,
			Raw:  "c",
		},
		token.Token{
			Pos:  token.Position{Line: 2, Column: 2, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		token.Token{
			Pos:  token.Position{Line: 2, Column: 4, Len: 1},
			Type: token.Identifier,
			Raw:  "a",
		},
		token.Token{
			Pos:  token.Position{Line: 2, Column: 6, Len: 1},
			Type: token.Plus,
			Raw:  "+",
		},
		token.Token{
			Pos:  token.Position{Line: 2, Column: 8, Len: 1},
			Type: token.Identifier,
			Raw:  "b",
		},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected != token {
			t.Errorf("Expected: %#v, got: %#v.", expected, token)
		}

		i++
	}
}

func TestConditions(t *testing.T) {
	stream := StreamFromString("a<=b")
	l := NewLexer(stream)
	l.Start()

	expectedValues := []token.Token{
		token.Token{
			Pos:  token.Position{Line: 0, Column: 0, Len: 1},
			Type: token.Identifier,
			Raw:  "a",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 1, Len: 2},
			Type: token.LessEqual,
			Raw:  "<=",
		},
		token.Token{
			Pos:  token.Position{Line: 0, Column: 3, Len: 1},
			Type: token.Identifier,
			Raw:  "b",
		},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected != token {
			t.Errorf("Expected: %#v, got: %#v.", expected, token)
		}

		i++
	}
}

func TestFunction(t *testing.T) {
	stream := StreamFromString(`
a = (a int, b int) -> int {
	return a + b
}
	`)

	l := NewLexer(stream)
	l.Start()

	expectedValues := []struct {
		Type token.Type
		Raw  string
	}{
		{token.Identifier, "a"},
		{token.Assign, "="},
		{token.ParenthesisOpen, "("},
		{token.Identifier, "a"},
		{token.BuildInType, "int"},
		{token.Comma, ","},
		{token.Identifier, "b"},
		{token.BuildInType, "int"},
		{token.ParenthesisClose, ")"},
		{token.Arrow, "->"},
		{token.BuildInType, "int"},
		{token.CurlyBracketOpen, "{"},
		{token.Keyword, "return"},
		{token.Identifier, "a"},
		{token.Plus, "+"},
		{token.Identifier, "b"},
		{token.CurlyBracketClose, "}"},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected.Raw != token.Raw {
			t.Errorf("Expected Raw: %#v, got: %#v.", expected.Raw, token.Raw)
		}

		if expected.Type != token.Type {
			t.Errorf("Expected Type: %#v, got: %#v.", expected.Type, token.Type)
		}

		i++
	}
}
<commit_msg>gofmt -s<commit_after>package lexer

import (
	"testing"

	"github.com/BenchR267/lbd/lexer/token"
)

func TestIgnoreWhitespace(t *testing.T) {
	stream := StreamFromString("a=    5")
	l := NewLexer(stream)
	l.Start()

	expectedValues := []token.Token{
		{
			Pos:  token.Position{Line: 0, Column: 0, Len: 1},
			Type: token.Identifier,
			Raw:  "a",
		},
		{
			Pos:  token.Position{Line: 0, Column: 1, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		{
			Pos:  token.Position{Line: 0, Column: 6, Len: 1},
			Type: token.Integer,
			Raw:  "5",
		},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected != token {
			t.Errorf("Expected: %#v, got: %#v.", expected, token)
		}

		i++
	}
}

func TestStart_OneLine(t *testing.T) {
	stream := StreamFromString("abc = dfe + 3")
	l := NewLexer(stream)
	l.Start()

	expectedValues := []token.Token{
		{
			Pos:  token.Position{Line: 0, Column: 0, Len: 3},
			Type: token.Identifier,
			Raw:  "abc",
		},
		{
			Pos:  token.Position{Line: 0, Column: 4, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		{
			Pos:  token.Position{Line: 0, Column: 6, Len: 3},
			Type: token.Identifier,
			Raw:  "dfe",
		},
		{
			Pos:  token.Position{Line: 0, Column: 10, Len: 1},
			Type: token.Plus,
			Raw:  "+",
		},
		{
			Pos:  token.Position{Line: 0, Column: 12, Len: 1},
			Type: token.Integer,
			Raw:  "3",
		},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected != token {
			t.Errorf("Expected: %#v, got: %#v.", expected, token)
		}

		i++
	}
}

func TestStart_MultipleLines(t *testing.T) {
	stream := StreamFromString(`a = 5
b = 4
c = a + b`)
	l := NewLexer(stream)
	l.Start()

	expectedValues := []token.Token{
		{
			Pos:  token.Position{Line: 0, Column: 0, Len: 1},
			Type: token.Identifier,
			Raw:  "a",
		},
		{
			Pos:  token.Position{Line: 0, Column: 2, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		{
			Pos:  token.Position{Line: 0, Column: 4, Len: 1},
			Type: token.Integer,
			Raw:  "5",
		},
		{
			Pos:  token.Position{Line: 1, Column: 0, Len: 1},
			Type: token.Identifier,
			Raw:  "b",
		},
		{
			Pos:  token.Position{Line: 1, Column: 2, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		{
			Pos:  token.Position{Line: 1, Column: 4, Len: 1},
			Type: token.Integer,
			Raw:  "4",
		},
		{
			Pos:  token.Position{Line: 2, Column: 0, Len: 1},
			Type: token.Identifier,
			Raw:  "c",
		},
		{
			Pos:  token.Position{Line: 2, Column: 2, Len: 1},
			Type: token.Assign,
			Raw:  "=",
		},
		{
			Pos:  token.Position{Line: 2, Column: 4, Len: 1},
			Type: token.Identifier,
			Raw:  "a",
		},
		{
			Pos:  token.Position{Line: 2, Column: 6, Len: 1},
			Type: token.Plus,
			Raw:  "+",
		},
		{
			Pos:  token.Position{Line: 2, Column: 8, Len: 1},
			Type: token.Identifier,
			Raw:  "b",
		},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected != token {
			t.Errorf("Expected: %#v, got: %#v.", expected, token)
		}

		i++
	}
}

func TestConditions(t *testing.T) {
	stream := StreamFromString("a<=b")
	l := NewLexer(stream)
	l.Start()

	expectedValues := []token.Token{
		{
			Pos:  token.Position{Line: 0, Column: 0, Len: 1},
			Type: token.Identifier,
			Raw:  "a",
		},
		{
			Pos:  token.Position{Line: 0, Column: 1, Len: 2},
			Type: token.LessEqual,
			Raw:  "<=",
		},
		{
			Pos:  token.Position{Line: 0, Column: 3, Len: 1},
			Type: token.Identifier,
			Raw:  "b",
		},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected != token {
			t.Errorf("Expected: %#v, got: %#v.", expected, token)
		}

		i++
	}
}

func TestFunction(t *testing.T) {
	stream := StreamFromString(`
a = (a int, b int) -> int {
	return a + b
}
	`)

	l := NewLexer(stream)
	l.Start()

	expectedValues := []struct {
		Type token.Type
		Raw  string
	}{
		{token.Identifier, "a"},
		{token.Assign, "="},
		{token.ParenthesisOpen, "("},
		{token.Identifier, "a"},
		{token.BuildInType, "int"},
		{token.Comma, ","},
		{token.Identifier, "b"},
		{token.BuildInType, "int"},
		{token.ParenthesisClose, ")"},
		{token.Arrow, "->"},
		{token.BuildInType, "int"},
		{token.CurlyBracketOpen, "{"},
		{token.Keyword, "return"},
		{token.Identifier, "a"},
		{token.Plus, "+"},
		{token.Identifier, "b"},
		{token.CurlyBracketClose, "}"},
	}

	i := 0
	for token := range l.NextToken {
		expected := expectedValues[i]

		if expected.Raw != token.Raw {
			t.Errorf("Expected Raw: %#v, got: %#v.", expected.Raw, token.Raw)
		}

		if expected.Type != token.Type {
			t.Errorf("Expected Type: %#v, got: %#v.", expected.Type, token.Type)
		}

		i++
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"fmt"
	"github.com/cheggaaa/pb"
	"os"
	"path/filepath"
	"sync"
)

// Uploadable describes a file that can be uploaded.
type Uploadable struct {
	OIDPath  string
	Filename string
	CB       CopyCallback
	Size     int64
}

// NewUploadable builds the Uploadable from the given information.
func NewUploadable(oid, filename string, index, totalFiles int) (*Uploadable, *WrappedError) {
	path, err := LocalMediaPath(oid)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	if err := ensureFile(filename, path); err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	fi, err := os.Stat(filename)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	cb, file, _ := CopyCallbackFile("push", filename, index, totalFiles)
	// TODO: fix this, Error() is in `commands`. This is not a fatal error, it should display
	// but not return.
	// if cbErr != nil {
	// 	Error(cbErr.Error())
	// }

	if file != nil {
		defer file.Close()
	}

	return &Uploadable{path, filename, cb, fi.Size()}, nil
}

// UploadQueue provides a queue that will allow concurrent uploads.
type UploadQueue struct {
	uploadc chan *Uploadable
	errorc  chan *WrappedError
	errors  []*WrappedError
	wg      sync.WaitGroup
	workers int
	size    int64
}

// NewUploadQueue builds an UploadQueue, allowing `workers` concurrent uploads.
func NewUploadQueue(workers, files int) *UploadQueue {
	return &UploadQueue{
		uploadc: make(chan *Uploadable, files),
		errorc:  make(chan *WrappedError),
		workers: workers,
	}
}

// Upload adds an Uploadable to the upload queue.
func (q *UploadQueue) Upload(u *Uploadable) {
	q.wg.Add(1)
	q.size += u.Size
	q.uploadc <- u
}

// Process starts the upload queue and displays a progress bar.
func (q *UploadQueue) Process() {
	bar := pb.New64(q.size)
	bar.SetUnits(pb.U_BYTES)
	bar.Start()

	go func() {
		for err := range q.errorc {
			q.errors = append(q.errors, err)
		}
	}()

	for i := 0; i < q.workers; i++ {
		go func(n int) {
			for upload := range q.uploadc {

				cb := func(total, read int64, current int) error {
					bar.Add(current)
					if upload.CB != nil {
						return upload.CB(total, read, current)
					}
					return nil
				}

				err := Upload(upload.OIDPath, upload.Filename, cb)
				if err != nil {
					q.errorc <- err
				}
				q.wg.Done()
			}
		}(i)
	}

	close(q.uploadc)
	q.wg.Wait()
	close(q.errorc)

	bar.Finish()
}

// Errors returns any errors encountered during uploading.
func (q *UploadQueue) Errors() []*WrappedError {
	return q.errors
}

// ensureFile makes sure that the cleanPath exists before pushing it.  If it
// does not exist, it attempts to clean it by reading the file at smudgePath.
func ensureFile(smudgePath, cleanPath string) error {
	if _, err := os.Stat(cleanPath); err == nil {
		return nil
	}

	expectedOid := filepath.Base(cleanPath)
	localPath := filepath.Join(LocalWorkingDir, smudgePath)
	file, err := os.Open(localPath)
	if err != nil {
		return err
	}

	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return err
	}

	cleaned, err := PointerClean(file, stat.Size(), nil)
	if err != nil {
		return err
	}

	cleaned.Close()

	if expectedOid != cleaned.Oid {
		return fmt.Errorf("Expected %s to have an OID of %s, got %s", smudgePath, expectedOid, cleaned.Oid)
	}

	return nil
}
<commit_msg>Use pb's new Prefix() to count completed transfers<commit_after>package lfs

import (
	"fmt"
	"github.com/cheggaaa/pb"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
)

// Uploadable describes a file that can be uploaded.
type Uploadable struct {
	OIDPath  string
	Filename string
	CB       CopyCallback
	Size     int64
}

// NewUploadable builds the Uploadable from the given information.
func NewUploadable(oid, filename string, index, totalFiles int) (*Uploadable, *WrappedError) {
	path, err := LocalMediaPath(oid)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	if err := ensureFile(filename, path); err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	fi, err := os.Stat(filename)
	if err != nil {
		return nil, Errorf(err, "Error uploading file %s (%s)", filename, oid)
	}

	cb, file, _ := CopyCallbackFile("push", filename, index, totalFiles)
	// TODO: fix this, Error() is in `commands`. This is not a fatal error, it should display
	// but not return.
	// if cbErr != nil {
	// 	Error(cbErr.Error())
	// }

	if file != nil {
		defer file.Close()
	}

	return &Uploadable{path, filename, cb, fi.Size()}, nil
}

// UploadQueue provides a queue that will allow concurrent uploads.
type UploadQueue struct {
	uploadc  chan *Uploadable
	errorc   chan *WrappedError
	errors   []*WrappedError
	wg       sync.WaitGroup
	workers  int
	files    int
	finished int64
	size     int64
}

// NewUploadQueue builds an UploadQueue, allowing `workers` concurrent uploads.
func NewUploadQueue(workers, files int) *UploadQueue {
	return &UploadQueue{
		uploadc: make(chan *Uploadable, files),
		errorc:  make(chan *WrappedError),
		workers: workers,
		files:   files,
	}
}

// Upload adds an Uploadable to the upload queue.
func (q *UploadQueue) Upload(u *Uploadable) {
	q.wg.Add(1)
	q.size += u.Size
	q.uploadc <- u
}

// Process starts the upload queue and displays a progress bar.
func (q *UploadQueue) Process() {
	bar := pb.New64(q.size)
	bar.SetUnits(pb.U_BYTES)
	bar.ShowBar = false
	bar.Prefix(fmt.Sprintf("(%d of %d files) ", q.finished, q.files))
	bar.Start()

	go func() {
		for err := range q.errorc {
			q.errors = append(q.errors, err)
		}
	}()

	for i := 0; i < q.workers; i++ {
		go func(n int) {
			for upload := range q.uploadc {
				cb := func(total, read int64, current int) error {
					bar.Add(current)
					if upload.CB != nil {
						return upload.CB(total, read, current)
					}
					return nil
				}

				err := Upload(upload.OIDPath, upload.Filename, cb)
				if err != nil {
					q.errorc <- err
				}
				f := atomic.AddInt64(&q.finished, 1)
				bar.Prefix(fmt.Sprintf("(%d of %d files) ", f, q.files))
				q.wg.Done()
			}
		}(i)
	}

	close(q.uploadc)
	q.wg.Wait()
	close(q.errorc)

	bar.Finish()
}

// Errors returns any errors encountered during uploading.
func (q *UploadQueue) Errors() []*WrappedError {
	return q.errors
}

// ensureFile makes sure that the cleanPath exists before pushing it.  If it
// does not exist, it attempts to clean it by reading the file at smudgePath.
func ensureFile(smudgePath, cleanPath string) error {
	if _, err := os.Stat(cleanPath); err == nil {
		return nil
	}

	expectedOid := filepath.Base(cleanPath)
	localPath := filepath.Join(LocalWorkingDir, smudgePath)
	file, err := os.Open(localPath)
	if err != nil {
		return err
	}

	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return err
	}

	cleaned, err := PointerClean(file, stat.Size(), nil)
	if err != nil {
		return err
	}

	cleaned.Close()

	if expectedOid != cleaned.Oid {
		return fmt.Errorf("Expected %s to have an OID of %s, got %s", smudgePath, expectedOid, cleaned.Oid)
	}

	return nil
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"os"
	"regexp"
	"strconv"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/gengo/goship/lib/pivotal"
	"github.com/golang/glog"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

const (
	gitHubAPITokenEnvVar = "GITHUB_API_TOKEN"
)

// Config is a set of Goship configurations
type Config struct {
	Projects   []Project             `json:"-" yaml:"projects,omitempty"`
	DeployUser string                `json:"deploy_user" yaml:"deploy_user"`
	Notify     string                `json:"notify" yaml:"notify"`
	Pivotal    *PivotalConfiguration `json:"pivotal,omitempty" yaml:"pivotal,omitempty"`
}

// Project stores information about a GitHub project, such as its GitHub URL and repo name, and a list of extra columns (PluginColumns)
type Project struct {
	Name         string `json:"-" yaml:"name"`
	Repo         `json:",inline" yaml:",inline"`
	RepoType     RepositoryType `json:"repo_type" yaml:"repo_type"`
	Environments []Environment  `json:"-" yaml:"envs"`
	TravisToken  string         `json:"travis_token" yaml:"travis_token"`
	// Source is an additional revision control system.
	// It is effective only if RepoType does not serve source codes.
	Source *Repo `json:"source,omitempty" yaml:"source,omitempty"`
}

func (p Project) SourceRepo() Repo {
	if p.Source != nil {
		return *p.Source
	}
	return p.Repo
}

// A RepositoryType describes a type of revision control system which manages target revisions of deployment.
type RepositoryType string

const (
	// RepoTypeGithub means sources codes of the targets of deployment are stored in github and we deploy from the codes.
	RepoTypeGithub = RepositoryType("github")
	// RepoTypeDocker means prebuilt docker images are the targets of deployment.
	RepoTypeDocker = RepositoryType("docker")
)

func (t RepositoryType) Valid() bool {
	switch t {
	case RepoTypeGithub, RepoTypeDocker:
		return true
	}
	return false
}

// Environment stores information about an individual environment, such as its name and whether it is deployable.
type Environment struct {
	Name     string   `json:"-" yaml:"name"`
	Deploy   string   `json:"deploy" yaml:"deploy"`
	RepoPath string   `json:"repo_path" yaml:"repo_path"`
	Hosts    []string `json:"hosts" yaml:"hosts,omitempty"`
	Branch   string   `json:"branch" yaml:"branch"`
	Comment  string   `json:"comment" yaml:"comment"`
	IsLocked bool     `json:"is_locked,omitempty" yaml:"is_locked,omitempty"`
}

// Repo identifies a revision repository
type Repo struct {
	RepoOwner string `json:"repo_owner" yaml:"repo_owner"`
	RepoName  string `json:"repo_name" yaml:"repo_name"`
}

// PivotalConfiguration used to store Pivotal interface
type PivotalConfiguration struct {
	Token string `json:"token" yaml:"token"`
}

func PostToPivotal(piv *PivotalConfiguration, env, owner, name, latest, current string) error {
	layout := "2006-01-02 15:04:05"
	timestamp := time.Now()
	loc, err := time.LoadLocation("Asia/Tokyo")
	if err != nil {
		layout += " (UTC)"
		glog.Error("time zone information for Asia/Tokyo not found")
	} else {
		layout += " (JST)"
		timestamp = timestamp.In(loc)
	}
	ids, err := GetPivotalIDFromCommits(owner, name, latest, current)
	if err != nil {
		return err
	}
	pivClient := pivotal.NewClient(piv.Token)
	for _, id := range ids {
		project, err := pivClient.FindProjectForStory(id)
		if err != nil {
			glog.Errorf("error getting project for story %d: %v", id, err)
			continue
		}
		m := fmt.Sprintf("Deployed to %s: %s", env, timestamp.Format(layout))
		go func() {
			if err := pivClient.AddComment(id, project, m); err != nil {
				glog.Errorf("failed to post a comment %q to story %d", m, id)
			}
		}()
		if env == "live" {
			year, week := time.Now().ISOWeek()
			label := fmt.Sprintf("released_w%d/%d", week, year)
			go func() {
				if err := pivClient.AddLabel(id, project, label); err != nil {
					glog.Errorf("Failed to add a label %q to story %d", label, id)
				}
			}()
		}
	}
	return nil
}

func appendIfUnique(list []int, elem int) []int {
	for _, item := range list {
		if item == elem {
			return list
		}
	}
	return append(list, elem)
}

func GetPivotalIDFromCommits(owner, repoName, latest, current string) ([]int, error) {
	// gets a list pivotal IDs from commit messages from repository based on latest and current commit
	gt := os.Getenv(gitHubAPITokenEnvVar)
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: gt})
	c := github.NewClient(oauth2.NewClient(oauth2.NoContext, ts))
	comp, _, err := c.Repositories.CompareCommits(owner, repoName, current, latest)
	if err != nil {
		return nil, err
	}
	pivRE, err := regexp.Compile("\\[.*#(\\d+)\\].*")
	if err != nil {
		return nil, err
	}
	var pivotalIDs []int
	for _, commit := range comp.Commits {
		cmi := *commit.Commit
		cm := *cmi.Message
		ids := pivRE.FindStringSubmatch(cm)
		if ids != nil {
			id := ids[1]
			n, err := strconv.Atoi(id)
			if err != nil {
				return nil, err
			}
			pivotalIDs = appendIfUnique(pivotalIDs, n)
		}
	}
	return pivotalIDs, nil
}

// ProjectFromName takes a project name as a string and returns
// a project by that name if it can find one.
func ProjectFromName(projects []Project, projectName string) (Project, error) {
	for _, project := range projects {
		if project.Name == projectName {
			return project, nil
		}
	}
	return Project{}, fmt.Errorf("No project found: %s", projectName)
}

// EnvironmentFromName takes an environment and project name as a string and returns
// an environment by the given environment name under a project with the given
// project name if it can find one.
func EnvironmentFromName(projects []Project, projectName, environmentName string) (*Environment, error) {
	p, err := ProjectFromName(projects, projectName)
	if err != nil {
		return nil, err
	}
	for _, environment := range p.Environments {
		if environment.Name == environmentName {
			return &environment, nil
		}
	}
	return nil, fmt.Errorf("No environment found: %s", environmentName)
}

// ETCDInterface emulates ETCD to allow testing
type ETCDInterface interface {
	Get(string, bool, bool) (*etcd.Response, error)
	Set(string, string, uint64) (*etcd.Response, error)
}
<commit_msg>pivotal_comment_fix<commit_after>package config

import (
	"fmt"
	"os"
	"regexp"
	"strconv"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/gengo/goship/lib/pivotal"
	"github.com/golang/glog"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

const (
	gitHubAPITokenEnvVar = "GITHUB_API_TOKEN"
)

// Config is a set of Goship configurations
type Config struct {
	Projects   []Project             `json:"-" yaml:"projects,omitempty"`
	DeployUser string                `json:"deploy_user" yaml:"deploy_user"`
	Notify     string                `json:"notify" yaml:"notify"`
	Pivotal    *PivotalConfiguration `json:"pivotal,omitempty" yaml:"pivotal,omitempty"`
}

// Project stores information about a GitHub project, such as its GitHub URL and repo name, and a list of extra columns (PluginColumns)
type Project struct {
	Name         string `json:"-" yaml:"name"`
	Repo         `json:",inline" yaml:",inline"`
	RepoType     RepositoryType `json:"repo_type" yaml:"repo_type"`
	Environments []Environment  `json:"-" yaml:"envs"`
	TravisToken  string         `json:"travis_token" yaml:"travis_token"`
	// Source is an additional revision control system.
	// It is effective only if RepoType does not serve source codes.
	Source *Repo `json:"source,omitempty" yaml:"source,omitempty"`
}

func (p Project) SourceRepo() Repo {
	if p.Source != nil {
		return *p.Source
	}
	return p.Repo
}

// A RepositoryType describes a type of revision control system which manages target revisions of deployment.
type RepositoryType string

const (
	// RepoTypeGithub means sources codes of the targets of deployment are stored in github and we deploy from the codes.
	RepoTypeGithub = RepositoryType("github")
	// RepoTypeDocker means prebuilt docker images are the targets of deployment.
	RepoTypeDocker = RepositoryType("docker")
)

func (t RepositoryType) Valid() bool {
	switch t {
	case RepoTypeGithub, RepoTypeDocker:
		return true
	}
	return false
}

// Environment stores information about an individual environment, such as its name and whether it is deployable.
type Environment struct {
	Name     string   `json:"-" yaml:"name"`
	Deploy   string   `json:"deploy" yaml:"deploy"`
	RepoPath string   `json:"repo_path" yaml:"repo_path"`
	Hosts    []string `json:"hosts" yaml:"hosts,omitempty"`
	Branch   string   `json:"branch" yaml:"branch"`
	Comment  string   `json:"comment" yaml:"comment"`
	IsLocked bool     `json:"is_locked,omitempty" yaml:"is_locked,omitempty"`
}

// Repo identifies a revision repository
type Repo struct {
	RepoOwner string `json:"repo_owner" yaml:"repo_owner"`
	RepoName  string `json:"repo_name" yaml:"repo_name"`
}

// PivotalConfiguration used to store Pivotal interface
type PivotalConfiguration struct {
	Token string `json:"token" yaml:"token"`
}

func PostToPivotal(piv *PivotalConfiguration, env, owner, name, current, latest string) error {
	layout := "2006-01-02 15:04:05"
	timestamp := time.Now()
	loc, err := time.LoadLocation("Asia/Tokyo")
	if err != nil {
		layout += " (UTC)"
		glog.Error("time zone information for Asia/Tokyo not found")
	} else {
		layout += " (JST)"
		timestamp = timestamp.In(loc)
	}
	ids, err := GetPivotalIDFromCommits(owner, name, current, latest)
	if err != nil {
		return err
	}
	pivClient := pivotal.NewClient(piv.Token)
	for _, id := range ids {
		project, err := pivClient.FindProjectForStory(id)
		if err != nil {
			glog.Errorf("error getting project for story %d: %v", id, err)
			continue
		}
		m := fmt.Sprintf("Deployed to %s: %s", env, timestamp.Format(layout))
		go func() {
			if err := pivClient.AddComment(id, project, m); err != nil {
				glog.Errorf("failed to post a comment %q to story %d", m, id)
			}
		}()
		if env == "live" {
			year, week := time.Now().ISOWeek()
			label := fmt.Sprintf("released_w%d/%d", week, year)
			go func() {
				if err := pivClient.AddLabel(id, project, label); err != nil {
					glog.Errorf("Failed to add a label %q to story %d", label, id)
				}
			}()
		}
	}
	return nil
}

func appendIfUnique(list []int, elem int) []int {
	for _, item := range list {
		if item == elem {
			return list
		}
	}
	return append(list, elem)
}

func GetPivotalIDFromCommits(owner, repoName, current, latest string) ([]int, error) {
	// gets a list pivotal IDs from commit messages from repository based on latest and current commit
	gt := os.Getenv(gitHubAPITokenEnvVar)
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: gt})
	c := github.NewClient(oauth2.NewClient(oauth2.NoContext, ts))
	comp, _, err := c.Repositories.CompareCommits(owner, repoName, current, latest)
	if err != nil {
		return nil, err
	}
	pivRE, err := regexp.Compile("\\[.*#(\\d+)\\].*")
	if err != nil {
		return nil, err
	}
	var pivotalIDs []int
	for _, commit := range comp.Commits {
		cmi := *commit.Commit
		cm := *cmi.Message
		ids := pivRE.FindStringSubmatch(cm)
		if ids != nil {
			id := ids[1]
			n, err := strconv.Atoi(id)
			if err != nil {
				return nil, err
			}
			pivotalIDs = appendIfUnique(pivotalIDs, n)
		}
	}
	return pivotalIDs, nil
}

// ProjectFromName takes a project name as a string and returns
// a project by that name if it can find one.
func ProjectFromName(projects []Project, projectName string) (Project, error) {
	for _, project := range projects {
		if project.Name == projectName {
			return project, nil
		}
	}
	return Project{}, fmt.Errorf("No project found: %s", projectName)
}

// EnvironmentFromName takes an environment and project name as a string and returns
// an environment by the given environment name under a project with the given
// project name if it can find one.
func EnvironmentFromName(projects []Project, projectName, environmentName string) (*Environment, error) {
	p, err := ProjectFromName(projects, projectName)
	if err != nil {
		return nil, err
	}
	for _, environment := range p.Environments {
		if environment.Name == environmentName {
			return &environment, nil
		}
	}
	return nil, fmt.Errorf("No environment found: %s", environmentName)
}

// ETCDInterface emulates ETCD to allow testing
type ETCDInterface interface {
	Get(string, bool, bool) (*etcd.Response, error)
	Set(string, string, uint64) (*etcd.Response, error)
}
<|endoftext|>


<commit_before>package env

import (
	"bytes"
	"compress/zlib"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
)

// A list of keys we don't want to deal with
var IGNORED_KEYS = []string{"_", "PWD", "OLDPWD", "SHLVL", "SHELL"}

func IgnoredKey(key string) bool {
	if len(key) > 4 && key[:5] == "DIRENV_" {
		return true
	}
	// FIXME: Is there a higher-level function for that ? Eg. indexOf in JavaScript
	for _, ikey := range IGNORED_KEYS {
		if ikey == key {
			return true
		}
	}
	return false
}

type Env map[string]string

// FIXME: apparently it's possible to have two variables with the
//        same name in the env but I never seen that.
//        If that happens I might have to change the return
//        type signature
func FilteredEnv() Env {
	env := make(Env)

	for _, kv := range os.Environ() {
		kv2 := strings.SplitN(kv, "=", 2)
		// Is there a better way to deconstruct a tuple ?
		key := kv2[0]
		value := kv2[1]

		if !IgnoredKey(key) {
			env[key] = value
		}
	}

	return env
}

func ParseEnv(base64env string) (Env, error) {
	zlibData, err := base64.URLEncoding.DecodeString(base64env)
	if err != nil {
		return nil, fmt.Errorf("base64 decoding: %v", err)
	}

	zlibReader := bytes.NewReader(zlibData)
	w, err := zlib.NewReader(zlibReader)
	if err != nil {
		return nil, fmt.Errorf("zlib opening: %v", err)
	}

	envData := new(bytesReceiver)
	io.Copy(envData, w)
	//_, err = io.Copy(w, envData)
	if err != nil {
		return nil, fmt.Errorf("zlib decoding: %v", err)
	}
	w.Close()

	env := make(Env)
	err = json.Unmarshal(envData.Bytes(), &env)
	if err != nil {
		return nil, fmt.Errorf("json parsing: %v", err)
	}

	return env, nil
}

func (env Env) Serialize() (string, error) {
	jsonData, err := json.Marshal(env)
	if err != nil {
		return "", err
	}

	var zlibData bytes.Buffer
	w := zlib.NewWriter(&zlibData)
	w.Write(jsonData)
	w.Close()

	base64Data := base64.URLEncoding.EncodeToString(zlibData.Bytes())

	return base64Data, nil
}

// Erk. There's probably something in the stdlib that does what this
// micro type does.
type bytesReceiver struct {
	bytes []byte
}

func (self *bytesReceiver) Bytes() []byte {
	return self.bytes
}

func (self *bytesReceiver) Write(data []byte) (int, error) {
	// HACK: We know only one write is going to happen :)
	// ..... but still make sure the assertion is correct
	if len(self.bytes) > 0 {
		return 0, errors.New("You fool, thinking you can be right")
	}
	self.bytes = data
	return len(data), nil
}
<commit_msg>Get rid of our customer bytes buffer. Thanks @pwaller<commit_after>package env

import (
	"bytes"
	"compress/zlib"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
)

// A list of keys we don't want to deal with
var IGNORED_KEYS = []string{"_", "PWD", "OLDPWD", "SHLVL", "SHELL"}

func IgnoredKey(key string) bool {
	if len(key) > 4 && key[:5] == "DIRENV_" {
		return true
	}
	// FIXME: Is there a higher-level function for that ? Eg. indexOf in JavaScript
	for _, ikey := range IGNORED_KEYS {
		if ikey == key {
			return true
		}
	}
	return false
}

type Env map[string]string

// FIXME: apparently it's possible to have two variables with the
//        same name in the env but I never seen that.
//        If that happens I might have to change the return
//        type signature
func FilteredEnv() Env {
	env := make(Env)

	for _, kv := range os.Environ() {
		kv2 := strings.SplitN(kv, "=", 2)
		// Is there a better way to deconstruct a tuple ?
		key := kv2[0]
		value := kv2[1]

		if !IgnoredKey(key) {
			env[key] = value
		}
	}

	return env
}

func ParseEnv(base64env string) (Env, error) {
	zlibData, err := base64.URLEncoding.DecodeString(base64env)
	if err != nil {
		return nil, fmt.Errorf("base64 decoding: %v", err)
	}

	zlibReader := bytes.NewReader(zlibData)
	w, err := zlib.NewReader(zlibReader)
	if err != nil {
		return nil, fmt.Errorf("zlib opening: %v", err)
	}

	envData := bytes.NewBuffer([]byte{})
	_, err = io.Copy(envData, w)
	if err != nil {
		return nil, fmt.Errorf("zlib decoding: %v", err)
	}
	w.Close()

	env := make(Env)
	err = json.Unmarshal(envData.Bytes(), &env)
	if err != nil {
		return nil, fmt.Errorf("json parsing: %v", err)
	}

	return env, nil
}

func (env Env) Serialize() (string, error) {
	jsonData, err := json.Marshal(env)
	if err != nil {
		return "", err
	}

	zlibData := bytes.NewBuffer([]byte{})
	w := zlib.NewWriter(zlibData)
	w.Write(jsonData)
	w.Close()

	base64Data := base64.URLEncoding.EncodeToString(zlibData.Bytes())

	return base64Data, nil
}
<|endoftext|>


<commit_before>package query

import (
	"fmt"
	"strings"
	"time"

	"github.com/pkg/errors"

	"github.com/yuuki/diamondb/lib/series"
	"github.com/yuuki/diamondb/lib/storage"
)

// UnsupportedFunctionError represents the error of unsupported query function.
type UnsupportedFunctionError struct {
	funcName string
}

// Error returns the error message for UnsupportedFunctionError.
// UnsupportedFunctionError satisfies error interface.
func (e *UnsupportedFunctionError) Error() string {
	return fmt.Sprintf("unsupported function %s", e.funcName)
}

type UnknownExpressionError struct {
	expr Expr
}

func (e *UnknownExpressionError) Error() string {
	return fmt.Sprintf("unknown expression %v", e.expr)
}

type funcArg struct {
	expr        Expr
	seriesSlice series.SeriesSlice
}

type funcArgs []*funcArg

// EvalTargets evaluates the targets concurrently. It is guaranteed that the order
// of the targets as input value and SeriesSlice as retuen value is the same.
func EvalTargets(fetcher storage.Fetcher, targets []string, startTime, endTime time.Time) (series.SeriesSlice, error) {
	type result struct {
		value series.SeriesSlice
		err   error
		index int
	}

	c := make(chan *result)
	for i, target := range targets {
		go func(target string, start, end time.Time, i int) {
			ss, err := EvalTarget(fetcher, target, start, end)
			c <- &result{value: ss, err: err, index: i}
		}(target, startTime, endTime, i)
	}
	ordered := make([]series.SeriesSlice, len(targets))
	for i := 0; i < len(targets); i++ {
		ret := <-c
		if ret.err != nil {
			// return err that is found firstly.
			return nil, errors.Wrapf(ret.err, "failed to evaluate target (%s)", targets[i])
		}
		ordered[ret.index] = ret.value
	}
	results := series.SeriesSlice{}
	for _, ss := range ordered {
		results = append(results, ss...)
	}
	return results, nil
}

// EvalTarget evaluates the target. It parses the target into AST structure and fetches datapoints from storage.
//
// ex. target: "alias(sumSeries(server1.loadavg5,server2.loadavg5),\"server_loadavg5\")"
func EvalTarget(fetcher storage.Fetcher, target string, startTime, endTime time.Time) (series.SeriesSlice, error) {
	expr, err := ParseTarget(target)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to parse target (%s)", target)
	}
	ss, err := invokeExpr(fetcher, expr, startTime, endTime)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to invoke %s", expr)
	}
	return ss, err
}

func invokeExpr(fetcher storage.Fetcher, expr Expr, startTime, endTime time.Time) (series.SeriesSlice, error) {
	switch e := expr.(type) {
	case SeriesListExpr:
		ss, err := fetcher.Fetch(e.Literal, startTime, endTime)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to fetch (%s,%d,%d)", e.Literal, startTime.Unix(), endTime.Unix())
		}
		return ss, nil
	case GroupSeriesExpr:
		joinedValues := make([]string, 0, len(e.ValueList))
		for _, value := range e.ValueList {
			joinedValues = append(joinedValues, e.Prefix+value+e.Postfix)
		}
		expr = SeriesListExpr{Literal: strings.Join(joinedValues, ",")}
		ss, err := invokeExpr(fetcher, expr, startTime, endTime)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to invoke (%s,%d,%d)", e, startTime.Unix(), endTime.Unix())
		}
		return ss, nil
	case FuncExpr:
		args := funcArgs{}
		for _, expr := range e.SubExprs {
			switch e2 := expr.(type) {
			case BoolExpr:
				args = append(args, &funcArg{expr: expr})
			case NumberExpr:
				args = append(args, &funcArg{expr: expr})
			case StringExpr:
				args = append(args, &funcArg{expr: expr})
			case SeriesListExpr, GroupSeriesExpr:
				ss, err := invokeExpr(fetcher, expr, startTime, endTime)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to invoke %s", e2)
				}
				ex := SeriesListExpr{Literal: ss.FormattedName()}
				args = append(args, &funcArg{expr: ex, seriesSlice: ss})
			case FuncExpr:
				ss, err := invokeExpr(fetcher, expr, startTime, endTime)
				if err != nil {
					return nil, errors.Wrapf(err, "failed to invoke %s", e2)
				}
				// Regard FuncExpr as SeriesListExpr after process function
				ex := SeriesListExpr{Literal: fmt.Sprintf("%s(%s)", e2.Name, ss.FormattedName())}
				args = append(args, &funcArg{expr: ex, seriesSlice: ss})
			default:
				return nil, &UnknownExpressionError{expr: expr}
			}
		}
		switch e.Name {
		case "alias":
			ss, err := doAlias(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "offset":
			ss, err := doOffset(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "group":
			ss, err := doGroup(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "averageSeries", "avg":
			ss, err := doAverageSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "sumSeries", "sum":
			ss, err := doSumSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "minSeries":
			ss, err := doMinSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "maxSeries":
			ss, err := doMaxSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "multiplySeries":
			ss, err := doMultiplySeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "divideSeries":
			ss, err := doDivideSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "percentileOfSeries":
			ss, err := doPercentileOfSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "summarize":
			ss, err := doSummarize(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "sumSeriesWithWildcards":
			ss, err := doSumSeriesWithWildcards(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		default:
			return nil, &UnsupportedFunctionError{funcName: e.Name}
		}
	default:
		return nil, &UnknownExpressionError{expr: expr}
	}
}
<commit_msg>Move process for subexprs to invokeSubExprs function<commit_after>package query

import (
	"fmt"
	"strings"
	"time"

	"github.com/pkg/errors"

	"github.com/yuuki/diamondb/lib/series"
	"github.com/yuuki/diamondb/lib/storage"
)

// UnsupportedFunctionError represents the error of unsupported query function.
type UnsupportedFunctionError struct {
	funcName string
}

// Error returns the error message for UnsupportedFunctionError.
// UnsupportedFunctionError satisfies error interface.
func (e *UnsupportedFunctionError) Error() string {
	return fmt.Sprintf("unsupported function %s", e.funcName)
}

type UnknownExpressionError struct {
	expr Expr
}

func (e *UnknownExpressionError) Error() string {
	return fmt.Sprintf("unknown expression %v", e.expr)
}

type funcArg struct {
	expr        Expr
	seriesSlice series.SeriesSlice
}

type funcArgs []*funcArg

// EvalTargets evaluates the targets concurrently. It is guaranteed that the order
// of the targets as input value and SeriesSlice as retuen value is the same.
func EvalTargets(fetcher storage.Fetcher, targets []string, startTime, endTime time.Time) (series.SeriesSlice, error) {
	type result struct {
		value series.SeriesSlice
		err   error
		index int
	}

	c := make(chan *result)
	for i, target := range targets {
		go func(target string, start, end time.Time, i int) {
			ss, err := EvalTarget(fetcher, target, start, end)
			c <- &result{value: ss, err: err, index: i}
		}(target, startTime, endTime, i)
	}
	ordered := make([]series.SeriesSlice, len(targets))
	for i := 0; i < len(targets); i++ {
		ret := <-c
		if ret.err != nil {
			// return err that is found firstly.
			return nil, errors.Wrapf(ret.err, "failed to evaluate target (%s)", targets[i])
		}
		ordered[ret.index] = ret.value
	}
	results := series.SeriesSlice{}
	for _, ss := range ordered {
		results = append(results, ss...)
	}
	return results, nil
}

// EvalTarget evaluates the target. It parses the target into AST structure and fetches datapoints from storage.
//
// ex. target: "alias(sumSeries(server1.loadavg5,server2.loadavg5),\"server_loadavg5\")"
func EvalTarget(fetcher storage.Fetcher, target string, startTime, endTime time.Time) (series.SeriesSlice, error) {
	expr, err := ParseTarget(target)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to parse target (%s)", target)
	}
	ss, err := invokeExpr(fetcher, expr, startTime, endTime)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to invoke %s", expr)
	}
	return ss, err
}

func invokeExpr(fetcher storage.Fetcher, expr Expr, startTime, endTime time.Time) (series.SeriesSlice, error) {
	switch e := expr.(type) {
	case SeriesListExpr:
		ss, err := fetcher.Fetch(e.Literal, startTime, endTime)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to fetch (%s,%d,%d)", e.Literal, startTime.Unix(), endTime.Unix())
		}
		return ss, nil
	case GroupSeriesExpr:
		joinedValues := make([]string, 0, len(e.ValueList))
		for _, value := range e.ValueList {
			joinedValues = append(joinedValues, e.Prefix+value+e.Postfix)
		}
		expr = SeriesListExpr{Literal: strings.Join(joinedValues, ",")}
		ss, err := invokeExpr(fetcher, expr, startTime, endTime)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to invoke (%s,%d,%d)", e, startTime.Unix(), endTime.Unix())
		}
		return ss, nil
	case FuncExpr:
		args, err := invokeSubExprs(fetcher, e.SubExprs, startTime, endTime)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to invoke arguments of (%s) function", e.Name)
		}
		switch e.Name {
		case "alias":
			ss, err := doAlias(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "offset":
			ss, err := doOffset(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "group":
			ss, err := doGroup(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "averageSeries", "avg":
			ss, err := doAverageSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "sumSeries", "sum":
			ss, err := doSumSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "minSeries":
			ss, err := doMinSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "maxSeries":
			ss, err := doMaxSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "multiplySeries":
			ss, err := doMultiplySeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "divideSeries":
			ss, err := doDivideSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "percentileOfSeries":
			ss, err := doPercentileOfSeries(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "summarize":
			ss, err := doSummarize(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		case "sumSeriesWithWildcards":
			ss, err := doSumSeriesWithWildcards(args)
			if err != nil {
				return nil, errors.WithStack(err)
			}
			return ss, err
		default:
			return nil, &UnsupportedFunctionError{funcName: e.Name}
		}
	default:
		return nil, &UnknownExpressionError{expr: expr}
	}
}

func invokeSubExprs(fetcher storage.Fetcher, exprs []Expr, startTime, endTime time.Time) (funcArgs, error) {
	args := funcArgs{}
	for _, expr := range exprs {
		switch e := expr.(type) {
		case BoolExpr, NumberExpr, StringExpr:
			args = append(args, &funcArg{expr: e})
		case SeriesListExpr, GroupSeriesExpr, FuncExpr:
			ss, err := invokeExpr(fetcher, e, startTime, endTime)
			if err != nil {
				return nil, errors.Wrapf(err, "failed to invoke %s", e)
			}
			args = append(args, &funcArg{
				expr:        SeriesListExpr{Literal: ss.FormattedName()},
				seriesSlice: ss,
			})
		default:
			return nil, &UnknownExpressionError{expr: expr}
		}
	}
	return args, nil
}
<|endoftext|>


<commit_before>package lntypes

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
)

// PreimageSize of array used to store preimagees.
const PreimageSize = 32

// Preimage is used in several of the lightning messages and common structures.
// It represents a payment preimage.
type Preimage [PreimageSize]byte

// String returns the Preimage as a hexadecimal string.
func (p Preimage) String() string {
	return hex.EncodeToString(p[:])
}

// MakePreimage returns a new Preimage from a bytes slice. An error is returned
// if the number of bytes passed in is not PreimageSize.
func MakePreimage(newPreimage []byte) (Preimage, error) {
	nhlen := len(newPreimage)
	if nhlen != PreimageSize {
		return Preimage{}, fmt.Errorf("invalid preimage length of %v, "+
			"want %v", nhlen, PreimageSize)
	}

	var preimage Preimage
	copy(preimage[:], newPreimage)

	return preimage, nil
}

// MakePreimageFromStr creates a Preimage from a hex preimage string.
func MakePreimageFromStr(newPreimage string) (Preimage, error) {
	// Return error if preimage string is of incorrect length.
	if len(newPreimage) != PreimageSize*2 {
		return Preimage{}, fmt.Errorf("invalid preimage string length "+
			"of %v, want %v", len(newPreimage), PreimageSize*2)
	}

	preimage, err := hex.DecodeString(newPreimage)
	if err != nil {
		return Preimage{}, err
	}

	return MakePreimage(preimage)
}

// Hash returns the sha256 hash of the preimage.
func (p *Preimage) Hash() Hash {
	return Hash(sha256.Sum256(p[:]))
}
<commit_msg>lntypes: add preimage Matches method<commit_after>package lntypes

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
)

// PreimageSize of array used to store preimagees.
const PreimageSize = 32

// Preimage is used in several of the lightning messages and common structures.
// It represents a payment preimage.
type Preimage [PreimageSize]byte

// String returns the Preimage as a hexadecimal string.
func (p Preimage) String() string {
	return hex.EncodeToString(p[:])
}

// MakePreimage returns a new Preimage from a bytes slice. An error is returned
// if the number of bytes passed in is not PreimageSize.
func MakePreimage(newPreimage []byte) (Preimage, error) {
	nhlen := len(newPreimage)
	if nhlen != PreimageSize {
		return Preimage{}, fmt.Errorf("invalid preimage length of %v, "+
			"want %v", nhlen, PreimageSize)
	}

	var preimage Preimage
	copy(preimage[:], newPreimage)

	return preimage, nil
}

// MakePreimageFromStr creates a Preimage from a hex preimage string.
func MakePreimageFromStr(newPreimage string) (Preimage, error) {
	// Return error if preimage string is of incorrect length.
	if len(newPreimage) != PreimageSize*2 {
		return Preimage{}, fmt.Errorf("invalid preimage string length "+
			"of %v, want %v", len(newPreimage), PreimageSize*2)
	}

	preimage, err := hex.DecodeString(newPreimage)
	if err != nil {
		return Preimage{}, err
	}

	return MakePreimage(preimage)
}

// Hash returns the sha256 hash of the preimage.
func (p *Preimage) Hash() Hash {
	return Hash(sha256.Sum256(p[:]))
}

// Matches returns whether this preimage is the preimage of the given hash.
func (p *Preimage) Matches(h Hash) bool {
	return h == p.Hash()
}
<|endoftext|>