<commit_before>package datapackage

import (
	"bufio"
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
)

const (
	PackRecurse  = 1 << iota
	PackCalcHash = 1 << iota
)

type Packer struct {
	Visiter
	w        *bufio.Writer
	f        *os.File
	recurse  bool
	data     Datapackage
	pattern  []string
	calchash bool
}

func newString(s string) *string {
	st := s
	return &st
}

func processFile(path string, info os.FileInfo, calc_hash bool) (*Resource, error) {
	res := &Resource{}
	var rawdata *[]byte

	res.Path = newString(path)
	ext := filepath.Ext(path)
	res.Format = newString(ext)

	res.Name = newString(path) // TODO: realy use the filepath as the Name?

	size := info.Size()
	res.Bytes = &size

	switch ext {
	case "csv":
		res.Mediatype = newString("text/csv")
	}

	if res.Mediatype == nil {
		// TODO: try to determine the media type by eg. unix `file`?
	}

	if calc_hash {
		if rawdata == nil {
			rdata, err := ioutil.ReadFile(path)
			if err != nil {
				return nil, err
			}
			rawdata = &rdata
		}
		res.Hash = newString(MD5SumforData(*rawdata))
	}

	res.Modified = &ISO8601{Time: info.ModTime()}

	return res, nil
}

func (p *Packer) walkerFunc(path string, info os.FileInfo, err error) error {
	if path == "." || path == Filename {
		return nil
	}
	// when it's a directory, check if directories should be processed recursively
	if info.IsDir() {
		if !p.recurse {
			return filepath.SkipDir
		} else {
			// directory hit. Signal to recurse
			return nil
		}
	}
	// os it's a regular file, process it
	match := true
	// If a pattern should be matched, check if the received file matches the pattern
	if len(p.pattern) > 0 {
		var err error
		for _, pattern := range p.pattern {
			match, err = filepath.Match(pattern, path)
			if err != nil {
				return err
			}
			if match {
				break
			}
		}
	}

	if match {
		res, err := processFile(path, info, p.calchash)
		if err != nil {
			return err
		}
		p.data.Resources = append(p.data.Resources, *res)
	}
	return nil
}

func (p *Packer) Init() (filepath.WalkFunc, error) {
	// TODO: check if realy sensible
	// if a datapackage.json - file already exists, fail
	f, err := os.OpenFile(Filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0664)
	if err != nil {
		return nil, err
	}
	p.w = bufio.NewWriter(f)
	p.f = f
	return p.walkerFunc, nil
}

func (p *Packer) TearDown() error {
	enc := json.NewEncoder(p.w)
	if err := enc.Encode(p.data); err != nil {
		return err
	}
	p.w.Flush()
	return p.f.Close()
}

// NewPacker initialises a new datapackage Packer. The operation mode has to be
// provided using the Pack... constants
func NewPacker(om int) *Packer {
	return &Packer{calchash: om&PackCalcHash == 1, recurse: om&PackRecurse == 1}
}
<commit_msg>new flag<commit_after>package datapackage

import (
	"bufio"
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
)

const (
	PackRecurse  = 1 << iota
	PackCalcHash = 1 << iota
	PackInline   = 1 << iota
)

type Packer struct {
	Visiter
	w        *bufio.Writer
	f        *os.File
	recurse  bool
	data     Datapackage
	pattern  []string
	calchash bool
}

func newString(s string) *string {
	st := s
	return &st
}

func processFile(path string, info os.FileInfo, calc_hash bool) (*Resource, error) {
	res := &Resource{}
	var rawdata *[]byte

	res.Path = newString(path)
	ext := filepath.Ext(path)
	res.Format = newString(ext)

	res.Name = newString(path) // TODO: realy use the filepath as the Name?

	size := info.Size()
	res.Bytes = &size

	switch ext {
	case "csv":
		res.Mediatype = newString("text/csv")
	}

	if res.Mediatype == nil {
		// TODO: try to determine the media type by eg. unix `file`?
	}

	if calc_hash {
		if rawdata == nil {
			rdata, err := ioutil.ReadFile(path)
			if err != nil {
				return nil, err
			}
			rawdata = &rdata
		}
		res.Hash = newString(MD5SumforData(*rawdata))
	}

	res.Modified = &ISO8601{Time: info.ModTime()}

	return res, nil
}

func (p *Packer) walkerFunc(path string, info os.FileInfo, err error) error {
	if path == "." || path == Filename {
		return nil
	}
	// when it's a directory, check if directories should be processed recursively
	if info.IsDir() {
		if !p.recurse {
			return filepath.SkipDir
		} else {
			// directory hit. Signal to recurse
			return nil
		}
	}
	// os it's a regular file, process it
	match := true
	// If a pattern should be matched, check if the received file matches the pattern
	if len(p.pattern) > 0 {
		var err error
		for _, pattern := range p.pattern {
			match, err = filepath.Match(pattern, path)
			if err != nil {
				return err
			}
			if match {
				break
			}
		}
	}

	if match {
		res, err := processFile(path, info, p.calchash)
		if err != nil {
			return err
		}
		p.data.Resources = append(p.data.Resources, *res)
	}
	return nil
}

func (p *Packer) Init() (filepath.WalkFunc, error) {
	// TODO: check if realy sensible
	// if a datapackage.json - file already exists, fail
	f, err := os.OpenFile(Filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0664)
	if err != nil {
		return nil, err
	}
	p.w = bufio.NewWriter(f)
	p.f = f
	return p.walkerFunc, nil
}

func (p *Packer) TearDown() error {
	enc := json.NewEncoder(p.w)
	if err := enc.Encode(p.data); err != nil {
		return err
	}
	p.w.Flush()
	return p.f.Close()
}

// NewPacker initialises a new datapackage Packer. The operation mode has to be
// provided using the Pack... constants
func NewPacker(om int) *Packer {
	return &Packer{calchash: om&PackCalcHash == 1, recurse: om&PackRecurse == 1}
}
<|endoftext|>


<commit_before>package main

// This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.

import (
	"bufio"
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
)

// GitObjectType Represents the type of a git object. Content is in "blob"
// types, everything else is metadata.
type GitObjectType string

const (
	// GitBlob represents content
	GitBlob = "blob"
	// GitTree represents a 'filesystem' in Git
	GitTree = "tree"
	// GitCommit represents a snapshot of state with comment
	GitCommit = "commit"
	// GitTag represents a named snapshot of state with comment
	GitTag = "tag"
)

// GitObject is a metadata blob describing everything about an object in git
// but the content
type GitObject struct {
	Permission uint32
	ObjectType GitObjectType
	Hash       string
	Name       string
}

func pathRsplit(path string) (first, rest string) {
	if path == "" || path == "/" {
		return "", ""
	}
	i := strings.Index(path, "/")
	if i == 0 {
		return pathRsplit(path[1:]) // Just... forget handling this
	} else if i == -1 {
		return path, ""
	}
	return path[:i], path[i+1:]
}

func gitShow(hash string) ([]byte, error) {
	cmd := exec.Command("git", "show", hash)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	err = cmd.Start()
	if err != nil {
		return nil, err
	}
	return ioutil.ReadAll(stdout)
}

func getRefs() ([]string, error) {
	var refs []string
	refsCmd := exec.Command("git", "show-ref")
	stdout, err := refsCmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	err = refsCmd.Start()
	if err != nil {
		return nil, err
	}
	// Should consider hoisting this
	refsR, err := regexp.Compile(`^([0-9a-f]{40})\s+(.+)`)
	if err != nil {
		return nil, err
	}
	bufStdout := bufio.NewReader(stdout)
	for line, err := bufStdout.ReadBytes('\n'); err == nil; line, err = bufStdout.ReadBytes('\n') {
		results := refsR.FindSubmatch(line)
		if len(results) != 3 {
			return nil, fmt.Errorf("Confused by your refs- got %d matches out of something that's supposed to have 2 fields. Line: %s Results: %q", len(results), line, results)
		}
		refs = append(refs, string(results[2][len("refs/"):]))
	}
	return refs, err
}

func gitList(hash, startingString string) ([]byte, error) {
	objects, err := lstree(hash)
	if err != nil {
		return nil, err
	}
	t, err := template.New("list").Parse(`

	{{ $prefix := .Prefix }}
	{{- range $element := .Objects}}

{{$element.Name}}
	{{- end}}

	`)
	if err != nil {
		return nil, err
	}
	var buf bytes.Buffer
	var data = struct {
		Objects []GitObject
		Prefix  string
	}{objects, startingString}
	err = t.Execute(&buf, data)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), err
}

func lstree(commit string) ([]GitObject, error) {
	// Inspect the cwd
	cmd := exec.Command("git", "ls-tree", commit)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	err = cmd.Start()
	if err != nil {
		return nil, err
	}
	bufStdout := bufio.NewReader(stdout)

	var obs []GitObject
	// for reasons I do not understand, saying "this ends with a newline" means
	// the newline gets eaten by the final capture. Also, ending with a '$' breaks
	// the whole match.
	treeR, err := regexp.Compile(`^([0-9]+)\s([a-z]+)\s([a-z0-9]+)\s(.+)`)
	if err != nil {
		panic(err)
	}
	for line, err := bufStdout.ReadBytes('\n'); err == nil; line, err = bufStdout.ReadBytes('\n') {
		results := treeR.FindSubmatch(line)
		if len(results) != 5 {
			// We expect to get back [original, perms, ob, hash, filename]
			return nil, fmt.Errorf("Unexpected parse of `git ls-tree` output: %q", results)
		}
		permissions, err := strconv.ParseUint(string(results[1]), 10, 32)
		if err != nil {
			return nil, fmt.Errorf("Could not parse permissions for file %s, got %s", results[4], results[1])
		}
		obs = append(obs, GitObject{uint32(permissions), GitObjectType(results[2]), string(results[3]), string(results[4])})
	}
	return obs, err
}

func getObject(startingHash, startingString, finalPath string) ([]byte, error) {
	objects, err := lstree(startingHash)
	if err != nil {
		return nil, err
	}

	if finalPath == "" {
		return gitList(startingHash, startingString)
	}

	nextPrefix, rest := pathRsplit(finalPath)

	for _, object := range objects {
		if rest == "" { // end of the line
			if object.Name == nextPrefix {
				if object.ObjectType == GitTree {
					return gitList(object.Hash, startingString)
				} else if object.ObjectType == GitBlob {
					return gitShow(object.Hash)
				} else {
					return nil, fmt.Errorf("Unsupported object type, %s", object.ObjectType)
				}
			}
		} else {
			if object.Name == nextPrefix {
				if object.ObjectType == GitTree {
					return getObject(object.Hash, startingString, rest)
				} else if object.ObjectType == GitBlob {
					return nil, fmt.Errorf("This is a directory, not an object, %s %s %s", object.ObjectType, object.Hash, object.Name)
				} else {
					return nil, fmt.Errorf("Unsupported object type, %s", object.ObjectType)
				}
			}
		}
	}
	return nil, fmt.Errorf("file not found in tree %s %s", startingString, finalPath)
}
<commit_msg>Add styling<commit_after>package main

// This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.

import (
	"bufio"
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
)

// GitObjectType Represents the type of a git object. Content is in "blob"
// types, everything else is metadata.
type GitObjectType string

const (
	// GitBlob represents content
	GitBlob = "blob"
	// GitTree represents a 'filesystem' in Git
	GitTree = "tree"
	// GitCommit represents a snapshot of state with comment
	GitCommit = "commit"
	// GitTag represents a named snapshot of state with comment
	GitTag = "tag"
)

// GitObject is a metadata blob describing everything about an object in git
// but the content
type GitObject struct {
	Permission uint32
	ObjectType GitObjectType
	Hash       string
	Name       string
}

func pathRsplit(path string) (first, rest string) {
	if path == "" || path == "/" {
		return "", ""
	}
	i := strings.Index(path, "/")
	if i == 0 {
		return pathRsplit(path[1:]) // Just... forget handling this
	} else if i == -1 {
		return path, ""
	}
	return path[:i], path[i+1:]
}

func gitShow(hash string) ([]byte, error) {
	cmd := exec.Command("git", "show", hash)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	err = cmd.Start()
	if err != nil {
		return nil, err
	}
	return ioutil.ReadAll(stdout)
}

func getRefs() ([]string, error) {
	var refs []string
	refsCmd := exec.Command("git", "show-ref")
	stdout, err := refsCmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	err = refsCmd.Start()
	if err != nil {
		return nil, err
	}
	// Should consider hoisting this
	refsR, err := regexp.Compile(`^([0-9a-f]{40})\s+(.+)`)
	if err != nil {
		return nil, err
	}
	bufStdout := bufio.NewReader(stdout)
	for line, err := bufStdout.ReadBytes('\n'); err == nil; line, err = bufStdout.ReadBytes('\n') {
		results := refsR.FindSubmatch(line)
		if len(results) != 3 {
			return nil, fmt.Errorf("Confused by your refs- got %d matches out of something that's supposed to have 2 fields. Line: %s Results: %q", len(results), line, results)
		}
		refs = append(refs, string(results[2][len("refs/"):]))
	}
	return refs, err
}

func gitList(hash, startingString string) ([]byte, error) {
	objects, err := lstree(hash)
	if err != nil {
		return nil, err
	}
	t, err := template.New("list").Parse(`

	{{ $prefix := .Prefix }}
	{{- range $element := .Objects}}

{{$element.Name}}
	{{- end}}

	`)
	if err != nil {
		return nil, err
	}
	var buf bytes.Buffer
	var data = struct {
		Objects []GitObject
		Prefix  string
	}{objects, startingString}
	err = t.Execute(&buf, data)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), err
}

func lstree(commit string) ([]GitObject, error) {
	// Inspect the cwd
	cmd := exec.Command("git", "ls-tree", commit)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	err = cmd.Start()
	if err != nil {
		return nil, err
	}
	bufStdout := bufio.NewReader(stdout)

	var obs []GitObject
	// for reasons I do not understand, saying "this ends with a newline" means
	// the newline gets eaten by the final capture. Also, ending with a '$' breaks
	// the whole match.
	treeR, err := regexp.Compile(`^([0-9]+)\s([a-z]+)\s([a-z0-9]+)\s(.+)`)
	if err != nil {
		panic(err)
	}
	for line, err := bufStdout.ReadBytes('\n'); err == nil; line, err = bufStdout.ReadBytes('\n') {
		results := treeR.FindSubmatch(line)
		if len(results) != 5 {
			// We expect to get back [original, perms, ob, hash, filename]
			return nil, fmt.Errorf("Unexpected parse of `git ls-tree` output: %q", results)
		}
		permissions, err := strconv.ParseUint(string(results[1]), 10, 32)
		if err != nil {
			return nil, fmt.Errorf("Could not parse permissions for file %s, got %s", results[4], results[1])
		}
		obs = append(obs, GitObject{uint32(permissions), GitObjectType(results[2]), string(results[3]), string(results[4])})
	}
	return obs, err
}

func getObject(startingHash, startingString, finalPath string) ([]byte, error) {
	objects, err := lstree(startingHash)
	if err != nil {
		return nil, err
	}

	if finalPath == "" {
		return gitList(startingHash, startingString)
	}

	nextPrefix, rest := pathRsplit(finalPath)

	for _, object := range objects {
		if rest == "" { // end of the line
			if object.Name == nextPrefix {
				if object.ObjectType == GitTree {
					return gitList(object.Hash, startingString)
				} else if object.ObjectType == GitBlob {
					return gitShow(object.Hash)
				} else {
					return nil, fmt.Errorf("Unsupported object type, %s", object.ObjectType)
				}
			}
		} else {
			if object.Name == nextPrefix {
				if object.ObjectType == GitTree {
					return getObject(object.Hash, startingString, rest)
				} else if object.ObjectType == GitBlob {
					return nil, fmt.Errorf("This is a directory, not an object, %s %s %s", object.ObjectType, object.Hash, object.Name)
				} else {
					return nil, fmt.Errorf("Unsupported object type, %s", object.ObjectType)
				}
			}
		}
	}
	return nil, fmt.Errorf("file not found in tree %s %s", startingString, finalPath)
}
<|endoftext|>


<commit_before>package stripe

import "encoding/json"

// SubStatus is the list of allowed values for the subscription's status.
// Allowed values are "trialing", "active", "past_due", "canceled", "unpaid".
type SubStatus string

// SubParams is the set of parameters that can be used when creating or updating a subscription.
// For more details see https://stripe.com/docs/api#create_subscription and https://stripe.com/docs/api#update_subscription.
type SubParams struct {
	Params
	Customer, Plan                                  string
	Coupon, Token                                   string
	TrialEnd                                        int64
	Card                                            *CardParams
	Quantity                                        uint64
	FeePercent                                      float64
	NoProrate, EndCancel, QuantityZero, TrialEndNow bool
}

// SubListParams is the set of parameters that can be used when listing active subscriptions.
// For more details see https://stripe.com/docs/api#list_subscriptions.
type SubListParams struct {
	ListParams
	Customer string
}

// Sub is the resource representing a Stripe subscription.
// For more details see https://stripe.com/docs/api#subscriptions.
type Sub struct {
	ID          string            `json:"id"`
	EndCancel   bool              `json:"cancel_at_period_end"`
	Customer    *Customer         `json:"customer"`
	Plan        *Plan             `json:"plan"`
	Quantity    uint64            `json:"quantity"`
	Status      SubStatus         `json:"status"`
	FeePercent  float64           `json:"application_fee_percent"`
	Canceled    int64             `json:"canceled_at"`
	PeriodEnd   int64             `json:"current_period_end"`
	PeriodStart int64             `json:"current_period_start"`
	Discount    *Discount         `json:"discount"`
	Ended       int64             `json:"ended_at"`
	Meta        map[string]string `json:"metadata"`
	TaxPercent  int64             `json:"tax_percent"`
	TrialEnd    int64             `json:"trial_end"`
	TrialStart  int64             `json:"trial_start"`
}

// SubList is a list object for subscriptions.
type SubList struct {
	ListMeta
	Values []*Sub `json:"data"`
}

// UnmarshalJSON handles deserialization of a Sub.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (s *Sub) UnmarshalJSON(data []byte) error {
	type sub Sub
	var ss sub
	err := json.Unmarshal(data, &ss)
	if err == nil {
		*s = Sub(ss)
	} else {
		// the id is surrounded by "\" characters, so strip them
		s.ID = string(data[1 : len(data)-1])
	}

	return nil
}
<commit_msg>tax_percent is a float<commit_after>package stripe

import "encoding/json"

// SubStatus is the list of allowed values for the subscription's status.
// Allowed values are "trialing", "active", "past_due", "canceled", "unpaid".
type SubStatus string

// SubParams is the set of parameters that can be used when creating or updating a subscription.
// For more details see https://stripe.com/docs/api#create_subscription and https://stripe.com/docs/api#update_subscription.
type SubParams struct {
	Params
	Customer, Plan                                  string
	Coupon, Token                                   string
	TrialEnd                                        int64
	Card                                            *CardParams
	Quantity                                        uint64
	FeePercent                                      float64
	NoProrate, EndCancel, QuantityZero, TrialEndNow bool
}

// SubListParams is the set of parameters that can be used when listing active subscriptions.
// For more details see https://stripe.com/docs/api#list_subscriptions.
type SubListParams struct {
	ListParams
	Customer string
}

// Sub is the resource representing a Stripe subscription.
// For more details see https://stripe.com/docs/api#subscriptions.
type Sub struct {
	ID          string            `json:"id"`
	EndCancel   bool              `json:"cancel_at_period_end"`
	Customer    *Customer         `json:"customer"`
	Plan        *Plan             `json:"plan"`
	Quantity    uint64            `json:"quantity"`
	Status      SubStatus         `json:"status"`
	FeePercent  float64           `json:"application_fee_percent"`
	Canceled    int64             `json:"canceled_at"`
	PeriodEnd   int64             `json:"current_period_end"`
	PeriodStart int64             `json:"current_period_start"`
	Discount    *Discount         `json:"discount"`
	Ended       int64             `json:"ended_at"`
	Meta        map[string]string `json:"metadata"`
	TaxPercent  float64           `json:"tax_percent"`
	TrialEnd    int64             `json:"trial_end"`
	TrialStart  int64             `json:"trial_start"`
}

// SubList is a list object for subscriptions.
type SubList struct {
	ListMeta
	Values []*Sub `json:"data"`
}

// UnmarshalJSON handles deserialization of a Sub.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (s *Sub) UnmarshalJSON(data []byte) error {
	type sub Sub
	var ss sub
	err := json.Unmarshal(data, &ss)
	if err == nil {
		*s = Sub(ss)
	} else {
		// the id is surrounded by "\" characters, so strip them
		s.ID = string(data[1 : len(data)-1])
	}

	return nil
}
<|endoftext|>


<commit_before>package main
import "fmt"
import "os"
import "io"
import "io/ioutil"
import "path/filepath"
import "encoding/hex"
import "compress/gzip"
import "archive/tar"
import "strings"
import "bytes"

var (
    fileExtension = "gok"
    createdFiles map[string]string;
);

func main() {
    //TODO remove this after stabilization.
    if len(os.Args) == 2 {
        os.Chdir(os.Args[1]);
    }
    createdFiles = make(map[string]string);
    convertGokToGoFiles(".");
    unpackResource();
    injectRoutes();
    fmt.Println("Gok exported golang code, execute go build")
}

func convertGokToGoFiles(dir string) {
    files, err := filepath.Glob(dir+"/*.gok");
    if err != nil {
        errExit(err, "");
    }
    for _, s := range files {
        gokContent, err := ioutil.ReadFile(s);
        if err != nil {
            errExit(err, "");
        }
        gocode, mainFunc, err := processGokContent(string(gokContent));
        if err != nil {
            errExit(err, "");
        }
        createdFiles[s] = mainFunc;
        ioutil.WriteFile(s+".go", []byte(gocode), 0644);
    }
    if dirs, err := ioutil.ReadDir(dir); err == nil {
        for _, d := range dirs {
            if d.IsDir() {
                convertGokToGoFiles(dir+"/"+d.Name());
            }
        }
    }
}

func unpackResource() {
    orignal, err := hex.DecodeString(Resource);
    if err != nil {
        errExit(err, "");
    }
    unzip, err := gzip.NewReader(bytes.NewBuffer([]byte(orignal)))
    if err != nil {
        errExit(err, "");
    }
    untar := tar.NewReader(unzip);
    var fileContent bytes.Buffer;
    for h, err := untar.Next(); err == nil;  h, err = untar.Next() {
        fileContent.Reset();
        io.Copy(&fileContent, untar);
        var name string;
        if h.Name[0] == '/' {
            name = h.Name[1:];
        } else {
            name = h.Name;
        }
        if strings.Contains(name, "/") {
            lstIndx := strings.LastIndex(name, "/");
            if err := os.MkdirAll(name[0:lstIndx], 0744); err != nil {
                errExit(err, "");
            }
        }
        ioutil.WriteFile(name, fileContent.Bytes(), os.FileMode(h.Mode));
    }
}

func injectRoutes() {
    var routes bytes.Buffer;
    s, err := ioutil.ReadFile("serverb596f256.go");
    if err != nil {
        errExit(err, "");
    }
    parts := strings.Split(string(s), "//<gok inject routes>");
    if len(parts) != 2 {
        errExit(nil, "Unusual resource file");
    }
    for k,v := range createdFiles {
        if k == "index.gok" {
            routes.Write([]byte("\"\":"+v+",\n"));
        }
        routes.Write([]byte("\""+k+"\":"+v+",\n"));
    }
    final := strings.Join([]string{parts[0], string(routes.Bytes()), parts[1]}, "\n");
    ioutil.WriteFile("serverb596f256.go", []byte(final), 0644);
}
<commit_msg>standard<commit_after>package main
import "fmt"
import "os"
import "io"
import "io/ioutil"
import "path/filepath"
import "encoding/hex"
import "compress/gzip"
import "archive/tar"
import "strings"
import "bytes"
import "os/exec"
import "container/list"

var (
    fileExtension = "gok"
    webRoutes map[string]string;
    shouldDelete *list.List;
);

func main() {
    //TODO remove this after stabilization.
    if len(os.Args) == 2 {
        os.Chdir(os.Args[1]);
    }
    webRoutes = make(map[string]string);
    shouldDelete = list.New();
    convertGokToGoFiles(".");
    unpackResource();
    injectRoutes();
    fmt.Println("now building go code");
    goBuild();
    deleteBuiltFiles();
}

func convertGokToGoFiles(dir string) {
    files, err := filepath.Glob(dir+"/*.gok");
    if err != nil {
        errExit(err, "");
    }
    for _, s := range files {
        gokContent, err := ioutil.ReadFile(s);
        if err != nil {
            errExit(err, "");
        }
        gocode, mainFunc, err := processGokContent(string(gokContent));
        if err != nil {
            errExit(err, "");
        }
        webRoutes[s] = mainFunc;
        shouldDelete.PushBack(s+".go");
        ioutil.WriteFile(s+".go", []byte(gocode), 0644);
    }
    if dirs, err := ioutil.ReadDir(dir); err == nil {
        for _, d := range dirs {
            if d.IsDir() {
                convertGokToGoFiles(dir+"/"+d.Name());
            }
        }
    }
}

func unpackResource() {
    orignal, err := hex.DecodeString(Resource);
    if err != nil {
        errExit(err, "");
    }
    unzip, err := gzip.NewReader(bytes.NewBuffer([]byte(orignal)))
    if err != nil {
        errExit(err, "");
    }
    untar := tar.NewReader(unzip);
    var fileContent bytes.Buffer;
    for h, err := untar.Next(); err == nil;  h, err = untar.Next() {
        fileContent.Reset();
        io.Copy(&fileContent, untar);
        var name string;
        if h.Name[0] == '/' {
            name = h.Name[1:];
        } else {
            name = h.Name;
        }
        if strings.Contains(name, "/") {
            lstIndx := strings.LastIndex(name, "/");
            if err := os.MkdirAll(name[0:lstIndx], 0744); err != nil {
                errExit(err, "");
            }
        }
        ioutil.WriteFile(name, fileContent.Bytes(), os.FileMode(h.Mode));
        if name[len(name)-3:] == ".go" {
            shouldDelete.PushBack(name);
        }
    }
}

func injectRoutes() {
    var routes bytes.Buffer;
    s, err := ioutil.ReadFile("serverb596f256.go");
    if err != nil {
        errExit(err, "");
    }
    parts := strings.Split(string(s), "//<gok inject routes>");
    if len(parts) != 2 {
        errExit(nil, "Unusual resource file");
    }
    for k,v := range webRoutes {
        if k == "index.gok" {
            routes.Write([]byte("\"\":"+v+",\n"));
        }
        routes.Write([]byte("\""+k+"\":"+v+",\n"));
    }
    final := strings.Join([]string{parts[0], string(routes.Bytes()), parts[1]}, "\n");
    ioutil.WriteFile("serverb596f256.go", []byte(final), 0644);
}

func goBuild() {
    cmd := exec.Command("go", "build");
    output, err := cmd.Output();
    if err != nil {
        errExit(err, "");
    }
    fmt.Println(string(output));
}

func deleteBuiltFiles() {
    for e := shouldDelete.Front(); e != nil; e = e.Next() {
       os.Remove(e.Value.(string));
   }
}
<|endoftext|>


<commit_before>package gop

import (
    "github.com/gorilla/mux"
    "github.com/gorilla/context"
    "github.com/gorilla/schema"
    "github.com/jbert/timber"

    "fmt"
    "net"
    "time"
    "runtime"
    "net/http"
    "strings"
    "log"
	"syscall"
)

// Stuff we include in both App and Req, for convenience
type common struct {
    timber.Logger
    Cfg             *Config
    Stats           StatsdClient
    Decoder         *schema.Decoder
}

// Represents a managed gop app. Returned by init.
// Embeds logging, provides .Cfg for configuration access.
type App struct {
    common

    AppName         string
    ProjectName     string
    GorillaRouter   *mux.Router
    listener        net.Listener
    wantReq         chan *wantReq
    doneReq         chan *Req
    getReqs         chan chan *Req
    startTime       time.Time
    currentReqs     int
    totalReqs       int
    doingGraceful   bool
}

// Used for RPC to get a new request
type wantReq struct {
    r       *http.Request
    reply   chan *Req
}

// Per request struct. has convenience references to functionality
// in the app singleton. Passed into the request handler.
type Req struct {
    common

    id              int
    startTime       time.Time
    app             *App
    r               *http.Request
    RealRemoteIP    string
    IsHTTPS         bool
}

// The function signature your http handlers need.
type HandlerFunc func(g *Req, w http.ResponseWriter, r *http.Request)

// Set up the application. Reads config. Panic if runtime environment is deficient.
func Init(projectName, appName string) *App {
    app := &App{
        common: common{
            Decoder:        schema.NewDecoder(),
        },
        AppName:        appName,
        ProjectName:    projectName,
        GorillaRouter:  mux.NewRouter(),
        wantReq:        make(chan *wantReq),
        doneReq:        make(chan *Req),
        getReqs:        make(chan chan *Req),
        startTime:      time.Now(),
    }

    app.loadAppConfigFile()

    // Linux setuid() doesn't work with threaded procs :-O
    // and the go runtime threads before we can get going.
    //
    // http://homepage.ntlworld.com/jonathan.deboynepollard/FGA/linux-thread-problems.html
//    app.setUserAndGroup()

    app.initLogging()

    maxProcs, _ := app.Cfg.GetInt("gop", "maxprocs", 4 * runtime.NumCPU())
    app.Debug("Setting maxprocs to %d\n", maxProcs)
    runtime.GOMAXPROCS(maxProcs)

    app.initStatsd()

    return app
}

// Clean shutdown
func (a *App) Finish() {
    // Start a log flush
    timber.Close()
}

//
// Commented out - not reliable - see comment at call site
// func (a *App) setUserAndGroup() {
//     // We do not have logging set up yet. We just panic() on error.
//
//     desiredUserName, _ := a.Cfg.Get("gop", "user", "")
//     // Usernames it's ok to run as, in order of preference
//     possibleUserNames := []string{desiredUserName, a.AppName, a.ProjectName}
//
//     doneIt := false
//     for _, desiredUserName := range possibleUserNames {
//         if runAsUserName(desiredUserName) {
//             doneIt = true
//             break
//         }
//     }
//
//     if !doneIt {
//         panic(fmt.Sprintf("Can't run as any of these users: %v, please set config and/or create user", possibleUserNames))
//     }
//
// // Can't log at this stage :-/
// //    a.Info("Running as user %s (%d)", desiredUserName, desiredUser.Uid)
// }

func (a *App) setProcessGroupForNelly() {
	// Nelly knows our pid and will check that there is always at
	// least one process in the process group with the same id as our pid
	mypid := syscall.Getpid()
	err := syscall.Setpgid(mypid, mypid)
	if err != nil {
        panic(fmt.Sprintf("Failed to setprgp]: %s\n", mypid, mypid, err.Error()))
	}
}

// Hands out 'request' objects
func (a *App) requestMaker() {
    nextReqId := 0
    openReqs := make(map[int] *Req)
    for {
        select {
            case wantReq := <- a.wantReq: {
                realRemoteIP := wantReq.r.RemoteAddr
                isHTTPS := false
                useXF, _:= a.Cfg.GetBool("gop", "use_xf_headers", false)
                if useXF {
                    xff := wantReq.r.Header.Get("X-Forwarded-For")
                    if xff != "" {
                        ips := strings.Split(xff, ",")
                        for i, ip := range ips {
                            ips[i] = strings.TrimSpace(ip)
                        }
                        // The only trustworthy component is the *last* (and only if we
                        // are behind nginx or other proxy which is stripping x-f-f from incoming requests)
                        realRemoteIP = ips[len(ips)-1]
                    }
                    xfp := wantReq.r.Header.Get("X-Forwarded-Proto")
                    isHTTPS = strings.ToLower(xfp) == "https"
                }
                req := Req{
                    common: common{
                        Logger:     a.Logger,
                        Cfg:        a.Cfg,
                        Stats:      a.Stats,
                        Decoder:    a.Decoder,
                    },

                    id:         nextReqId,
                    app:        a,
                    startTime:  time.Now(),
                    r:          wantReq.r,
                    RealRemoteIP: realRemoteIP,
                    IsHTTPS:    isHTTPS,
                }
                openReqs[req.id] = &req
                nextReqId++
                a.Stats.Inc("http_reqs", 1)
                a.Stats.Inc("current_http_reqs", 1)
                a.totalReqs++
                a.currentReqs++
                wantReq.reply <- &req
            }
            case doneReq := <- a.doneReq: {
                _, found := openReqs[doneReq.id]
                if !found {
                    a.Error("BUG! Unknown request id [%d] being retired")
                } else {
                    a.Stats.Dec("current_http_reqs", 1)
                    doneReq.finished()
                    a.currentReqs--
                    delete (openReqs, doneReq.id)
                }
            }
            case reply := <- a.getReqs: {
                go func() {
                    for _, req := range openReqs {
                        reply <- req
                    }
                    close(reply)
                }()
            }
        }
    }
}

// Ask requestMaker for a request
func (a *App) getReq(r *http.Request) *Req {
    reply := make(chan *Req)
    a.wantReq <- &wantReq{r: r, reply: reply}
    return <- reply
}

func (g *Req) finished() {
    context.Clear(g.r)      // Cleanup  gorilla stash

    reqDuration := time.Since(g.startTime)
    slowReqSecs, _ := g.Cfg.GetFloat32("gop", "slow_req_secs", 10)
    if reqDuration.Seconds() > float64(slowReqSecs) {
        g.Error("Slow request [%s] took %s", g.r.URL, reqDuration)
    } else {
        g.Info("Request took %s", reqDuration)
    }

    // Tidy up request finalistion (requestMaker, req.finish() method, app.requestFinished())
    restartReqs, _ := g.Cfg.GetInt("gop", "max_requests", 0)
    if restartReqs > 0 && g.app.totalReqs > restartReqs {
        g.Error("Graceful restart after max_requests: %d", restartReqs)
        g.app.StartGracefulRestart("Max requests reached")
    }

    gcEveryReqs, _ := g.Cfg.GetInt("gop", "gc_requests", 0)
    if gcEveryReqs > 0 && g.app.totalReqs % gcEveryReqs == 0 {
        g.Info("Forcing GC after %d reqs", g.app.totalReqs)
        runtime.GC()
    }
}

func (a *App) initLogging() {

    logPattern, _ := a.Cfg.Get("gop", "log_pattern", "[%D %T] [%L] %S %M")

    // If set, hack all logging to stdout for dev
    forceStdout, _:= a.Cfg.GetBool("gop", "stdout_only_logging", false)
    configLogger := timber.ConfigLogger{
        LogWriter:  new(timber.ConsoleWriter),
        Level:      timber.INFO,
        Formatter:  timber.NewPatFormatter(logPattern),
    }

    if !forceStdout {
        defaultLogDir, _ := a.Cfg.Get("gop", "log_dir", "/var/log")
        defaultLogFname := defaultLogDir + "/" + a.ProjectName + "/" + a.AppName + ".log"
        logFname, _ := a.Cfg.Get("gop", "log_file", defaultLogFname)

        newWriter, err := timber.NewFileWriter(logFname)
        if err != nil {
            panic(fmt.Sprintf("Can't open log file: %s", err))
        }
        configLogger.LogWriter = newWriter
    }

    logLevelStr, _ := a.Cfg.Get("gop", "log_level", "INFO")
    logLevelStr = strings.ToUpper(logLevelStr)
    for logLevel, levelStr := range timber.LongLevelStrings {
        if logLevelStr == levelStr {
            configLogger.Level = timber.Level(logLevel)
            break
        }
    }

    // *Don't* create a NewTImber here. Logs are only flushed on Close() and if we
    // have more than one timber, it's easy to only Close() one of them...
    l := timber.Global
    l.AddLogger(configLogger)
    a.Logger = l

    // Set up the default go logger to go here too, so 3rd party
    // module logging plays nicely
    log.SetFlags(0)
    log.SetOutput(l)
}

func (a *App) watchdog() {
    repeat, _ := a.Cfg.GetInt("gop", "watchdog_secs", 300)
    ticker := time.Tick(time.Second * time.Duration(repeat))

    for {
        sysMemBytesLimit, _ := a.Cfg.GetInt64("gop", "sysmem_bytes_limit", 0)
        allocMemBytesLimit, _ := a.Cfg.GetInt64("gop", "allocmem_bytes_limit", 0)
        numFDsLimit, _ := a.Cfg.GetInt64("gop", "numfds_limit", 0)
        numGorosLimit, _ := a.Cfg.GetInt64("gop", "numgoros_limit", 0)

        sysMemBytes, allocMemBytes := getMemInfo()
        numFDs, err := fdsInUse()
        numGoros := int64(runtime.NumGoroutine())
        if err != nil {
            a.Error("Failed to get number of fds in use: %s", err.Error())
            // Continue without
        }

        a.Info("TICK: sys=%d,alloc=%d,fds=%d,current_req=%d,total_req=%d,goros=%d",
            sysMemBytes,
            allocMemBytes,
            numFDs,
            a.currentReqs,
            a.totalReqs,
            numGoros)
        a.Stats.Gauge("mem.sys", sysMemBytes)
        a.Stats.Gauge("mem.alloc", allocMemBytes)
        a.Stats.Gauge("numfds", numFDs)
        a.Stats.Gauge("numgoro", numGoros)

        if sysMemBytesLimit > 0 && sysMemBytes >= sysMemBytesLimit {
            a.Error("SYS MEM LIMIT REACHED [%d >= %d] - starting graceful restart", sysMemBytes, sysMemBytesLimit)
            a.StartGracefulRestart("Sys Memory limit reached")
        }
        if allocMemBytesLimit > 0 && allocMemBytes >= allocMemBytesLimit {
            a.Error("ALLOC MEM LIMIT REACHED [%d >= %d] - starting graceful restart", allocMemBytes, allocMemBytesLimit)
            a.StartGracefulRestart("Alloc Memory limit reached")
        }
        if numFDsLimit > 0 && numFDs >= numFDsLimit {
            a.Error("NUM FDS LIMIT REACHED [%d >= %d] - starting graceful restart", numFDs, numFDsLimit)
            a.StartGracefulRestart("Number of fds limit reached")
        }
        if numGorosLimit > 0 && numGoros >= numGorosLimit {
            a.Error("NUM GOROS LIMIT REACHED [%d >= %d] - starting graceful restart", numGoros, numGorosLimit)
            a.StartGracefulRestart("Number of goros limit reached")
        }

        restartAfterSecs, _ := a.Cfg.GetFloat32("gop", "restart_after_secs", 0)
        appRunTime := time.Since(a.startTime).Seconds()
        if restartAfterSecs > 0 && appRunTime > float64(restartAfterSecs) {
            a.Error("TIME LIMIT REACHED [%f >= %f] - starting graceful restart", appRunTime, restartAfterSecs)
            a.StartGracefulRestart("Run time limit reached")
        }
        <- ticker
    }
}

type responseWriter struct {
    http.ResponseWriter
    app *App
    code int
    notedCode bool
}

// Satisfy the interface
func (w *responseWriter) Header() http.Header {
    return w.ResponseWriter.Header()
}

func (w *responseWriter) Write(buf []byte) (int, error) {
    w.noteCode()
    return w.ResponseWriter.Write(buf)
}

func (w *responseWriter) CloseNotify() <-chan bool {
    return w.ResponseWriter.(http.CloseNotifier).CloseNotify()
}

func (w *responseWriter) WriteHeader(code int) {
    w.code = code
    w.noteCode()
    w.ResponseWriter.WriteHeader(code)
}

func (w *responseWriter) noteCode() {
    if !w.notedCode {
        statsKey := fmt.Sprintf("http_status.%d", w.code)
        w.app.Stats.Inc(statsKey, 1)
        w.notedCode = true
    }
}

func (a *App) WrapHandler(h HandlerFunc) http.HandlerFunc {
    // Wrap the handler, so we can do before/after logic
    f := func(w http.ResponseWriter, r *http.Request) {
        gopRequest := a.getReq(r)
        defer func() {
            a.doneReq <- gopRequest
        }()
        gopWriter := responseWriter{code: 200, app: a, ResponseWriter: w}

        err := r.ParseForm()
        if err != nil {
            http.Error(&gopWriter, "Failed to parse form: " + err.Error(), http.StatusInternalServerError)
            return
        }

        // Pass in the gop, for logging, cfg etc
        h(gopRequest, &gopWriter, r)
    }

    return http.HandlerFunc(f)
}

// Register an http handler managed by gop.
// We use Gorilla muxxer, since it is back-compatible and nice to use :-)
func (a *App) HandleFunc(u string, h HandlerFunc) {
    gopHandler := a.WrapHandler(h)

    a.GorillaRouter.HandleFunc(u, gopHandler)
}

func (a *App) Run() {
	a.setProcessGroupForNelly()

    a.goAgainSetup()

    a.registerGopHandlers()

    go a.watchdog()

    go a.requestMaker()

    listenAddr, _ := a.Cfg.Get("gop", "listen_addr", ":http")
    listenNet, _ := a.Cfg.Get("gop", "listen_net", "tcp")
    a.goAgainListenAndServe(listenNet, listenAddr)
}

func (a *App) Serve(l net.Listener) {
    http.Serve(l, a.GorillaRouter)
}

func getMemInfo() (int64, int64) {
    var memStats runtime.MemStats
    runtime.ReadMemStats(&memStats)
    // There's lots of stuff in here.
    return int64(memStats.Sys), int64(memStats.Alloc)
}
<commit_msg>log and continue on parseform errors, go appears to error (incorrectly) on missing content-type<commit_after>package gop

import (
    "github.com/gorilla/mux"
    "github.com/gorilla/context"
    "github.com/gorilla/schema"
    "github.com/jbert/timber"

    "fmt"
    "net"
    "time"
    "runtime"
    "net/http"
    "strings"
    "log"
	"syscall"
)

// Stuff we include in both App and Req, for convenience
type common struct {
    timber.Logger
    Cfg             *Config
    Stats           StatsdClient
    Decoder         *schema.Decoder
}

// Represents a managed gop app. Returned by init.
// Embeds logging, provides .Cfg for configuration access.
type App struct {
    common

    AppName         string
    ProjectName     string
    GorillaRouter   *mux.Router
    listener        net.Listener
    wantReq         chan *wantReq
    doneReq         chan *Req
    getReqs         chan chan *Req
    startTime       time.Time
    currentReqs     int
    totalReqs       int
    doingGraceful   bool
}

// Used for RPC to get a new request
type wantReq struct {
    r       *http.Request
    reply   chan *Req
}

// Per request struct. has convenience references to functionality
// in the app singleton. Passed into the request handler.
type Req struct {
    common

    id              int
    startTime       time.Time
    app             *App
    r               *http.Request
    RealRemoteIP    string
    IsHTTPS         bool
}

// The function signature your http handlers need.
type HandlerFunc func(g *Req, w http.ResponseWriter, r *http.Request)

// Set up the application. Reads config. Panic if runtime environment is deficient.
func Init(projectName, appName string) *App {
    app := &App{
        common: common{
            Decoder:        schema.NewDecoder(),
        },
        AppName:        appName,
        ProjectName:    projectName,
        GorillaRouter:  mux.NewRouter(),
        wantReq:        make(chan *wantReq),
        doneReq:        make(chan *Req),
        getReqs:        make(chan chan *Req),
        startTime:      time.Now(),
    }

    app.loadAppConfigFile()

    // Linux setuid() doesn't work with threaded procs :-O
    // and the go runtime threads before we can get going.
    //
    // http://homepage.ntlworld.com/jonathan.deboynepollard/FGA/linux-thread-problems.html
//    app.setUserAndGroup()

    app.initLogging()

    maxProcs, _ := app.Cfg.GetInt("gop", "maxprocs", 4 * runtime.NumCPU())
    app.Debug("Setting maxprocs to %d\n", maxProcs)
    runtime.GOMAXPROCS(maxProcs)

    app.initStatsd()

    return app
}

// Clean shutdown
func (a *App) Finish() {
    // Start a log flush
    timber.Close()
}

//
// Commented out - not reliable - see comment at call site
// func (a *App) setUserAndGroup() {
//     // We do not have logging set up yet. We just panic() on error.
//
//     desiredUserName, _ := a.Cfg.Get("gop", "user", "")
//     // Usernames it's ok to run as, in order of preference
//     possibleUserNames := []string{desiredUserName, a.AppName, a.ProjectName}
//
//     doneIt := false
//     for _, desiredUserName := range possibleUserNames {
//         if runAsUserName(desiredUserName) {
//             doneIt = true
//             break
//         }
//     }
//
//     if !doneIt {
//         panic(fmt.Sprintf("Can't run as any of these users: %v, please set config and/or create user", possibleUserNames))
//     }
//
// // Can't log at this stage :-/
// //    a.Info("Running as user %s (%d)", desiredUserName, desiredUser.Uid)
// }

func (a *App) setProcessGroupForNelly() {
	// Nelly knows our pid and will check that there is always at
	// least one process in the process group with the same id as our pid
	mypid := syscall.Getpid()
	err := syscall.Setpgid(mypid, mypid)
	if err != nil {
        panic(fmt.Sprintf("Failed to setprgp]: %s\n", mypid, mypid, err.Error()))
	}
}

// Hands out 'request' objects
func (a *App) requestMaker() {
    nextReqId := 0
    openReqs := make(map[int] *Req)
    for {
        select {
            case wantReq := <- a.wantReq: {
                realRemoteIP := wantReq.r.RemoteAddr
                isHTTPS := false
                useXF, _:= a.Cfg.GetBool("gop", "use_xf_headers", false)
                if useXF {
                    xff := wantReq.r.Header.Get("X-Forwarded-For")
                    if xff != "" {
                        ips := strings.Split(xff, ",")
                        for i, ip := range ips {
                            ips[i] = strings.TrimSpace(ip)
                        }
                        // The only trustworthy component is the *last* (and only if we
                        // are behind nginx or other proxy which is stripping x-f-f from incoming requests)
                        realRemoteIP = ips[len(ips)-1]
                    }
                    xfp := wantReq.r.Header.Get("X-Forwarded-Proto")
                    isHTTPS = strings.ToLower(xfp) == "https"
                }
                req := Req{
                    common: common{
                        Logger:     a.Logger,
                        Cfg:        a.Cfg,
                        Stats:      a.Stats,
                        Decoder:    a.Decoder,
                    },

                    id:         nextReqId,
                    app:        a,
                    startTime:  time.Now(),
                    r:          wantReq.r,
                    RealRemoteIP: realRemoteIP,
                    IsHTTPS:    isHTTPS,
                }
                openReqs[req.id] = &req
                nextReqId++
                a.Stats.Inc("http_reqs", 1)
                a.Stats.Inc("current_http_reqs", 1)
                a.totalReqs++
                a.currentReqs++
                wantReq.reply <- &req
            }
            case doneReq := <- a.doneReq: {
                _, found := openReqs[doneReq.id]
                if !found {
                    a.Error("BUG! Unknown request id [%d] being retired")
                } else {
                    a.Stats.Dec("current_http_reqs", 1)
                    doneReq.finished()
                    a.currentReqs--
                    delete (openReqs, doneReq.id)
                }
            }
            case reply := <- a.getReqs: {
                go func() {
                    for _, req := range openReqs {
                        reply <- req
                    }
                    close(reply)
                }()
            }
        }
    }
}

// Ask requestMaker for a request
func (a *App) getReq(r *http.Request) *Req {
    reply := make(chan *Req)
    a.wantReq <- &wantReq{r: r, reply: reply}
    return <- reply
}

func (g *Req) finished() {
    context.Clear(g.r)      // Cleanup  gorilla stash

    reqDuration := time.Since(g.startTime)
    slowReqSecs, _ := g.Cfg.GetFloat32("gop", "slow_req_secs", 10)
    if reqDuration.Seconds() > float64(slowReqSecs) {
        g.Error("Slow request [%s] took %s", g.r.URL, reqDuration)
    } else {
        g.Info("Request took %s", reqDuration)
    }

    // Tidy up request finalistion (requestMaker, req.finish() method, app.requestFinished())
    restartReqs, _ := g.Cfg.GetInt("gop", "max_requests", 0)
    if restartReqs > 0 && g.app.totalReqs > restartReqs {
        g.Error("Graceful restart after max_requests: %d", restartReqs)
        g.app.StartGracefulRestart("Max requests reached")
    }

    gcEveryReqs, _ := g.Cfg.GetInt("gop", "gc_requests", 0)
    if gcEveryReqs > 0 && g.app.totalReqs % gcEveryReqs == 0 {
        g.Info("Forcing GC after %d reqs", g.app.totalReqs)
        runtime.GC()
    }
}

func (a *App) initLogging() {

    logPattern, _ := a.Cfg.Get("gop", "log_pattern", "[%D %T] [%L] %S %M")

    // If set, hack all logging to stdout for dev
    forceStdout, _:= a.Cfg.GetBool("gop", "stdout_only_logging", false)
    configLogger := timber.ConfigLogger{
        LogWriter:  new(timber.ConsoleWriter),
        Level:      timber.INFO,
        Formatter:  timber.NewPatFormatter(logPattern),
    }

    if !forceStdout {
        defaultLogDir, _ := a.Cfg.Get("gop", "log_dir", "/var/log")
        defaultLogFname := defaultLogDir + "/" + a.ProjectName + "/" + a.AppName + ".log"
        logFname, _ := a.Cfg.Get("gop", "log_file", defaultLogFname)

        newWriter, err := timber.NewFileWriter(logFname)
        if err != nil {
            panic(fmt.Sprintf("Can't open log file: %s", err))
        }
        configLogger.LogWriter = newWriter
    }

    logLevelStr, _ := a.Cfg.Get("gop", "log_level", "INFO")
    logLevelStr = strings.ToUpper(logLevelStr)
    for logLevel, levelStr := range timber.LongLevelStrings {
        if logLevelStr == levelStr {
            configLogger.Level = timber.Level(logLevel)
            break
        }
    }

    // *Don't* create a NewTImber here. Logs are only flushed on Close() and if we
    // have more than one timber, it's easy to only Close() one of them...
    l := timber.Global
    l.AddLogger(configLogger)
    a.Logger = l

    // Set up the default go logger to go here too, so 3rd party
    // module logging plays nicely
    log.SetFlags(0)
    log.SetOutput(l)
}

func (a *App) watchdog() {
    repeat, _ := a.Cfg.GetInt("gop", "watchdog_secs", 300)
    ticker := time.Tick(time.Second * time.Duration(repeat))

    for {
        sysMemBytesLimit, _ := a.Cfg.GetInt64("gop", "sysmem_bytes_limit", 0)
        allocMemBytesLimit, _ := a.Cfg.GetInt64("gop", "allocmem_bytes_limit", 0)
        numFDsLimit, _ := a.Cfg.GetInt64("gop", "numfds_limit", 0)
        numGorosLimit, _ := a.Cfg.GetInt64("gop", "numgoros_limit", 0)

        sysMemBytes, allocMemBytes := getMemInfo()
        numFDs, err := fdsInUse()
        numGoros := int64(runtime.NumGoroutine())
        if err != nil {
            a.Error("Failed to get number of fds in use: %s", err.Error())
            // Continue without
        }

        a.Info("TICK: sys=%d,alloc=%d,fds=%d,current_req=%d,total_req=%d,goros=%d",
            sysMemBytes,
            allocMemBytes,
            numFDs,
            a.currentReqs,
            a.totalReqs,
            numGoros)
        a.Stats.Gauge("mem.sys", sysMemBytes)
        a.Stats.Gauge("mem.alloc", allocMemBytes)
        a.Stats.Gauge("numfds", numFDs)
        a.Stats.Gauge("numgoro", numGoros)

        if sysMemBytesLimit > 0 && sysMemBytes >= sysMemBytesLimit {
            a.Error("SYS MEM LIMIT REACHED [%d >= %d] - starting graceful restart", sysMemBytes, sysMemBytesLimit)
            a.StartGracefulRestart("Sys Memory limit reached")
        }
        if allocMemBytesLimit > 0 && allocMemBytes >= allocMemBytesLimit {
            a.Error("ALLOC MEM LIMIT REACHED [%d >= %d] - starting graceful restart", allocMemBytes, allocMemBytesLimit)
            a.StartGracefulRestart("Alloc Memory limit reached")
        }
        if numFDsLimit > 0 && numFDs >= numFDsLimit {
            a.Error("NUM FDS LIMIT REACHED [%d >= %d] - starting graceful restart", numFDs, numFDsLimit)
            a.StartGracefulRestart("Number of fds limit reached")
        }
        if numGorosLimit > 0 && numGoros >= numGorosLimit {
            a.Error("NUM GOROS LIMIT REACHED [%d >= %d] - starting graceful restart", numGoros, numGorosLimit)
            a.StartGracefulRestart("Number of goros limit reached")
        }

        restartAfterSecs, _ := a.Cfg.GetFloat32("gop", "restart_after_secs", 0)
        appRunTime := time.Since(a.startTime).Seconds()
        if restartAfterSecs > 0 && appRunTime > float64(restartAfterSecs) {
            a.Error("TIME LIMIT REACHED [%f >= %f] - starting graceful restart", appRunTime, restartAfterSecs)
            a.StartGracefulRestart("Run time limit reached")
        }
        <- ticker
    }
}

type responseWriter struct {
    http.ResponseWriter
    app *App
    code int
    notedCode bool
}

// Satisfy the interface
func (w *responseWriter) Header() http.Header {
    return w.ResponseWriter.Header()
}

func (w *responseWriter) Write(buf []byte) (int, error) {
    w.noteCode()
    return w.ResponseWriter.Write(buf)
}

func (w *responseWriter) CloseNotify() <-chan bool {
    return w.ResponseWriter.(http.CloseNotifier).CloseNotify()
}

func (w *responseWriter) WriteHeader(code int) {
    w.code = code
    w.noteCode()
    w.ResponseWriter.WriteHeader(code)
}

func (w *responseWriter) noteCode() {
    if !w.notedCode {
        statsKey := fmt.Sprintf("http_status.%d", w.code)
        w.app.Stats.Inc(statsKey, 1)
        w.notedCode = true
    }
}

func (a *App) WrapHandler(h HandlerFunc) http.HandlerFunc {
    // Wrap the handler, so we can do before/after logic
    f := func(w http.ResponseWriter, r *http.Request) {
        gopRequest := a.getReq(r)
        defer func() {
            a.doneReq <- gopRequest
        }()
        gopWriter := responseWriter{code: 200, app: a, ResponseWriter: w}

        err := r.ParseForm()
        if err != nil {
            a.Error("Failed to parse form: " + err.Error() + " (continuing)")
//            http.Error(&gopWriter, "Failed to parse form: " + err.Error(), http.StatusInternalServerError)
//          return
        }

        // Pass in the gop, for logging, cfg etc
        h(gopRequest, &gopWriter, r)
    }

    return http.HandlerFunc(f)
}

// Register an http handler managed by gop.
// We use Gorilla muxxer, since it is back-compatible and nice to use :-)
func (a *App) HandleFunc(u string, h HandlerFunc) {
    gopHandler := a.WrapHandler(h)

    a.GorillaRouter.HandleFunc(u, gopHandler)
}

func (a *App) Run() {
	a.setProcessGroupForNelly()

    a.goAgainSetup()

    a.registerGopHandlers()

    go a.watchdog()

    go a.requestMaker()

    listenAddr, _ := a.Cfg.Get("gop", "listen_addr", ":http")
    listenNet, _ := a.Cfg.Get("gop", "listen_net", "tcp")
    a.goAgainListenAndServe(listenNet, listenAddr)
}

func (a *App) Serve(l net.Listener) {
    http.Serve(l, a.GorillaRouter)
}

func getMemInfo() (int64, int64) {
    var memStats runtime.MemStats
    runtime.ReadMemStats(&memStats)
    // There's lots of stuff in here.
    return int64(memStats.Sys), int64(memStats.Alloc)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/chrisseto/pty"
	"github.com/chrisseto/sux/pansi"
	"github.com/nsf/termbox-go"
	"io"
	"os"
	"os/exec"
)

type Pane struct {
	*exec.Cmd

	cx, cy        int
	sx, sy        int
	fg, bg        termbox.Attribute
	width, height uint16
	scrollOffset  int

	Prog string
	Args []string

	Pty    *os.File
	output io.Reader
	cells  [][]termbox.Cell
}

func CreatePane(width, height uint16, prog string, args ...string) *Pane {
	return &Pane{
		Cmd: exec.Command(prog, args...),
		cx:  0, cy: 0,
		sx: 0, sy: 0,
		fg: 0, bg: 0,
		scrollOffset: 0,
		Prog:         prog, Args: args,
		width: width, height: height,
		Pty: nil,
	}
}

func (p *Pane) Start() error {
	pterm, err := pty.Start(p.Cmd)
	if err != nil {
		panic(err)
	}
	if err = pty.Setsize(pterm, p.height, p.width); err != nil {
		panic(err)
	}
	p.Pty = pterm
	p.cells = make([][]termbox.Cell, 1, p.height)
	p.cells[0] = make([]termbox.Cell, p.width)
	go p.outputPipe()
	return nil
}

func (p *Pane) Close() error {
	return p.Process.Kill()
}

func (p *Pane) Cells() [][]termbox.Cell {
	if offset := len(p.cells) + p.scrollOffset - int(p.height); offset > 0 {
		return p.cells[offset:]
	}
	return p.cells
}

func (p *Pane) Width() uint16 {
	return p.width
}

func (p *Pane) Height() uint16 {
	return p.height
}

func (p *Pane) Scroll(far int) {
	p.scrollOffset += far
	select {
	case Redraw <- struct{}{}:
	default: //Failed to send, a redraw is already happening
	}
}

func (p *Pane) Redraw() {
	for y, line := range p.Cells() {
		for x, cell := range line {
			termbox.SetCell(x, y, cell.Ch, cell.Fg, cell.Bg)
		}
	}
	termbox.SetCursor(p.Cursor())
}

func (p *Pane) Cursor() (int, int) {
	return p.cx, p.cy
}

func (p *Pane) outputPipe() {
	parser := pansi.NewParser()
	buf := make([]byte, 32*1024)

	for {
		nr, err := p.Pty.Read(buf)
		if nr > 0 {
			row := &p.cells[p.sy]

			for _, char := range buf[:nr] {
				parser.Feed(char)
				if parser.State() != nil {
					continue
				}
				if res := parser.Result(); res != nil {
					p.handleEscapeCode(res)
					parser.Clear()
					continue
				}

				switch char {
				case 0xA:
					p.sy++
					p.cy++
					p.cells = append(p.cells, make([]termbox.Cell, p.width))
					row = &p.cells[p.sy]
				case 0xD:
					p.sx = 0
					p.cx = 0
				case 0x8:
					(*row)[p.sx] = termbox.Cell{' ', p.fg, p.bg}
					p.sx--
					p.cx--
				default:
					(*row)[p.sx] = termbox.Cell{rune(char), p.fg, p.bg}
					p.sx++
					p.cx++
				}
			}

			Redraw <- struct{}{}
		}
		if err != nil {
			if err == io.EOF {
				break
			}
			panic(err)
		}

	}
}

func (p *Pane) handleEscapeCode(c *pansi.AnsiEscapeCode) {
	switch c.Type {
	case pansi.SetGraphicMode:
		p.SetGraphicMode(c.Values)
	case pansi.CursorPosition:
		p.cx, p.cy = c.Values[1], c.Values[2]
	case pansi.CursorUp:
		p.cy--
	case pansi.CursorDown:
		p.cy++
	case pansi.CursorBackward:
		p.cx--
	case pansi.CursorForward:
		p.cx++
	}
}

func (p *Pane) SetGraphicMode(vals []int) {
	for i := 0; i < len(vals); i++ {
		switch vals[i] {
		case 0:
			p.fg, p.bg = 0, 0
		case 1:
			p.fg |= termbox.AttrBold
		case 38:
			i++
			switch vals[i] {
			case 5:
				i++
				p.fg = termbox.Attribute(vals[i] + 1)
			case 2:
				i += 3 //TODO
			}
		case 48:
			i++
			switch vals[i] {
			case 5:
				i++
				p.bg = termbox.Attribute(vals[i] + 1)
			case 2:
				i += 3 //TODO
			}
		}
	}
}
<commit_msg>Handle bash a little bit better<commit_after>package main

import (
	"github.com/chrisseto/pty"
	"github.com/chrisseto/sux/pansi"
	"github.com/nsf/termbox-go"
	"io"
	"os"
	"os/exec"
)

type Pane struct {
	*exec.Cmd

	cx, cy        int
	sx, sy        int
	fg, bg        termbox.Attribute
	width, height uint16
	scrollOffset  int

	Prog string
	Args []string

	Pty    *os.File
	output io.Reader
	cells  [][]termbox.Cell
}

func CreatePane(width, height uint16, prog string, args ...string) *Pane {
	return &Pane{
		Cmd: exec.Command(prog, args...),
		cx:  0, cy: 0,
		sx: 0, sy: 0,
		fg: 0, bg: 0,
		scrollOffset: 0,
		Prog:         prog, Args: args,
		width: width, height: height,
		Pty: nil,
	}
}

func (p *Pane) Start() error {
	pterm, err := pty.Start(p.Cmd)
	if err != nil {
		panic(err)
	}
	if err = pty.Setsize(pterm, p.height, p.width); err != nil {
		panic(err)
	}
	p.Pty = pterm
	p.cells = make([][]termbox.Cell, 1, p.height)
	p.cells[0] = make([]termbox.Cell, p.width)
	go p.outputPipe()
	return nil
}

func (p *Pane) Close() error {
	return p.Process.Kill()
}

func (p *Pane) Cells() [][]termbox.Cell {
	if offset := len(p.cells) + p.scrollOffset - int(p.height); offset > 0 {
		return p.cells[offset:]
	}
	return p.cells
}

func (p *Pane) Width() uint16 {
	return p.width
}

func (p *Pane) Height() uint16 {
	return p.height
}

func (p *Pane) Scroll(far int) {
	p.scrollOffset += far
	select {
	case Redraw <- struct{}{}:
	default: //Failed to send, a redraw is already happening
	}
}

func (p *Pane) Redraw() {
	for y, line := range p.Cells() {
		for x, cell := range line {
			termbox.SetCell(x, y, cell.Ch, cell.Fg, cell.Bg)
		}
	}
	termbox.SetCursor(p.Cursor())
}

func (p *Pane) Cursor() (int, int) {
	return p.cx, p.cy
}

func (p *Pane) outputPipe() {
	parser := pansi.NewParser()
	buf := make([]byte, 32*1024)
	for {
		nr, err := p.Pty.Read(buf)
		if nr > 0 {
			row := &p.cells[p.sy]

			for _, char := range buf[:nr] {
				parser.Feed(char)
				if parser.State() != nil {
					continue
				}
				if res := parser.Result(); res != nil {
					p.handleEscapeCode(res)
					parser.Clear()
					continue
				}

				switch char {
				case 0x7: //Terminal Bell. Skip for the moment
				case 0xA:
					p.sy++
					p.cy++
					p.cells = append(p.cells, make([]termbox.Cell, p.width))
					row = &p.cells[p.sy]
				case 0xD:
					p.sx = 0
					p.cx = 0
				case 0x8:
					p.sx--
					p.cx--
					(*row)[p.sx] = termbox.Cell{' ', p.fg, p.bg}
				default:
					(*row)[p.sx] = termbox.Cell{rune(char), p.fg, p.bg}
					p.sx++
					p.cx++
				}
			}

			Redraw <- struct{}{}
		}
		if err != nil {
			if err == io.EOF {
				break
			}
			panic(err)
		}

	}
}

func (p *Pane) handleEscapeCode(c *pansi.AnsiEscapeCode) {
	switch c.Type {
	case pansi.SetGraphicMode:
		p.SetGraphicMode(c.Values)
	case pansi.CursorPosition:
		p.cx, p.cy = c.Values[1], c.Values[2]
	case pansi.CursorUp:
		p.cy--
	case pansi.CursorDown:
		p.cy++
	case pansi.CursorBackward:
		p.cx--
	case pansi.CursorForward:
		p.cx++
	case pansi.EraseLine:
		row := &p.cells[p.sy]
		for i := p.cx; i < len(*row); i++ {
			(*row)[i] = termbox.Cell{' ', p.fg, p.bg}
		}
	}
}

func (p *Pane) SetGraphicMode(vals []int) {
	for i := 0; i < len(vals); i++ {
		switch vals[i] {
		case 0:
			p.fg, p.bg = 0, 0
		case 1:
			p.fg |= termbox.AttrBold
		case 38:
			i++
			switch vals[i] {
			case 5:
				i++
				p.fg = termbox.Attribute(vals[i] + 1)
			case 2:
				i += 3 //TODO
			}
		case 48:
			i++
			switch vals[i] {
			case 5:
				i++
				p.bg = termbox.Attribute(vals[i] + 1)
			case 2:
				i += 3 //TODO
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package id3v2

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"

	"github.com/bogem/id3v2/bytesbufferpool"
	"github.com/bogem/id3v2/util"
)

// Tag stores all frames of opened file.
type Tag struct {
	framesCoords map[string][]frameCoordinates
	frames       map[string]Framer
	sequences    map[string]sequencer
	ids          map[string]string

	file         *os.File
	originalSize int64
}

func (t *Tag) AddFrame(id string, f Framer) {
	if addFunc := t.findSpecificAddFunction(id); addFunc != nil {
		addFunc(f)
	} else {
		t.frames[id] = f
	}
}

// ID returns ID3v2.3 or ID3v2.4 (in appropriate to version of Tag) frame ID
// from given description.
// For example, ID("Mood") will return "TMOO".
func (t Tag) ID(description string) string {
	return t.ids[description]
}

func (t *Tag) findSpecificAddFunction(id string) func(Framer) {
	switch id {
	case t.ids["Attached picture"]:
		return t.addAttachedPicture
	case t.ids["Comments"]:
		return t.addCommentFrame
	case t.ids["Unsynchronised lyrics/text transcription"]:
		return t.addUnsynchronisedLyricsFrame
	}
	return nil
}

func (t *Tag) addAttachedPicture(f Framer) {
	id := t.ids["Attached picture"]
	t.checkExistenceOfSequence(id, newPictureSequence)
	t.addFrameToSequence(id, f)
}

func (t *Tag) addCommentFrame(f Framer) {
	id := t.ids["Comments"]
	t.checkExistenceOfSequence(id, newCommentSequence)
	t.addFrameToSequence(id, f)
}

func (t *Tag) addUnsynchronisedLyricsFrame(f Framer) {
	id := t.ids["Unsynchronised lyrics/text transcription"]
	t.checkExistenceOfSequence(id, newUSLFSequence)
	t.addFrameToSequence(id, f)
}

func (t *Tag) checkExistenceOfSequence(id string, newSequence func() sequencer) {
	if t.sequences[id] == nil {
		t.sequences[id] = newSequence()
	}
}

func (t *Tag) addFrameToSequence(id string, f Framer) {
	t.sequences[id].AddFrame(f)
}

// AllFrames returns map, that contains all frames in tag, that could be parsed.
// The key of this map is an ID of frame and value is an array of frames.
func (t *Tag) AllFrames() map[string][]Framer {
	t.parseAllFramesCoords()

	frames := make(map[string][]Framer)

	// Add frames from t.frames
	for id, frame := range t.frames {
		frames[id] = append(frames[id], frame)
	}

	// Add frames from t.sequences
	for id, sequence := range t.sequences {
		frames[id] = append(frames[id], sequence.Frames()...)
	}

	return frames
}

// GetLastFrame returns last frame from slice, that is returned from GetFrames function.
// GetLastFrame is suitable for frames, that can be only one in whole tag.
// For example, for text frames.
//
// Example of usage:
//	bpmFramer := t.GetLastFrame(t.ID("BPM"))
//	bpm, ok := bpmFramer.(id3v2.TextFrame)
//	if !ok {
//		log.Fatal("Couldn't assert bpm frame")
//	}
//	fmt.Println(bpm.Text)
func (t *Tag) GetLastFrame(id string) Framer {
	fs := t.GetFrames(id)
	if len(fs) == 0 || fs == nil {
		return nil
	}
	return fs[len(fs)-1]
}

// GetFrames returns frames with corresponding id.
//
// Example of usage:
//	pictures := tag.GetFrames(tag.ID("Attached picture"))
//	for _, f := range pictures {
//		pic, ok := f.(id3v2.PictureFrame)
//		if !ok {
//			log.Fatal("Couldn't assert picture frame")
//		}
//
//		// Do some operations with picture frame:
//		fmt.Println(pic.Description) // For example, print description of picture frame
//		image, err := ioutil.ReadAll(pic.Picture) // Or read a picture from picture frame
//		if err != nil {
//			log.Fatal("Error while reading a picture from picture frame: ", err)
//		}
//	}
func (t *Tag) GetFrames(id string) []Framer {
	// If frames with id didn't parsed yet, parse them
	if _, exists := t.framesCoords[id]; exists {
		t.parseFramesCoordsWithID(id)
	}

	if f, exists := t.frames[id]; exists {
		return []Framer{f}
	}

	if s, exists := t.sequences[id]; exists {
		return s.Frames()
	}

	return nil
}

// GetFrames returns text frame with corresponding id.
func (t Tag) GetTextFrame(id string) TextFrame {
	f := t.GetLastFrame(id)
	if f == nil {
		return TextFrame{}
	}
	tf := f.(TextFrame)
	return tf
}

func (t Tag) Title() string {
	f := t.GetTextFrame(t.ids["Title/Songname/Content description"])
	return f.Text
}

func (t *Tag) SetTitle(title string) {
	t.AddFrame(t.ids["Title/Songname/Content description"], TextFrame{Encoding: ENUTF8, Text: title})
}

func (t Tag) Artist() string {
	f := t.GetTextFrame(t.ids["Lead artist/Lead performer/Soloist/Performing group"])
	return f.Text
}

func (t *Tag) SetArtist(artist string) {
	t.AddFrame(t.ids["Lead artist/Lead performer/Soloist/Performing group"], TextFrame{Encoding: ENUTF8, Text: artist})
}

func (t Tag) Album() string {
	f := t.GetTextFrame(t.ids["Album/Movie/Show title"])
	return f.Text
}

func (t *Tag) SetAlbum(album string) {
	t.AddFrame(t.ids["Album/Movie/Show title"], TextFrame{Encoding: ENUTF8, Text: album})
}

func (t Tag) Year() string {
	f := t.GetTextFrame(t.ids["Recording time"])
	return f.Text
}

func (t *Tag) SetYear(year string) {
	t.AddFrame(t.ids["Recording time"], TextFrame{Encoding: ENUTF8, Text: year})
}

func (t Tag) Genre() string {
	f := t.GetTextFrame(t.ids["Content type"])
	return f.Text
}

func (t *Tag) SetGenre(genre string) {
	t.AddFrame(t.ids["Content type"], TextFrame{Encoding: ENUTF8, Text: genre})
}

// Save writes tag to the file.
func (t *Tag) Save() error {
	// Forming new frames
	frames := t.formAllFrames()

	// Forming size of new frames
	framesSize := util.FormSize(int64(len(frames)))

	// Creating a temp file for mp3 file, which will contain new tag
	newFile, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}

	// Writing to new file new tag header
	if _, err = newFile.Write(formTagHeader(framesSize)); err != nil {
		return err
	}

	// Writing to new file new frames
	if _, err = newFile.Write(frames); err != nil {
		return err
	}

	// Seeking to a music part of mp3
	originalFile := t.file
	defer originalFile.Close()
	if _, err = originalFile.Seek(t.originalSize, os.SEEK_SET); err != nil {
		return err
	}

	// Writing to new file the music part
	if _, err = io.Copy(newFile, originalFile); err != nil {
		return err
	}

	// Getting original file mode
	originalFileStat, err := originalFile.Stat()
	if err != nil {
		return err
	}
	originalFileMode := originalFileStat.Mode()

	// Setting new file mode
	if err = newFile.Chmod(originalFileMode); err != nil {
		return err
	}

	// Replacing original file with new file
	if err = os.Rename(newFile.Name(), originalFile.Name()); err != nil {
		return err
	}
	t.file = newFile

	return nil
}

// Close closes the tag's file, rendering it unusable for I/O.
// It returns an error, if any.
func (t *Tag) Close() error {
	return t.file.Close()
}

func (t Tag) formAllFrames() []byte {
	framesBuffer := bytesbufferpool.Get()
	defer bytesbufferpool.Put(framesBuffer)

	t.writeFrames(framesBuffer)

	return framesBuffer.Bytes()
}

func (t Tag) writeFrames(w io.Writer) {
	for id, frames := range t.AllFrames() {
		for _, f := range frames {
			w.Write(formFrame(id, f))
		}
	}
}

func formFrame(id string, frame Framer) []byte {
	if id == "" {
		panic("there is blank ID in frames")
	}

	frameBuffer := bytesbufferpool.Get()
	defer bytesbufferpool.Put(frameBuffer)

	frameBody := frame.Body()
	writeFrameHeader(frameBuffer, id, int64(len(frameBody)))
	frameBuffer.Write(frameBody)

	return frameBuffer.Bytes()
}

func writeFrameHeader(buf *bytes.Buffer, id string, frameSize int64) {
	buf.WriteString(id)
	buf.Write(util.FormSize(frameSize))
	buf.Write([]byte{0, 0})
}
<commit_msg>Fix documentation of GetTextFrame<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package id3v2

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"

	"github.com/bogem/id3v2/bytesbufferpool"
	"github.com/bogem/id3v2/util"
)

// Tag stores all frames of opened file.
type Tag struct {
	framesCoords map[string][]frameCoordinates
	frames       map[string]Framer
	sequences    map[string]sequencer
	ids          map[string]string

	file         *os.File
	originalSize int64
}

func (t *Tag) AddFrame(id string, f Framer) {
	if addFunc := t.findSpecificAddFunction(id); addFunc != nil {
		addFunc(f)
	} else {
		t.frames[id] = f
	}
}

// ID returns ID3v2.3 or ID3v2.4 (in appropriate to version of Tag) frame ID
// from given description.
// For example, ID("Mood") will return "TMOO".
func (t Tag) ID(description string) string {
	return t.ids[description]
}

func (t *Tag) findSpecificAddFunction(id string) func(Framer) {
	switch id {
	case t.ids["Attached picture"]:
		return t.addAttachedPicture
	case t.ids["Comments"]:
		return t.addCommentFrame
	case t.ids["Unsynchronised lyrics/text transcription"]:
		return t.addUnsynchronisedLyricsFrame
	}
	return nil
}

func (t *Tag) addAttachedPicture(f Framer) {
	id := t.ids["Attached picture"]
	t.checkExistenceOfSequence(id, newPictureSequence)
	t.addFrameToSequence(id, f)
}

func (t *Tag) addCommentFrame(f Framer) {
	id := t.ids["Comments"]
	t.checkExistenceOfSequence(id, newCommentSequence)
	t.addFrameToSequence(id, f)
}

func (t *Tag) addUnsynchronisedLyricsFrame(f Framer) {
	id := t.ids["Unsynchronised lyrics/text transcription"]
	t.checkExistenceOfSequence(id, newUSLFSequence)
	t.addFrameToSequence(id, f)
}

func (t *Tag) checkExistenceOfSequence(id string, newSequence func() sequencer) {
	if t.sequences[id] == nil {
		t.sequences[id] = newSequence()
	}
}

func (t *Tag) addFrameToSequence(id string, f Framer) {
	t.sequences[id].AddFrame(f)
}

// AllFrames returns map, that contains all frames in tag, that could be parsed.
// The key of this map is an ID of frame and value is an array of frames.
func (t *Tag) AllFrames() map[string][]Framer {
	t.parseAllFramesCoords()

	frames := make(map[string][]Framer)

	// Add frames from t.frames
	for id, frame := range t.frames {
		frames[id] = append(frames[id], frame)
	}

	// Add frames from t.sequences
	for id, sequence := range t.sequences {
		frames[id] = append(frames[id], sequence.Frames()...)
	}

	return frames
}

// GetLastFrame returns last frame from slice, that is returned from GetFrames function.
// GetLastFrame is suitable for frames, that can be only one in whole tag.
// For example, for text frames.
//
// Example of usage:
//	bpmFramer := t.GetLastFrame(t.ID("BPM"))
//	bpm, ok := bpmFramer.(id3v2.TextFrame)
//	if !ok {
//		log.Fatal("Couldn't assert bpm frame")
//	}
//	fmt.Println(bpm.Text)
func (t *Tag) GetLastFrame(id string) Framer {
	fs := t.GetFrames(id)
	if len(fs) == 0 || fs == nil {
		return nil
	}
	return fs[len(fs)-1]
}

// GetFrames returns frames with corresponding id.
//
// Example of usage:
//	pictures := tag.GetFrames(tag.ID("Attached picture"))
//	for _, f := range pictures {
//		pic, ok := f.(id3v2.PictureFrame)
//		if !ok {
//			log.Fatal("Couldn't assert picture frame")
//		}
//
//		// Do some operations with picture frame:
//		fmt.Println(pic.Description) // For example, print description of picture frame
//		image, err := ioutil.ReadAll(pic.Picture) // Or read a picture from picture frame
//		if err != nil {
//			log.Fatal("Error while reading a picture from picture frame: ", err)
//		}
//	}
func (t *Tag) GetFrames(id string) []Framer {
	// If frames with id didn't parsed yet, parse them
	if _, exists := t.framesCoords[id]; exists {
		t.parseFramesCoordsWithID(id)
	}

	if f, exists := t.frames[id]; exists {
		return []Framer{f}
	}

	if s, exists := t.sequences[id]; exists {
		return s.Frames()
	}

	return nil
}

// GetTextFrame returns text frame with corresponding id.
func (t Tag) GetTextFrame(id string) TextFrame {
	f := t.GetLastFrame(id)
	if f == nil {
		return TextFrame{}
	}
	tf := f.(TextFrame)
	return tf
}

func (t Tag) Title() string {
	f := t.GetTextFrame(t.ids["Title/Songname/Content description"])
	return f.Text
}

func (t *Tag) SetTitle(title string) {
	t.AddFrame(t.ids["Title/Songname/Content description"], TextFrame{Encoding: ENUTF8, Text: title})
}

func (t Tag) Artist() string {
	f := t.GetTextFrame(t.ids["Lead artist/Lead performer/Soloist/Performing group"])
	return f.Text
}

func (t *Tag) SetArtist(artist string) {
	t.AddFrame(t.ids["Lead artist/Lead performer/Soloist/Performing group"], TextFrame{Encoding: ENUTF8, Text: artist})
}

func (t Tag) Album() string {
	f := t.GetTextFrame(t.ids["Album/Movie/Show title"])
	return f.Text
}

func (t *Tag) SetAlbum(album string) {
	t.AddFrame(t.ids["Album/Movie/Show title"], TextFrame{Encoding: ENUTF8, Text: album})
}

func (t Tag) Year() string {
	f := t.GetTextFrame(t.ids["Recording time"])
	return f.Text
}

func (t *Tag) SetYear(year string) {
	t.AddFrame(t.ids["Recording time"], TextFrame{Encoding: ENUTF8, Text: year})
}

func (t Tag) Genre() string {
	f := t.GetTextFrame(t.ids["Content type"])
	return f.Text
}

func (t *Tag) SetGenre(genre string) {
	t.AddFrame(t.ids["Content type"], TextFrame{Encoding: ENUTF8, Text: genre})
}

// Save writes tag to the file.
func (t *Tag) Save() error {
	// Forming new frames
	frames := t.formAllFrames()

	// Forming size of new frames
	framesSize := util.FormSize(int64(len(frames)))

	// Creating a temp file for mp3 file, which will contain new tag
	newFile, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}

	// Writing to new file new tag header
	if _, err = newFile.Write(formTagHeader(framesSize)); err != nil {
		return err
	}

	// Writing to new file new frames
	if _, err = newFile.Write(frames); err != nil {
		return err
	}

	// Seeking to a music part of mp3
	originalFile := t.file
	defer originalFile.Close()
	if _, err = originalFile.Seek(t.originalSize, os.SEEK_SET); err != nil {
		return err
	}

	// Writing to new file the music part
	if _, err = io.Copy(newFile, originalFile); err != nil {
		return err
	}

	// Getting original file mode
	originalFileStat, err := originalFile.Stat()
	if err != nil {
		return err
	}
	originalFileMode := originalFileStat.Mode()

	// Setting new file mode
	if err = newFile.Chmod(originalFileMode); err != nil {
		return err
	}

	// Replacing original file with new file
	if err = os.Rename(newFile.Name(), originalFile.Name()); err != nil {
		return err
	}
	t.file = newFile

	return nil
}

// Close closes the tag's file, rendering it unusable for I/O.
// It returns an error, if any.
func (t *Tag) Close() error {
	return t.file.Close()
}

func (t Tag) formAllFrames() []byte {
	framesBuffer := bytesbufferpool.Get()
	defer bytesbufferpool.Put(framesBuffer)

	t.writeFrames(framesBuffer)

	return framesBuffer.Bytes()
}

func (t Tag) writeFrames(w io.Writer) {
	for id, frames := range t.AllFrames() {
		for _, f := range frames {
			w.Write(formFrame(id, f))
		}
	}
}

func formFrame(id string, frame Framer) []byte {
	if id == "" {
		panic("there is blank ID in frames")
	}

	frameBuffer := bytesbufferpool.Get()
	defer bytesbufferpool.Put(frameBuffer)

	frameBody := frame.Body()
	writeFrameHeader(frameBuffer, id, int64(len(frameBody)))
	frameBuffer.Write(frameBody)

	return frameBuffer.Bytes()
}

func writeFrameHeader(buf *bytes.Buffer, id string, frameSize int64) {
	buf.WriteString(id)
	buf.Write(util.FormSize(frameSize))
	buf.Write([]byte{0, 0})
}
<|endoftext|>


<commit_before>package gopath

import (
	"reflect"
	"strings"
)

func Find(path string, container interface{}) []interface{} {
	var results = []interface{}{}
	if path == "" {
		path = "*"
	} else {
		path = strings.Trim(path, "/")
	}

	splitPath := strings.Split(path, "/")

	find(splitPath, reflect.ValueOf(container), &results)

	return results
}

func FindClean(path string, container interface{}) []interface{} {
	var results = Find(path, container)
	var clean = []interface{}{}
	var m = map[interface{}]string{}

	for _, val := range results {
		m[val] = ""
	}

	for key, _ := range m {
		if reflect.ValueOf(key) != reflect.Zero(reflect.TypeOf(key)) {
			clean = append(clean, key)
		}
	}

	return clean
}

func find(path []string, val reflect.Value, results *[]interface{}) {
	pv := val
	if pv.Kind() == reflect.Ptr {
		if pv.IsNil() {
			return
		}
		val = pv.Elem()
	}

	if len(path) == 0 {
		appendValue(results, pv)
		return
	}

	switch val.Kind() {
	case reflect.Struct:
		findStruct(path, pv, results)
	case reflect.Slice:
		findSlice(path, pv, results)
	case reflect.Map:
		findMap(path, pv, results)
	}
}

func appendValue(results *[]interface{}, val reflect.Value) {
	pv := val
	if pv.Kind() == reflect.Ptr {
		val = reflect.Indirect(pv)
	}

	if val.Kind() == reflect.Struct {
		if val.CanAddr() {
			*results = append(*results, val.Addr().Interface())
			return
		}
	}
	*results = append(*results, val.Interface())
}

func findStruct(path []string, val reflect.Value, results *[]interface{}) {
	pv := val
	if pv.Kind() == reflect.Ptr {
		if pv.IsNil() {
			pv.Set(reflect.New(pv.Type().Elem()))
		}
		val = pv.Elem()
	}

	t := val.Type()

	switch {
	case path[0] == "**":
		for i := 0; i < t.NumField(); i++ {
			field := t.Field(i)
			value := val.Field(i)
			if field.Anonymous {
				find(path, value, results)
				continue
			}
			if path[1] == field.Name {
				find(path[2:], value, results)
			} else {
				find(path, value, results)
			}
		}
	case path[0] == "*":
		for i := 0; i < t.NumField(); i++ {
			field := t.Field(i)
			value := val.Field(i)
			if field.Anonymous {
				find(path, value, results)
				continue
			}

			find(path[1:], value, results)
		}
	default:
		value := val.FieldByName(path[0])
		if value.IsValid() {
			find(path[1:], value, results)
		}
	}
}

func findMap(path []string, val reflect.Value, results *[]interface{}) {
	pv := val
	if pv.Kind() == reflect.Ptr {
		if pv.IsNil() {
			pv.Set(reflect.New(pv.Type().Elem()))
		}
		val = pv.Elem()
	}

	switch {
	case path[0] == "**":
		for _, key := range val.MapKeys() {
			value := val.MapIndex(key)
			if path[1] == key.String() {
				find(path[2:], value, results)
			} else {
				find(path, value, results)
			}
		}
	case path[0] == "*":
		for _, key := range val.MapKeys() {
			value := val.MapIndex(key)
			find(path[1:], value, results)
		}
	default:
		value := val.MapIndex(reflect.ValueOf(path[0]))
		if value.IsValid() {
			find(path[1:], value, results)
		}
	}
}

func findSlice(path []string, val reflect.Value, results *[]interface{}) {
	pv := val
	if pv.Kind() == reflect.Ptr {
		if pv.IsNil() {
			pv.Set(reflect.New(pv.Type().Elem()))
		}
		val = pv.Elem()
	}

	switch {
	case path[0] == "**":
		for i := 0; i < val.Len(); i++ {
			value := val.Index(i)
			find(path, value, results)
		}
	case path[0] == "*":
		for i := 0; i < val.Len(); i++ {
			value := val.Index(i)
			find(path[1:], value, results)
		}
	}

}
<commit_msg>Switched to using iterator<commit_after>package gopath

import (
	"reflect"
	"strings"
)

type Iter struct {
	values []reflect.Value
	index  int
	next   reflect.Value
}

func (self *Iter) Next() bool {
	self.next = reflect.Value{}
	if self.index > len(self.values) {
		return false
	}
	self.next = self.values[self.index]
	self.index += 1
	return true
}

func (self *Iter) Value() interface{} {
	if self.next.IsValid() {
		return self.next.Interface()
	}
	return nil
}

type Path struct {
	results   []reflect.Value
	container reflect.Value
	path      []string
}

func NewPath(path string, container interface{}) *Path {
	if path == "" {
		path = "*"
	} else {
		path = strings.Trim(path, "/")
	}

	splitPath := strings.Split(path, "/")

	p := &Path{path: splitPath, container: reflect.ValueOf(container)}
	p.results = []reflect.Value{}
	return p
}

func (self *Path) Iter() *Iter {
	return &Iter{values: self.results}
}

func (self *Path) appendValue(val reflect.Value) {
	pv := val
	if pv.Kind() == reflect.Ptr {
		val = reflect.Indirect(pv)
	}

	// if val.Kind() == reflect.Struct {
	// 	if val.CanAddr() {
	// 		*results = append(*results, val.Addr().Interface())
	// 		return
	// 	}
	// }
	self.results = append(self.results, val)
}

func (self *Path) find(path []string, val reflect.Value) {
	pv := val
	if pv.Kind() == reflect.Ptr {
		if pv.IsNil() {
			return
		}
		val = pv.Elem()
	}

	if len(path) == 0 {
		self.appendValue(pv)
		return
	}

	switch val.Kind() {
	case reflect.Struct:
		self.findStruct(path, pv)
	case reflect.Slice:
		self.findSlice(path, pv)
	case reflect.Map:
		self.findMap(path, pv)
	}
}

func (self *Path) findStruct(path []string, val reflect.Value) {
	pv := val
	if pv.Kind() == reflect.Ptr {
		if pv.IsNil() {
			pv.Set(reflect.New(pv.Type().Elem()))
		}
		val = pv.Elem()
	}

	t := val.Type()

	switch {
	case path[0] == "**":
		for i := 0; i < t.NumField(); i++ {
			field := t.Field(i)
			value := val.Field(i)
			if field.Anonymous {
				self.find(path, value)
				continue
			}
			if path[1] == field.Name {
				self.find(path[2:], value)
			} else {
				self.find(path, value)
			}
		}
	case path[0] == "*":
		for i := 0; i < t.NumField(); i++ {
			field := t.Field(i)
			value := val.Field(i)
			if field.Anonymous {
				self.find(path, value)
				continue
			}

			self.find(path[1:], value)
		}
	default:
		value := val.FieldByName(path[0])
		if value.IsValid() {
			self.find(path[1:], value)
		}
	}
}

func (self *Path) findMap(path []string, val reflect.Value) {
	pv := val
	if pv.Kind() == reflect.Ptr {
		if pv.IsNil() {
			pv.Set(reflect.New(pv.Type().Elem()))
		}
		val = pv.Elem()
	}

	switch {
	case path[0] == "**":
		for _, key := range val.MapKeys() {
			value := val.MapIndex(key)
			if path[1] == key.String() {
				self.find(path[2:], value)
			} else {
				self.find(path, value)
			}
		}
	case path[0] == "*":
		for _, key := range val.MapKeys() {
			value := val.MapIndex(key)
			self.find(path[1:], value)
		}
	default:
		value := val.MapIndex(reflect.ValueOf(path[0]))
		if value.IsValid() {
			self.find(path[1:], value)
		}
	}
}

func (self *Path) findSlice(path []string, val reflect.Value) {
	pv := val
	if pv.Kind() == reflect.Ptr {
		if pv.IsNil() {
			pv.Set(reflect.New(pv.Type().Elem()))
		}
		val = pv.Elem()
	}

	switch {
	case path[0] == "**":
		for i := 0; i < val.Len(); i++ {
			value := val.Index(i)
			self.find(path, value)
		}
	case path[0] == "*":
		for i := 0; i < val.Len(); i++ {
			value := val.Index(i)
			self.find(path[1:], value)
		}
	}

}
<|endoftext|>


<commit_before>package paza

import (
	"fmt"
	"regexp"
	"strconv"
	"sync/atomic"
	"unicode/utf8"
)

var (
	pt = fmt.Printf
)

type parserInfo struct {
	parser    Parser
	recursive bool
}

type Set struct {
	parsers map[string]parserInfo
	serial  uint64
}

type Parser func(input *Input, start int) (ok bool, n int)

type stackEntry struct {
	parser string
	start  int
	ok     bool
	length int
}

type Input struct {
	text  []byte
	stack []stackEntry
}

func NewInput(text []byte) *Input {
	return &Input{
		text: text,
	}
}

func NewSet() *Set {
	return &Set{
		parsers: make(map[string]parserInfo),
	}
}

func (s *Set) Add(name string, parser Parser) {
	s.parsers[name] = parserInfo{parser, false}
}

func (s *Set) AddRec(name string, parser Parser) {
	s.parsers[name] = parserInfo{parser, true}
}

func (s *Set) Regex(re string) Parser {
	regex := regexp.MustCompile(re)
	return func(input *Input, start int) (bool, int) {
		if loc := regex.FindIndex(input.text[start:]); loc != nil && loc[0] == 0 {
			return true, loc[1]
		}
		return false, 0
	}
}

func (s *Set) Rune(r rune) Parser {
	return func(input *Input, start int) (bool, int) {
		ru, l := utf8.DecodeRune(input.text[start:])
		if ru == utf8.RuneError {
			return false, 0
		}
		if ru != r {
			return false, 0
		}
		return true, l
	}
}

func (s *Set) Call(name string, input *Input, start int) (bool, int) {
	info, ok := s.parsers[name]
	if !ok {
		panic("parser not found " + name)
	}

	// non recursive parser
	if !info.recursive {
		return info.parser(input, start)
	}

	// search stack
	for i := len(input.stack) - 1; i >= 0; i-- {
		mem := input.stack[i]
		if mem.parser == name && mem.start == start { // found
			return mem.ok, mem.length
		}
	}
	// not found, append a new entry
	input.stack = append(input.stack, stackEntry{
		parser: name,
		start:  start,
		ok:     false,
		length: 0,
	})
	// find the right bound
	lastOk := false
	lastLen := 0
	for {
		stackSize := len(input.stack) // save stack size
		ok, l := info.parser(input, start)
		input.stack = input.stack[:stackSize] // unwind stack
		if !ok {
			return false, 0
		}
		if l < lastLen { // over bound
			return lastOk, lastLen
		} else if l == lastLen { // not extending
			return ok, l
		}
		lastOk = ok
		lastLen = l
		// update stack
		for i := len(input.stack) - 1; i >= 0; i-- {
			e := input.stack[i]
			if e.parser == name && e.start == start {
				input.stack[i].ok = ok
				input.stack[i].length = l
				break
			}
		}
	}

}

func (s *Set) getNames(parsers []interface{}) (ret []string) {
	for _, parser := range parsers {
		switch parser := parser.(type) {
		case string:
			ret = append(ret, parser)
		case Parser:
			name := "__parser__" + strconv.Itoa(int(atomic.AddUint64(&s.serial, 1)))
			s.Add(name, parser)
			ret = append(ret, name)
		default:
			panic(fmt.Sprintf("unknown parser type %T", parser))
		}
	}
	return
}

func (s *Set) Concat(parsers ...interface{}) Parser {
	names := s.getNames(parsers)
	return func(input *Input, start int) (bool, int) {
		index := start
		for _, name := range names {
			if ok, l := s.Call(name, input, index); !ok {
				return false, 0
			} else {
				index += l
			}
		}
		return true, index - start
	}
}

func (s *Set) OrdChoice(parsers ...interface{}) Parser {
	names := s.getNames(parsers)
	return func(input *Input, start int) (bool, int) {
		for _, name := range names {
			if ok, l := s.Call(name, input, start); ok {
				return ok, l
			}
		}
		return false, 0
	}
}
<commit_msg>minor optimize<commit_after>package paza

import (
	"fmt"
	"regexp"
	"strconv"
	"sync/atomic"
	"unicode/utf8"
)

var (
	pt = fmt.Printf
)

type parserInfo struct {
	parser    Parser
	recursive bool
}

type Set struct {
	parsers map[string]parserInfo
	serial  uint64
}

type Parser func(input *Input, start int) (ok bool, n int)

type stackEntry struct {
	parser string
	start  int
	ok     bool
	length int
}

type Input struct {
	text  []byte
	stack []stackEntry
}

func NewInput(text []byte) *Input {
	return &Input{
		text: text,
	}
}

func NewSet() *Set {
	return &Set{
		parsers: make(map[string]parserInfo),
	}
}

func (s *Set) Add(name string, parser Parser) {
	s.parsers[name] = parserInfo{parser, false}
}

func (s *Set) AddRec(name string, parser Parser) {
	s.parsers[name] = parserInfo{parser, true}
}

func (s *Set) Regex(re string) Parser {
	regex := regexp.MustCompile(re)
	return func(input *Input, start int) (bool, int) {
		if loc := regex.FindIndex(input.text[start:]); loc != nil && loc[0] == 0 {
			return true, loc[1]
		}
		return false, 0
	}
}

func (s *Set) Rune(r rune) Parser {
	return func(input *Input, start int) (bool, int) {
		ru, l := utf8.DecodeRune(input.text[start:])
		if ru == utf8.RuneError {
			return false, 0
		}
		if ru != r {
			return false, 0
		}
		return true, l
	}
}

func (s *Set) Call(name string, input *Input, start int) (bool, int) {
	info, ok := s.parsers[name]
	if !ok {
		panic("parser not found " + name)
	}

	// non recursive parser
	if !info.recursive {
		return info.parser(input, start)
	}

	// search stack
	for i := len(input.stack) - 1; i >= 0; i-- {
		mem := input.stack[i]
		if mem.parser == name && mem.start == start { // found
			return mem.ok, mem.length
		}
	}
	// not found, append a new entry
	input.stack = append(input.stack, stackEntry{
		parser: name,
		start:  start,
		ok:     false,
		length: 0,
	})
	// find the right bound
	lastOk := false
	lastLen := 0
	stackSize := len(input.stack) // save stack size
	for {
		ok, l := info.parser(input, start)
		input.stack = input.stack[:stackSize] // unwind stack
		if !ok {
			return false, 0
		}
		if l < lastLen { // over bound
			return lastOk, lastLen
		} else if l == lastLen { // not extending
			return ok, l
		}
		lastOk = ok
		lastLen = l
		// update stack
		for i := len(input.stack) - 1; i >= 0; i-- {
			e := input.stack[i]
			if e.parser == name && e.start == start {
				input.stack[i].ok = ok
				input.stack[i].length = l
				break
			}
		}
	}

}

func (s *Set) getNames(parsers []interface{}) (ret []string) {
	for _, parser := range parsers {
		switch parser := parser.(type) {
		case string:
			ret = append(ret, parser)
		case Parser:
			name := "__parser__" + strconv.Itoa(int(atomic.AddUint64(&s.serial, 1)))
			s.Add(name, parser)
			ret = append(ret, name)
		default:
			panic(fmt.Sprintf("unknown parser type %T", parser))
		}
	}
	return
}

func (s *Set) Concat(parsers ...interface{}) Parser {
	names := s.getNames(parsers)
	return func(input *Input, start int) (bool, int) {
		index := start
		for _, name := range names {
			if ok, l := s.Call(name, input, index); !ok {
				return false, 0
			} else {
				index += l
			}
		}
		return true, index - start
	}
}

func (s *Set) OrdChoice(parsers ...interface{}) Parser {
	names := s.getNames(parsers)
	return func(input *Input, start int) (bool, int) {
		for _, name := range names {
			if ok, l := s.Call(name, input, start); ok {
				return ok, l
			}
		}
		return false, 0
	}
}
<|endoftext|>


<commit_before>package main

import (
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	"github.com/btcsuite/btcd/wire"
	"github.com/op/go-logging"

	"github.com/CIRCL/pbtc/logger"
	"github.com/CIRCL/pbtc/manager"
	"github.com/CIRCL/pbtc/recorder"
	"github.com/CIRCL/pbtc/repository"
)

func main() {
	// catch signals
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Interrupt)

	// use all cpu cores
	runtime.GOMAXPROCS(runtime.NumCPU())

	// seed the random generator
	rand.Seed(time.Now().UnixNano())

	// logging
	log, err := logger.New(
		logger.EnableConsole(),
		logger.SetConsoleLevel(logging.INFO),
		logger.EnableFile(),
		logger.SetFileLevel(logging.DEBUG),
	)
	if err != nil {
		os.Exit(1)
	}

	// repository
	repo, err := repository.New(
		repository.SetLogger(log),
		repository.SetSeeds("seed.bitcoin.sipa.be"),
		repository.SetDefaultPort(8333),
		repository.DisableRestore(),
	)
	if err != nil {
		log.Critical("Unable to create repository (%v)", err)
		os.Exit(1)
	}

	// recorder
	rec, err := recorder.New(
		recorder.SetLogger(log),
		recorder.SetTypes(wire.CmdTx, wire.CmdVersion, wire.CmdInv,
			wire.CmdAddr),
		recorder.SetFileSize(0),
		recorder.SetFileAge(time.Minute*5),
		recorder.EnableReset(),
	)
	if err != nil {
		log.Critical("Unable to initialize recorder (%v)", err)
		os.Exit(1)
	}

	// manager
	mgr, err := manager.New(
		manager.SetLogger(log),
		manager.SetRepository(repo),
		manager.SetRecorder(rec),
		manager.SetNetwork(wire.MainNet),
		manager.SetVersion(wire.RejectVersion),
		manager.SetConnectionRate(time.Second/25),
		manager.SetInformationRate(time.Second),
		manager.SetPeerLimit(100),
	)
	if err != nil {
		log.Critical("Unable to create manager (%v)", err)
		os.Exit(1)
	}

	// wait for signals in blocking loop
SigLoop:
	for sig := range sigc {
		log.Notice("Signal caught (%v)", sig.String())

		switch sig {
		case syscall.SIGINT:
			break SigLoop
		}
	}

	mgr.Stop()
	repo.Stop()

	os.Exit(0)
}
<commit_msg>Set testing parameters<commit_after>package main

import (
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	"github.com/btcsuite/btcd/wire"
	"github.com/op/go-logging"

	"github.com/CIRCL/pbtc/logger"
	"github.com/CIRCL/pbtc/manager"
	"github.com/CIRCL/pbtc/recorder"
	"github.com/CIRCL/pbtc/repository"
)

func main() {
	// catch signals
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Interrupt)

	// use all cpu cores
	runtime.GOMAXPROCS(runtime.NumCPU())

	// seed the random generator
	rand.Seed(time.Now().UnixNano())

	// logging
	log, err := logger.New(
		logger.EnableConsole(),
		logger.SetConsoleLevel(logging.INFO),
		logger.EnableFile(),
		logger.SetFileLevel(logging.DEBUG),
	)
	if err != nil {
		os.Exit(1)
	}

	// repository
	repo, err := repository.New(
		repository.SetLogger(log),
		repository.SetSeeds("seed.bitcoin.sipa.be"),
		repository.SetDefaultPort(8333),
		repository.DisableRestore(),
	)
	if err != nil {
		log.Critical("Unable to create repository (%v)", err)
		os.Exit(1)
	}

	// recorder
	rec, err := recorder.New(
		recorder.SetLogger(log),
		recorder.SetTypes(wire.CmdTx, wire.CmdVersion, wire.CmdInv,
			wire.CmdAddr),
		recorder.SetFileSize(0),
		recorder.SetFileAge(time.Minute*5),
		recorder.EnableReset(),
	)
	if err != nil {
		log.Critical("Unable to initialize recorder (%v)", err)
		os.Exit(1)
	}

	// manager
	mgr, err := manager.New(
		manager.SetLogger(log),
		manager.SetRepository(repo),
		manager.SetRecorder(rec),
		manager.SetNetwork(wire.MainNet),
		manager.SetVersion(wire.RejectVersion),
		manager.SetConnectionRate(time.Second/25),
		manager.SetInformationRate(time.Second*5),
		manager.SetPeerLimit(1000),
	)
	if err != nil {
		log.Critical("Unable to create manager (%v)", err)
		os.Exit(1)
	}

	// wait for signals in blocking loop
SigLoop:
	for sig := range sigc {
		log.Notice("Signal caught (%v)", sig.String())

		switch sig {
		case syscall.SIGINT:
			break SigLoop
		}
	}

	mgr.Stop()
	repo.Stop()

	os.Exit(0)
}
<|endoftext|>