<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"
)

type LibratoMetric struct {
	Name   string      `json:"name"`
	Value  interface{} `json:"value"`
	When   int64       `json:"measure_time"`
	Source string      `json:"source,omitempty"`
}

type PostBody struct {
	Gauges   []LibratoMetric `json:"gauges,omitempty"`
	Counters []LibratoMetric `json:"counters,omitempty"`
}

const (
	LibratoBacklog = 8 // No more than N pending batches in-flight
	LibratoMaxAttempts = 4 // Max attempts before dropping batch
	LibratoStartingBackoffMillis = 200 * time.Millisecond
)

type Librato struct {
	Timeout      time.Duration
	BatchSize    int
	User         string
	Token        string
	Url          string
	measurements <-chan *Measurement
	batches      chan []*Measurement
	prefix       string
	source       string
	client       *http.Client
}

func NewLibratoOutputter(measurements <-chan *Measurement, config Config) *Librato {
	return &Librato{
		measurements: measurements,
	  batches:      make(chan []*Measurement, LibratoBacklog),
		Timeout:      config.LibratoBatchTimeout,
		BatchSize:    config.LibratoBatchSize,
		User:         config.LibratoUser,
		Token:        config.LibratoToken,
	  Url:          config.LibratoUrl,
	  client:       &http.Client{
			Transport: &http.Transport{
				ResponseHeaderTimeout: config.LibratoNetworkTimeout,
				Dial: func(network, address string) (net.Conn, error) {
					return net.DialTimeout(network, address, config.LibratoNetworkTimeout)
				},
			},
		},
	}
}

func (out *Librato) Start() {
	go out.deliver()
	go out.batch()
}

func (out *Librato) makeBatch() []*Measurement {
	return make([]*Measurement, 0, out.BatchSize)
}

func (out *Librato) batch() {
	var ready bool
	ctx := Slog{"fn": "batch", "outputter": "librato"}
	ticker := time.Tick(out.Timeout)
	batch := out.makeBatch()
	for {
		select {
		case measurement := <-out.measurements:
			batch = append(batch, measurement)
			if len(batch) == cap(batch) {
				ready = true
			}
		case <-ticker:
			if len(batch) > 0 {
				ready = true
			}
		}

		if ready {
			select {
			case out.batches <- batch:
			default:
				ctx.Error(nil, "Batches backlogged, dropping")
			}
			batch = out.makeBatch()
			ready = false
		}
	}
}

func (out *Librato) deliver() {
	ctx := Slog{"fn": "prepare", "outputter": "librato"}
	for batch := range out.batches {
		gauges := make([]LibratoMetric, 0, len(batch))
		counters := make([]LibratoMetric, 0, len(batch))
		for _, metric := range batch {
			libratoMetric := LibratoMetric{metric.Measured(out.prefix), metric.Value, metric.When.Unix(), out.source}
			switch metric.Value.(type) {
			case uint64:
				counters = append(counters, libratoMetric)
			case float64:
				gauges = append(gauges, libratoMetric)
			}
		}

		payload := PostBody{gauges, counters}
		j, err := json.Marshal(payload)
		if err != nil {
			ctx.FatalError(err, "marshaling json")
		}

		out.retry(j)
	}
   }

func (out *Librato) retry(payload []byte) bool {
	ctx := Slog{"fn": "retry", "outputter": "librato"}
	attempts := 0
	bo := 0 * time.Millisecond
	shouldBackoff := false

	for attempts < LibratoMaxAttempts {
		body := bytes.NewBuffer(payload)
		req, err := http.NewRequest("POST", out.Url, body)
		if err != nil {
			ctx.FatalError(err, "creating new request")
		}

		req.Header.Add("Content-Type", "application/json")
		req.SetBasicAuth(out.User, out.Token)

		resp, err := out.client.Do(req)
		if err != nil {
			if nerr, ok := err.(net.Error); ok && (nerr.Temporary() || nerr.Timeout()) {
				ctx["backoff"] = bo
				ctx["message"] = "Backing off due to transport error"
				fmt.Println(ctx)
				bo = backoff(bo)
			} else if strings.Contains(err.Error(), "timeout awaiting response") {
				return true
			} else {
				ctx.FatalError(err, "doing request")
			}
		}

		if resp.StatusCode >= 300 {
			b, _ := ioutil.ReadAll(resp.Body)
			ctx["body"] = string(b)
			ctx["code"] = resp.StatusCode

			if resp.StatusCode >= 500 {
				shouldBackoff = true
				ctx["message"] = "Backing off due to server error"
				ctx["backoff"] = bo
			} else {
				ctx["message"] = "Client error"
			}

			resp.Body.Close()
			fmt.Println(ctx)
			delete(ctx, "backoff")
			delete(ctx, "message")
			delete(ctx, "body")
			delete(ctx, "code")

			if shouldBackoff {
				bo = backoff(bo)
			} else {
				return false
			}
		} else {
			resp.Body.Close()
			return true
		}

		attempts += 1
	}

	return false
}

// Sleeps `bo` and then returns double, unless 0, in which case
// returns the initial starting sleep time.
func backoff(bo time.Duration) time.Duration {
	if bo > 0 {
		time.Sleep(bo)
		return bo * 2
	} else {
		return LibratoStartingBackoffMillis
	}
}<commit_msg>Refactor `retry`, and set Librato.source<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"
)

type LibratoMetric struct {
	Name   string      `json:"name"`
	Value  interface{} `json:"value"`
	When   int64       `json:"measure_time"`
	Source string      `json:"source,omitempty"`
}

type PostBody struct {
	Gauges   []LibratoMetric `json:"gauges,omitempty"`
	Counters []LibratoMetric `json:"counters,omitempty"`
}

const (
	LibratoBacklog               = 8 // No more than N pending batches in-flight
	LibratoMaxAttempts           = 4 // Max attempts before dropping batch
	LibratoStartingBackoffMillis = 200 * time.Millisecond
)

type Librato struct {
	Timeout      time.Duration
	BatchSize    int
	User         string
	Token        string
	Url          string
	measurements <-chan *Measurement
	batches      chan []*Measurement
	prefix       string
	source       string
	client       *http.Client
}

func NewLibratoOutputter(measurements <-chan *Measurement, config Config) *Librato {
	return &Librato{
		measurements: measurements,
		source:       config.Source,
		batches:      make(chan []*Measurement, LibratoBacklog),
		Timeout:      config.LibratoBatchTimeout,
		BatchSize:    config.LibratoBatchSize,
		User:         config.LibratoUser,
		Token:        config.LibratoToken,
		Url:          config.LibratoUrl,
		client: &http.Client{
			Transport: &http.Transport{
				ResponseHeaderTimeout: config.LibratoNetworkTimeout,
				Dial: func(network, address string) (net.Conn, error) {
					return net.DialTimeout(network, address, config.LibratoNetworkTimeout)
				},
			},
		},
	}
}

func (out *Librato) Start() {
	go out.deliver()
	go out.batch()
}

func (out *Librato) makeBatch() []*Measurement {
	return make([]*Measurement, 0, out.BatchSize)
}

func (out *Librato) batch() {
	var ready bool
	ctx := Slog{"fn": "batch", "outputter": "librato"}
	ticker := time.Tick(out.Timeout)
	batch := out.makeBatch()
	for {
		select {
		case measurement := <-out.measurements:
			batch = append(batch, measurement)
			if len(batch) == cap(batch) {
				ready = true
			}
		case <-ticker:
			if len(batch) > 0 {
				ready = true
			}
		}

		if ready {
			select {
			case out.batches <- batch:
			default:
				ctx.Error(nil, "Batches backlogged, dropping")
			}
			batch = out.makeBatch()
			ready = false
		}
	}
}

func (out *Librato) deliver() {
	ctx := Slog{"fn": "prepare", "outputter": "librato"}
	for batch := range out.batches {
		gauges := make([]LibratoMetric, 0, len(batch))
		counters := make([]LibratoMetric, 0, len(batch))
		for _, metric := range batch {
			libratoMetric := LibratoMetric{metric.Measured(out.prefix), metric.Value, metric.When.Unix(), out.source}
			switch metric.Value.(type) {
			case uint64:
				counters = append(counters, libratoMetric)
			case float64:
				gauges = append(gauges, libratoMetric)
			}
		}

		payload := PostBody{gauges, counters}
		j, err := json.Marshal(payload)
		if err != nil {
			ctx.FatalError(err, "marshaling json")
		}

		out.sendWithBackoff(j)
	}
}

func (out *Librato) sendWithBackoff(payload []byte) bool {
	ctx := Slog{"fn": "retry", "outputter": "librato"}
	attempts := 0
	bo := 0 * time.Millisecond

	for attempts < LibratoMaxAttempts {
		if retry, err := out.send(ctx, payload); retry {
			ctx["backoff"] = bo
			ctx["message"] = err
			fmt.Println(ctx)
			bo = backoff(bo)
		} else if err != nil {
			ctx["error"] = err
			fmt.Println(ctx)
			return false
		} else {
			return true
		}

		resetCtx(ctx)
		attempts++
	}
	return false
}

func (out *Librato) send(ctx Slog, payload []byte) (retry bool, e error) {
	body := bytes.NewBuffer(payload)
	req, err := http.NewRequest("POST", out.Url, body)
	if err != nil {
		ctx.FatalError(err, "creating new request")
	}

	req.Header.Add("Content-Type", "application/json")
	req.SetBasicAuth(out.User, out.Token)

	resp, err := out.client.Do(req)
	if err != nil {
		if nerr, ok := err.(net.Error); ok && (nerr.Temporary() || nerr.Timeout()) {
			retry = true
			e = fmt.Errorf("Backing off due to transport error")
		} else if strings.Contains(err.Error(), "timeout awaiting response") {
			retry = false
			e = nil
		} else {
			ctx.FatalError(err, "doing request")
		}
	} else {
		defer resp.Body.Close()

		if resp.StatusCode >= 300 {
			b, _ := ioutil.ReadAll(resp.Body)
			ctx["body"] = string(b)
			ctx["code"] = resp.StatusCode

			if resp.StatusCode >= 500 {
				retry = true
				e = fmt.Errorf("Backing off due to server error")
			} else {
				e = fmt.Errorf("Client error")
			}
		}
	}

	return
}

// Sleeps `bo` and then returns double, unless 0, in which case
// returns the initial starting sleep time.
func backoff(bo time.Duration) time.Duration {
	if bo > 0 {
		time.Sleep(bo)
		return bo * 2
	} else {
		return LibratoStartingBackoffMillis
	}
}

func resetCtx(ctx Slog) {
	delete(ctx, "backoff")
	delete(ctx, "message")
	delete(ctx, "body")
	delete(ctx, "code")
}
<|endoftext|>


<commit_before>package searcher

import (
	"fmt"
	"hound/config"
	"hound/git"
	"hound/index"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"time"
)

type Searcher struct {
	idx  *index.Index
	lck  sync.RWMutex
	Repo *config.Repo
}

func (s *Searcher) swapIndexes(idx *index.Index) error {
	s.lck.Lock()
	defer s.lck.Unlock()

	oldIdx := s.idx
	s.idx = idx

	return oldIdx.Destroy()
}

func (s *Searcher) Search(pat string, opt *index.SearchOptions) (*index.SearchResponse, error) {
	s.lck.RLock()
	defer s.lck.RUnlock()
	return s.idx.Search(pat, opt)
}

func (s *Searcher) GetExcludedFiles() string {
	path := filepath.Join(s.idx.GetDir(), "excluded_files.json")
	dat, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Couldn't read excluded_files.json %v\n", err)
	}
	return string(dat)
}

func expungeOldIndexes(sha, gitDir string) error {
	// TODO(knorton): This is a bandaid for issue #14, but should suffice
	// since people don't usually name their repos with 40 char hashes. In
	// the longer term, I want to remove this naming scheme to support
	// rebuilds of the current hash.
	pat := regexp.MustCompile("-[0-9a-f]{40}$")

	name := fmt.Sprintf("%s-%s", filepath.Base(gitDir), sha)

	dirs, err := filepath.Glob(fmt.Sprintf("%s-*", gitDir))
	if err != nil {
		return err
	}

	for _, dir := range dirs {
		if !pat.MatchString(dir) {
			continue
		}

		if strings.HasSuffix(dir, name) {
			continue
		}

		if err := os.RemoveAll(dir); err != nil {
			return err
		}
	}

	return nil
}

func buildAndOpenIndex(sha, gitDir string) (*index.Index, error) {
	idxDir := fmt.Sprintf("%s-%s", gitDir, sha)
	if _, err := os.Stat(idxDir); err != nil {
		_, err := index.Build(idxDir, gitDir)
		if err != nil {
			return nil, err
		}
	}

	return index.Open(idxDir)
}

func reportOnMemory() {
	var ms runtime.MemStats

	// Print out interesting heap info.
	runtime.ReadMemStats(&ms)
	fmt.Printf("HeapInUse = %0.2f\n", float64(ms.HeapInuse)/1e6)
	fmt.Printf("HeapIdle  = %0.2f\n", float64(ms.HeapIdle)/1e6)
}

// Creates a new Searcher for the gitDir but avoids any remote git operations.
// This requires that an existing gitDir be available in the data directory. This
// is intended for debugging and testing only. This will not start a watcher to
// monitor the remote repo for changes.
func NewFromExisting(gitDir string, repo *config.Repo) (*Searcher, error) {
	name := filepath.Base(gitDir)

	log.Printf("Search started for %s", name)
	log.Println("  WARNING: index is static and will not update")

	sha, err := git.HeadHash(gitDir)
	if err != nil {
		return nil, err
	}

	idx, err := buildAndOpenIndex(sha, gitDir)
	if err != nil {
		return nil, err
	}

	return &Searcher{
		idx:  idx,
		Repo: repo,
	}, nil
}

// Creates a new Searcher that is available for searches as soon as this returns.
// This will pull or clone the target repo and start watching the repo for changes.
func New(gitDir string, repo *config.Repo) (*Searcher, error) {
	name := filepath.Base(gitDir)

	log.Printf("Searcher started for %s", name)

	sha, err := git.PullOrClone(gitDir, repo.Url)
	if err != nil {
		return nil, err
	}

	if err := expungeOldIndexes(sha, gitDir); err != nil {
		return nil, err
	}

	idx, err := buildAndOpenIndex(sha, gitDir)
	if err != nil {
		return nil, err
	}

	s := &Searcher{
		idx:  idx,
		Repo: repo,
	}

	go func() {
		for {
			time.Sleep(time.Duration(repo.MsBetweenPolls) * time.Millisecond)

			newSha, err := git.PullOrClone(gitDir, repo.Url)
			if err != nil {
				log.Printf("git pull error (%s - %s): %s", name, repo.Url, err)
				continue
			}

			if newSha == sha {
				continue
			}

			log.Printf("Rebuilding %s for %s", name, newSha)
			idx, err := buildAndOpenIndex(newSha, gitDir)
			if err != nil {
				log.Printf("failed index build (%s): %s", name, err)
				os.RemoveAll(fmt.Sprintf("%s-%s", gitDir, newSha))
				continue
			}

			if err := s.swapIndexes(idx); err != nil {
				log.Printf("failed index swap (%s): %s", name, err)
				if err := idx.Destroy(); err != nil {
					log.Printf("failed to destroy index (%s): %s\n", name, err)
				}
				continue
			}

			sha = newSha

			// This is just a good time to GC since we know there will be a
			// whole set of dead posting lists on the heap. Ensuring these
			// go away quickly helps to prevent the heap from expanding
			// uncessarily.
			runtime.GC()

			reportOnMemory()
		}
	}()

	return s, nil
}
<commit_msg>Second attempt to bandaid #14. Sorry about that.<commit_after>package searcher

import (
	"fmt"
	"hound/config"
	"hound/git"
	"hound/index"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"sync"
	"time"
)

type Searcher struct {
	idx  *index.Index
	lck  sync.RWMutex
	Repo *config.Repo
}

func (s *Searcher) swapIndexes(idx *index.Index) error {
	s.lck.Lock()
	defer s.lck.Unlock()

	oldIdx := s.idx
	s.idx = idx

	return oldIdx.Destroy()
}

func (s *Searcher) Search(pat string, opt *index.SearchOptions) (*index.SearchResponse, error) {
	s.lck.RLock()
	defer s.lck.RUnlock()
	return s.idx.Search(pat, opt)
}

func (s *Searcher) GetExcludedFiles() string {
	path := filepath.Join(s.idx.GetDir(), "excluded_files.json")
	dat, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Couldn't read excluded_files.json %v\n", err)
	}
	return string(dat)
}

func expungeOldIndexes(sha, gitDir string) error {
	// TODO(knorton): This is a bandaid for issue #14, but should suffice
	// since people don't usually name their repos with 40 char hashes. In
	// the longer term, I want to remove this naming scheme to support
	// rebuilds of the current hash.
	pat := regexp.MustCompile("-[0-9a-f]{40}$")

	name := fmt.Sprintf("%s-%s", filepath.Base(gitDir), sha)

	dirs, err := filepath.Glob(fmt.Sprintf("%s-*", gitDir))
	if err != nil {
		return err
	}

	for _, dir := range dirs {
		bn := filepath.Base(dir)
		if !pat.MatchString(bn) || len(bn) != len(name) {
			continue
		}

		if bn == name {
			continue
		}

		if err := os.RemoveAll(dir); err != nil {
			return err
		}
	}

	return nil
}

func buildAndOpenIndex(sha, gitDir string) (*index.Index, error) {
	idxDir := fmt.Sprintf("%s-%s", gitDir, sha)
	if _, err := os.Stat(idxDir); err != nil {
		_, err := index.Build(idxDir, gitDir)
		if err != nil {
			return nil, err
		}
	}

	return index.Open(idxDir)
}

func reportOnMemory() {
	var ms runtime.MemStats

	// Print out interesting heap info.
	runtime.ReadMemStats(&ms)
	fmt.Printf("HeapInUse = %0.2f\n", float64(ms.HeapInuse)/1e6)
	fmt.Printf("HeapIdle  = %0.2f\n", float64(ms.HeapIdle)/1e6)
}

// Creates a new Searcher for the gitDir but avoids any remote git operations.
// This requires that an existing gitDir be available in the data directory. This
// is intended for debugging and testing only. This will not start a watcher to
// monitor the remote repo for changes.
func NewFromExisting(gitDir string, repo *config.Repo) (*Searcher, error) {
	name := filepath.Base(gitDir)

	log.Printf("Search started for %s", name)
	log.Println("  WARNING: index is static and will not update")

	sha, err := git.HeadHash(gitDir)
	if err != nil {
		return nil, err
	}

	idx, err := buildAndOpenIndex(sha, gitDir)
	if err != nil {
		return nil, err
	}

	return &Searcher{
		idx:  idx,
		Repo: repo,
	}, nil
}

// Creates a new Searcher that is available for searches as soon as this returns.
// This will pull or clone the target repo and start watching the repo for changes.
func New(gitDir string, repo *config.Repo) (*Searcher, error) {
	name := filepath.Base(gitDir)

	log.Printf("Searcher started for %s", name)

	sha, err := git.PullOrClone(gitDir, repo.Url)
	if err != nil {
		return nil, err
	}

	if err := expungeOldIndexes(sha, gitDir); err != nil {
		return nil, err
	}

	idx, err := buildAndOpenIndex(sha, gitDir)
	if err != nil {
		return nil, err
	}

	s := &Searcher{
		idx:  idx,
		Repo: repo,
	}

	go func() {
		for {
			time.Sleep(time.Duration(repo.MsBetweenPolls) * time.Millisecond)

			newSha, err := git.PullOrClone(gitDir, repo.Url)
			if err != nil {
				log.Printf("git pull error (%s - %s): %s", name, repo.Url, err)
				continue
			}

			if newSha == sha {
				continue
			}

			log.Printf("Rebuilding %s for %s", name, newSha)
			idx, err := buildAndOpenIndex(newSha, gitDir)
			if err != nil {
				log.Printf("failed index build (%s): %s", name, err)
				os.RemoveAll(fmt.Sprintf("%s-%s", gitDir, newSha))
				continue
			}

			if err := s.swapIndexes(idx); err != nil {
				log.Printf("failed index swap (%s): %s", name, err)
				if err := idx.Destroy(); err != nil {
					log.Printf("failed to destroy index (%s): %s\n", name, err)
				}
				continue
			}

			sha = newSha

			// This is just a good time to GC since we know there will be a
			// whole set of dead posting lists on the heap. Ensuring these
			// go away quickly helps to prevent the heap from expanding
			// uncessarily.
			runtime.GC()

			reportOnMemory()
		}
	}()

	return s, nil
}
<|endoftext|>


<commit_before>// Package cookoo is a Chain-of-Command (CoCo) framework for writing
// applications.
//
// Tutorials:
//
// * Building Web Apps with Cookoo: https://github.com/Masterminds/cookoo-web-tutorial
//
// * Building CLI Apps with Cookoo: https://github.com/Masterminds/cookoo-cli-tutorial
//
// A chain of command framework works as follows:
//
// 	* A "route" is constructed as a chain of commands -- a series of
// 	single-purpose tasks that are run in sequence.
// 	* An application is composed of one or more routes.
// 	* Commands in a route communicate using a Context.
// 	* An application Router is used to receive a route name and then
// 	execute the appropriate chain of commands.
//
// To create a new Cookoo application, use cookoo.Cookoo(). This will
// configure and create a new registry, request router, and context.
// From there, use the Registry to build chains of commands, and then
// use the Router to execute chains of commands.
//
// Example:
//
// 	package main
//
// 	import (
// 		//This is the path to Cookoo
// 		"github.com/Masterminds/cookoo/src/cookoo"
// 		"fmt"
// 	)
//
// 	func main() {
// 		// Build a new Cookoo app.
// 		registry, router, context := cookoo.Cookoo()
//
// 		// Fill the registry.
// 		registry.Route("TEST", "A test route").Does(HelloWorld, "hi") //...
//
// 		// Execute the route.
// 		router.HandleRequest("TEST", context, false)
// 	}
//
// 	// An example command
// 	func HelloWorld(cxt cookoo.Context, params *cookoo.Params) (interface{}, Interrupt) {
// 		fmt.Println("Hello World")
// 		return true, nil
// 	}
//
// Unlike other CoCo implementations (like Pronto.js or Fortissimo),
// Cookoo commands are just functions.
//
// Interrupts:
//
// There are four types of interrupts that you may wish to return:
//
// 	1. FatalError: This will stop the route immediately.
// 	2. RecoverableError: This will allow the route to continue moving.
// 	3. Stop: This will stop the current request, but not as an error.
// 	4. Reroute: This will stop executing the current route, and switch to executing another route.
//
// To learn how to write Cookoo applications, you may wish to examine
// the small Skunk application: https://github.com/technosophos/skunk.
package cookoo

// VERSION provides the current version of Cookoo.
const VERSION = "1.3.0"

// Cookoo creates a new Cookoo app.
func Cookoo() (reg *Registry, router *Router, cxt Context) {
	cxt = NewContext()
	reg = NewRegistry()
	router = NewRouter(reg)
	return
}

// Command executes a command and returns a result.
// A Cookoo app has a registry, which has zero or more routes. Each route
// executes a sequence of zero or more commands. A command is of this type.
type Command func(cxt Context, params *Params) (interface{}, Interrupt)

// Interrupt is a generic return for a command.
// Generally, a command should return one of the following in the interrupt slot:
// - A FatalError, which will stop processing.
// - A RecoverableError, which will continue the chain.
// - A Reroute, which will cause a different route to be run.
type Interrupt interface{}

// Reroute is a command can return a Reroute to tell the router to execute a
// different route.
type Reroute struct {
	Route string
}

// RouteTo returns the route to reroute to.
func (rr *Reroute) RouteTo() string {
	return rr.Route
}

// Stop a route, but not as an error condition.
type Stop struct{}

// RecoverableError is an error that should not cause the router to stop processing.
type RecoverableError struct {
	Message string
}

// Error returns the error message.
func (err *RecoverableError) Error() string {
	return err.Message
}

// FatalError is a fatal error, which will stop the router from continuing a route.
type FatalError struct {
	Message string
}

// Error returns the error message.
func (err *FatalError) Error() string {
	return err.Message
}
<commit_msg>Updating docs on main functions in Cookoo.<commit_after>// Package cookoo is a Chain-of-Command (CoCo) framework for writing
// applications.
//
// Tutorials:
//
// * Building Web Apps with Cookoo: https://github.com/Masterminds/cookoo-web-tutorial
//
// * Building CLI Apps with Cookoo: https://github.com/Masterminds/cookoo-cli-tutorial
//
// A chain of command framework works as follows:
//
// 	* A "route" is constructed as a chain of commands -- a series of
// 	single-purpose tasks that are run in sequence.
// 	* An application is composed of one or more routes.
// 	* Commands in a route communicate using a Context.
// 	* An application Router is used to receive a route name and then
// 	execute the appropriate chain of commands.
//
// To create a new Cookoo application, use cookoo.Cookoo(). This will
// configure and create a new registry, request router, and context.
// From there, use the Registry to build chains of commands, and then
// use the Router to execute chains of commands.
//
// Example:
//
// 	package main
//
// 	import (
// 		//This is the path to Cookoo
// 		"github.com/Masterminds/cookoo/src/cookoo"
// 		"fmt"
// 	)
//
// 	func main() {
// 		// Build a new Cookoo app.
// 		registry, router, context := cookoo.Cookoo()
//
// 		// Fill the registry.
// 		registry.Route("TEST", "A test route").Does(HelloWorld, "hi") //...
//
// 		// Execute the route.
// 		router.HandleRequest("TEST", context, false)
// 	}
//
// 	// An example command
// 	func HelloWorld(cxt cookoo.Context, params *cookoo.Params) (interface{}, Interrupt) {
// 		fmt.Println("Hello World")
// 		return true, nil
// 	}
//
// Unlike other CoCo implementations (like Pronto.js or Fortissimo),
// Cookoo commands are just functions.
//
// Interrupts:
//
// There are four types of interrupts that you may wish to return:
//
// 	1. FatalError: This will stop the route immediately.
// 	2. RecoverableError: This will allow the route to continue moving.
// 	3. Stop: This will stop the current request, but not as an error.
// 	4. Reroute: This will stop executing the current route, and switch to executing another route.
//
// To learn how to write Cookoo applications, you may wish to examine
// the small Skunk application: https://github.com/technosophos/skunk.
package cookoo

// VERSION provides the current version of Cookoo.
const VERSION = "1.3.0"

// Cookoo creates a new Cookoo app.
//
// This is the main progenitor of a Cookoo application. Whether a plain
// Cookoo app, or a Web or CLI program, this is the function you will use
// to bootstrap.
//
// The `*Registry` is used to declare new routes, where a "route" may be thought
// of as a task composed of a series of steps (commands).
//
// The `*Router` is responsible for the actual execution of a Cookoo route. The
// main method used to call a route is `Router.HandleRequest()`.
//
// The `Context` is a container for passing information down a chain of commands.
// Apps may insert "global" information to a context at startup and make it
// available to all commands.
func Cookoo() (reg *Registry, router *Router, cxt Context) {
	cxt = NewContext()
	reg = NewRegistry()
	router = NewRouter(reg)
	return
}

// Command executes a command and returns a result.
// A Cookoo app has a registry, which has zero or more routes. Each route
// executes a sequence of zero or more commands. A command is of this type.
type Command func(cxt Context, params *Params) (interface{}, Interrupt)

// Interrupt is a generic return for a command.
// Generally, a command should return one of the following in the interrupt slot:
// - A FatalError, which will stop processing.
// - A RecoverableError, which will continue the chain.
// - A Reroute, which will cause a different route to be run.
type Interrupt interface{}

// Reroute is a command can return a Reroute to tell the router to execute a
// different route.
//
// A `Command` may return a `Reroute` to cause Cookoo to stop executing the
// current route and jump to another.
//
// 	func Forward(c Context, p *Params) (interface{}, Interrupt) {
// 		return nil, &Reroute{"anotherRoute"}
// 	}
type Reroute struct {
	Route string
}

// RouteTo returns the route to reroute to.
func (rr *Reroute) RouteTo() string {
	return rr.Route
}

// Stop a route, but not as an error condition.
//
// When Cookoo encounters a `Stop`, it will not execute any more commands on a
// given route. However, it will not emit an error, either.
type Stop struct{}

// RecoverableError is an error that should not cause the router to stop processing.
//
// When Cookoo encounters a `RecoverableError`, it will log the error as a
// warning, but will then continue to execute the next command in the route.
type RecoverableError struct {
	Message string
}

// Error returns the error message.
func (err *RecoverableError) Error() string {
	return err.Message
}

// FatalError is a fatal error, which will stop the router from continuing a route.
//
// When Cookoo encounters a `FatalError`, it will log the error and immediately
// stop processing the route.
//
// Note that by default Cookoo treats and unhandled `error` as if it were a
// `FatalError`.
type FatalError struct {
	Message string
}

// Error returns the error message.
func (err *FatalError) Error() string {
	return err.Message
}
<|endoftext|>


<commit_before>package myrouter

import (
	"fmt"
	"regexp"
	"strings"
	"testing"
)

func testRegExpFromPathVerification(t *testing.T, path string, url string, expect string, requirements map[string]*regexp.Regexp) {
	var regexp, err = generateRegexpFromPath(path, requirements)
	if err != nil {
		fmt.Println(err)
		t.Fail()
	}

	if regexp.String() != expect {
		fmt.Print(regexp.String())
		fmt.Print("\n")
		t.Fail()
	}

	if !regexp.MatchString(url) {
		t.Fail()
	}
}

func testRegExpFromPathValueVerification(t *testing.T, path string, url string, expect []string, requirements map[string]*regexp.Regexp) {
	var regexp, err = generateRegexpFromPath(path, requirements)
	if err != nil {
		fmt.Println(err)
		t.Fail()
	}

	var result = regexp.FindAllStringSubmatch(url, -1)
	var parameters []string
	for i := 1; i < len(result[0]); i++ {
		parameters = append(parameters, result[0][i])
	}

	if !regexp.MatchString(url) {
		t.Fail()
	}

	if !arrayCompareString(expect, parameters) {
		fmt.Print(strings.Join(parameters, ", "))
		fmt.Print("\n")
		t.Fail()
	}
}

func TestEmptyPath(t *testing.T) {
	var path, err = generatePath("", make(map[string][]string), make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}
	if path != "" {
		t.Fail()
	}
}

func TestPathWithMissingParameter(t *testing.T) {
	var path, err = generatePath("/{id}", make(map[string][]string), make(map[string]*regexp.Regexp))
	if err == nil {
		t.Fail()
	}

	if path != "" {
		t.Fail()
	}
}

func TestPathWithParameter(t *testing.T) {
	var path, err = generatePath("/{id}", map[string][]string{"id": []string{"test"}}, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}

	if path != "/test" {
		t.Fail()
	}
}

func TestPathWithExtraParameter(t *testing.T) {
	var path, err = generatePath("/{id}", map[string][]string{"id": []string{"test"}, "slug": []string{"poltorak-dariusz"}}, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}

	if path != "/test?slug=poltorak-dariusz" {
		t.Fail()
	}
}

func TestPathWithTwoParameters(t *testing.T) {
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"test"}, "slug": []string{"poltorak-dariusz"}}, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}

	if path != "/test/poltorak-dariusz" {
		t.Fail()
	}
}

func TestRegExpFromPathNoArgs(t *testing.T) {
	var path = "/api/user"
	var url = "/api/user"
	var expect = "/api/user"

	testRegExpFromPathVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathSingleArg(t *testing.T) {
	var path = "/api/user/{id}"
	var url = "/api/user/5"
	var expect = "/api/user/([^/]+)"

	testRegExpFromPathVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathSingleCopiedArg(t *testing.T) {
	var path = "/api/user/{id}/user-{id}"
	var url = "/api/user/5/user-5"
	var expect = "/api/user/([^/]+)/user-([^/]+)"

	testRegExpFromPathVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathTwoArgs(t *testing.T) {
	var path = "/api/user/{id}/client-{slug}"
	var url = "/api/user/5/client-poltorak-dariusz"
	var expect = "/api/user/([^/]+)/client-([^/]+)"

	testRegExpFromPathVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathNoValues(t *testing.T) {
	var path = "/api/user/5/client-poltorak-dariusz"
	var url = "/api/user/5/client-poltorak-dariusz"
	var expect = []string{}

	testRegExpFromPathValueVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathOneValue(t *testing.T) {
	var path = "/api/user/{id}"
	var url = "/api/user/5"
	var expect = []string{"5"}

	testRegExpFromPathValueVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathTwoValues(t *testing.T) {
	var path = "/api/user/{id}/client-{slug}"
	var url = "/api/user/5/client-poltorak-dariusz"
	var expect = []string{"5", "poltorak-dariusz"}

	testRegExpFromPathValueVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathWithRequirement(t *testing.T) {
	var path = "/api/user/{id}/client-{slug}"
	var url = "/api/user/5/client-poltorak-dariusz"
	var expect = "/api/user/([^/]+)/client-([a-z\\-]+)"
	var requirements = map[string]*regexp.Regexp{"slug": regexp.MustCompile("[a-z\\-]+")}

	testRegExpFromPathVerification(t, path, url, expect, requirements)
}

func TestRegExpFromPathWithTwoRequirements(t *testing.T) {
	var path = "/api/user/{id}/client-{slug}"
	var url = "/api/user/5/client-poltorak-dariusz"
	var expect = "/api/user/([1-9]+[0-9]*)/client-([a-z\\-]+)"
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[1-9]+[0-9]*"), "slug": regexp.MustCompile("[a-z\\-]+")}

	testRegExpFromPathVerification(t, path, url, expect, requirements)
}

func TestUrl(t *testing.T) {
	var expect = "https://example.com/api/user/5"
	var scheme = "https"
	var host = "example.com"
	var port int
	var path = "/api/user/{id}"
	var parameters = map[string][]string{"id": []string{"5"}}

	var url, err = generateURL(scheme, "", host, port, path, parameters, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}
	if url != expect {
		t.Fail()
	}
}

func TestUrlTwoParameters(t *testing.T) {
	var expect = "https://example.com/api/user/5/dariusz-poltorak"
	var scheme = "https"
	var host = "example.com"
	var port int
	var path = "/api/user/{id}/{slug}"
	var parameters = map[string][]string{"slug": []string{"dariusz-poltorak"}, "id": []string{"5"}}

	var url, err = generateURL(scheme, "", host, port, path, parameters, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}
	if url != expect {
		t.Fail()
	}
}

func TestUrlMissingParameters(t *testing.T) {
	var expect = "https://example.com/api/user/5/dariusz-poltorak"
	var scheme = "https"
	var host = "example.com"
	var port int
	var path = "/api/user/{id}/{slug}"
	var parameters = map[string][]string{"id": []string{"5"}}

	var url, err = generateURL(scheme, "", host, port, path, parameters, make(map[string]*regexp.Regexp))
	if err == nil {
		t.Fail()
	}
	if url == expect {
		t.Fail()
	}
}

func TestUrlTwoParametersWithPort(t *testing.T) {
	var expect = "https://example.com:80/api/user/5/dariusz-poltorak"
	var scheme = "https"
	var host = "example.com"
	var port = 80
	var path = "/api/user/{id}/{slug}"
	var parameters = map[string][]string{"slug": []string{"dariusz-poltorak"}, "id": []string{"5"}}

	var url, err = generateURL(scheme, "", host, port, path, parameters, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}
	if url != expect {
		t.Fail()
	}
}

func TestPathWithTwoParametersWithInvalidRequirement(t *testing.T) {
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[0-9]+")}
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"test"}, "slug": []string{"poltorak-dariusz"}}, requirements)
	if err == nil {
		t.Fail()
	}

	if path != "" {
		t.Fail()
	}
}

func TestPathWithTwoParametersWithTwoInvalidRequirements(t *testing.T) {
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[0-9]+"), "slug": regexp.MustCompile("[A-Z]+")}
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"test"}, "slug": []string{"poltorak-dariusz"}}, requirements)
	if err == nil {
		t.Fail()
	}

	if path != "" {
		t.Fail()
	}
}

func TestPathWithTwoParametersWithValidRequirement(t *testing.T) {
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[0-9]+")}
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"5"}, "slug": []string{"poltorak-dariusz"}}, requirements)
	if err != nil {
		t.Fail()
	}

	if path != "/5/poltorak-dariusz" {
		t.Fail()
	}
}

func TestPathWithTwoParametersWithTwoValidRequirement(t *testing.T) {
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[0-9]+"), "slug": regexp.MustCompile("[a-z\\-]+")}
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"5"}, "slug": []string{"poltorak-dariusz"}}, requirements)
	if err != nil {
		t.Fail()
	}

	if path != "/5/poltorak-dariusz" {
		t.Fail()
	}
}

func TestPathWithTwoParametersWithTwoValidRequirementAndExtra(t *testing.T) {
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[0-9]+"), "slug": regexp.MustCompile("[a-z\\-]+")}
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"5"}, "slug": []string{"poltorak-dariusz"}, "test": []string{"4"}}, requirements)
	if err != nil {
		t.Fail()
	}

	if path != "/5/poltorak-dariusz?test=4" {
		t.Fail()
	}
}
<commit_msg>array parameters<commit_after>package myrouter

import (
	"fmt"
	"regexp"
	"strings"
	"testing"
)

func testRegExpFromPathVerification(t *testing.T, path string, url string, expect string, requirements map[string]*regexp.Regexp) {
	var regexp, err = generateRegexpFromPath(path, requirements)
	if err != nil {
		fmt.Println(err)
		t.Fail()
	}

	if regexp.String() != expect {
		fmt.Print(regexp.String())
		fmt.Print("\n")
		t.Fail()
	}

	if !regexp.MatchString(url) {
		t.Fail()
	}
}

func testRegExpFromPathValueVerification(t *testing.T, path string, url string, expect []string, requirements map[string]*regexp.Regexp) {
	var regexp, err = generateRegexpFromPath(path, requirements)
	if err != nil {
		fmt.Println(err)
		t.Fail()
	}

	var result = regexp.FindAllStringSubmatch(url, -1)
	var parameters []string
	for i := 1; i < len(result[0]); i++ {
		parameters = append(parameters, result[0][i])
	}

	if !regexp.MatchString(url) {
		t.Fail()
	}

	if !arrayCompareString(expect, parameters) {
		fmt.Print(strings.Join(parameters, ", "))
		fmt.Print("\n")
		t.Fail()
	}
}

func TestEmptyPath(t *testing.T) {
	var path, err = generatePath("", make(map[string][]string), make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}
	if path != "" {
		t.Fail()
	}
}

func TestPathWithMissingParameter(t *testing.T) {
	var path, err = generatePath("/{id}", make(map[string][]string), make(map[string]*regexp.Regexp))
	if err == nil {
		t.Fail()
	}

	if path != "" {
		t.Fail()
	}
}

func TestPathWithParameter(t *testing.T) {
	var path, err = generatePath("/{id}", map[string][]string{"id": []string{"test"}}, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}

	if path != "/test" {
		t.Fail()
	}
}

func TestPathWithExtraParameter(t *testing.T) {
	var path, err = generatePath("/{id}", map[string][]string{"id": []string{"test"}, "slug": []string{"poltorak-dariusz"}}, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}

	if path != "/test?slug=poltorak-dariusz" {
		t.Fail()
	}
}

func TestPathWithTwoParameters(t *testing.T) {
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"test"}, "slug": []string{"poltorak-dariusz"}}, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}

	if path != "/test/poltorak-dariusz" {
		t.Fail()
	}
}

func TestRegExpFromPathNoArgs(t *testing.T) {
	var path = "/api/user"
	var url = "/api/user"
	var expect = "/api/user"

	testRegExpFromPathVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathSingleArg(t *testing.T) {
	var path = "/api/user/{id}"
	var url = "/api/user/5"
	var expect = "/api/user/([^/]+)"

	testRegExpFromPathVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathSingleCopiedArg(t *testing.T) {
	var path = "/api/user/{id}/user-{id}"
	var url = "/api/user/5/user-5"
	var expect = "/api/user/([^/]+)/user-([^/]+)"

	testRegExpFromPathVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathTwoArgs(t *testing.T) {
	var path = "/api/user/{id}/client-{slug}"
	var url = "/api/user/5/client-poltorak-dariusz"
	var expect = "/api/user/([^/]+)/client-([^/]+)"

	testRegExpFromPathVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathNoValues(t *testing.T) {
	var path = "/api/user/5/client-poltorak-dariusz"
	var url = "/api/user/5/client-poltorak-dariusz"
	var expect = []string{}

	testRegExpFromPathValueVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathOneValue(t *testing.T) {
	var path = "/api/user/{id}"
	var url = "/api/user/5"
	var expect = []string{"5"}

	testRegExpFromPathValueVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathTwoValues(t *testing.T) {
	var path = "/api/user/{id}/client-{slug}"
	var url = "/api/user/5/client-poltorak-dariusz"
	var expect = []string{"5", "poltorak-dariusz"}

	testRegExpFromPathValueVerification(t, path, url, expect, map[string]*regexp.Regexp{})
}

func TestRegExpFromPathWithRequirement(t *testing.T) {
	var path = "/api/user/{id}/client-{slug}"
	var url = "/api/user/5/client-poltorak-dariusz"
	var expect = "/api/user/([^/]+)/client-([a-z\\-]+)"
	var requirements = map[string]*regexp.Regexp{"slug": regexp.MustCompile("[a-z\\-]+")}

	testRegExpFromPathVerification(t, path, url, expect, requirements)
}

func TestRegExpFromPathWithTwoRequirements(t *testing.T) {
	var path = "/api/user/{id}/client-{slug}"
	var url = "/api/user/5/client-poltorak-dariusz"
	var expect = "/api/user/([1-9]+[0-9]*)/client-([a-z\\-]+)"
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[1-9]+[0-9]*"), "slug": regexp.MustCompile("[a-z\\-]+")}

	testRegExpFromPathVerification(t, path, url, expect, requirements)
}

func TestUrl(t *testing.T) {
	var expect = "https://example.com/api/user/5"
	var scheme = "https"
	var host = "example.com"
	var port int
	var path = "/api/user/{id}"
	var parameters = map[string][]string{"id": []string{"5"}}

	var url, err = generateURL(scheme, "", host, port, path, parameters, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}
	if url != expect {
		t.Fail()
	}
}

func TestUrlTwoParameters(t *testing.T) {
	var expect = "https://example.com/api/user/5/dariusz-poltorak"
	var scheme = "https"
	var host = "example.com"
	var port int
	var path = "/api/user/{id}/{slug}"
	var parameters = map[string][]string{"slug": []string{"dariusz-poltorak"}, "id": []string{"5"}}

	var url, err = generateURL(scheme, "", host, port, path, parameters, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}
	if url != expect {
		t.Fail()
	}
}

func TestUrlMissingParameters(t *testing.T) {
	var expect = "https://example.com/api/user/5/dariusz-poltorak"
	var scheme = "https"
	var host = "example.com"
	var port int
	var path = "/api/user/{id}/{slug}"
	var parameters = map[string][]string{"id": []string{"5"}}

	var url, err = generateURL(scheme, "", host, port, path, parameters, make(map[string]*regexp.Regexp))
	if err == nil {
		t.Fail()
	}
	if url == expect {
		t.Fail()
	}
}

func TestUrlTwoParametersWithPort(t *testing.T) {
	var expect = "https://example.com:80/api/user/5/dariusz-poltorak"
	var scheme = "https"
	var host = "example.com"
	var port = 80
	var path = "/api/user/{id}/{slug}"
	var parameters = map[string][]string{"slug": []string{"dariusz-poltorak"}, "id": []string{"5"}}

	var url, err = generateURL(scheme, "", host, port, path, parameters, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}
	if url != expect {
		t.Fail()
	}
}

func TestPathWithTwoParametersWithInvalidRequirement(t *testing.T) {
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[0-9]+")}
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"test"}, "slug": []string{"poltorak-dariusz"}}, requirements)
	if err == nil {
		t.Fail()
	}

	if path != "" {
		t.Fail()
	}
}

func TestPathWithTwoParametersWithTwoInvalidRequirements(t *testing.T) {
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[0-9]+"), "slug": regexp.MustCompile("[A-Z]+")}
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"test"}, "slug": []string{"poltorak-dariusz"}}, requirements)
	if err == nil {
		t.Fail()
	}

	if path != "" {
		t.Fail()
	}
}

func TestPathWithTwoParametersWithValidRequirement(t *testing.T) {
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[0-9]+")}
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"5"}, "slug": []string{"poltorak-dariusz"}}, requirements)
	if err != nil {
		t.Fail()
	}

	if path != "/5/poltorak-dariusz" {
		t.Fail()
	}
}

func TestPathWithTwoParametersWithTwoValidRequirement(t *testing.T) {
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[0-9]+"), "slug": regexp.MustCompile("[a-z\\-]+")}
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"5"}, "slug": []string{"poltorak-dariusz"}}, requirements)
	if err != nil {
		t.Fail()
	}

	if path != "/5/poltorak-dariusz" {
		t.Fail()
	}
}

func TestPathWithTwoParametersWithTwoValidRequirementAndExtra(t *testing.T) {
	var requirements = map[string]*regexp.Regexp{"id": regexp.MustCompile("[0-9]+"), "slug": regexp.MustCompile("[a-z\\-]+")}
	var path, err = generatePath("/{id}/{slug}", map[string][]string{"id": []string{"5"}, "slug": []string{"poltorak-dariusz"}, "test": []string{"4"}}, requirements)
	if err != nil {
		t.Fail()
	}

	if path != "/5/poltorak-dariusz?test=4" {
		t.Fail()
	}
}

func TestArrayParameters(t *testing.T) {
	var parameters = map[string][]string{"ids": []string{"1", "2"}}
	var path, err = generatePath("/test", parameters, make(map[string]*regexp.Regexp))
	if err != nil {
		t.Fail()
	}

	var expected = "/test?ids=1&ids=2"
	if path != expected {
		fmt.Println(strings.Join([]string{"Expected", expected}, " "))
		fmt.Println(strings.Join([]string{"Actual", path}, " "))
		t.Fail()
	}
}
<|endoftext|>


<commit_before>// timeutil provides a set of time utilities including comparisons,
// conversion to "DT8" int32 and "DT14" int64 formats and other
// capabilities.
package timeutil

import (
	"time"
)

func DayStart(dt time.Time) time.Time {
	return time.Date(
		dt.Year(), dt.Month(), dt.Day(),
		0, 0, 0, 0,
		dt.Location())
}

// WeekStart takes a time.Time object and a week start day
// in the time.Weekday format.
func WeekStart(dt time.Time, dow time.Weekday) (time.Time, error) {
	return TimeDeltaDowInt(dt.UTC(), int(dow), -1, true, true)
}

func MonthStart(dt time.Time) time.Time {
	return time.Date(
		dt.Year(), dt.Month(), 1,
		0, 0, 0, 0,
		dt.Location())
}

// YearStart returns a a time.Time for the beginning of the year
// in UTC time.
func YearStart(dt time.Time) time.Time {
	return time.Date(dt.UTC().Year(), time.January, 1, 0, 0, 0, 0, time.UTC)
}

func NextYearStart(dt time.Time) time.Time {
	return time.Date(dt.UTC().Year()+1, time.January, 1, 0, 0, 0, 0, time.UTC)
}

func IsYearStart(t time.Time) bool {
	t = t.UTC()
	if t.Nanosecond() == 0 &&
		t.Second() == 0 &&
		t.Minute() == 0 &&
		t.Hour() == 0 &&
		t.Day() == 1 &&
		t.Month() == time.January {
		return true
	}
	return false
}

// QuarterEnd returns a time.Time for the end of the
// quarter by second in UTC time.
func QuarterEnd(dt time.Time) time.Time {
	qs := QuarterStart(dt.UTC())
	qn := TimeDt6AddNMonths(qs, 3)
	return time.Date(qn.Year(), qn.Month(), 0, 23, 59, 59, 0, time.UTC)
}

// YearEnd returns a a time.Time for the end of the year in UTC time.
func YearEnd(dt time.Time) time.Time {
	return time.Date(dt.UTC().Year(), time.December, 31, 23, 59, 59, 999999999, time.UTC)
}
<commit_msg>enhance: breaking: timeutil: make YearStart() and NextYearStart() more generic<commit_after>// timeutil provides a set of time utilities including comparisons,
// conversion to "DT8" int32 and "DT14" int64 formats and other
// capabilities.
package timeutil

import (
	"time"
)

func DayStart(dt time.Time) time.Time {
	return time.Date(
		dt.Year(), dt.Month(), dt.Day(),
		0, 0, 0, 0, dt.Location())
}

// WeekStart takes a time.Time object and a week start day
// in the time.Weekday format.
func WeekStart(dt time.Time, dow time.Weekday) (time.Time, error) {
	return TimeDeltaDowInt(dt.UTC(), int(dow), -1, true, true)
}

func MonthStart(dt time.Time) time.Time {
	return time.Date(
		dt.Year(), dt.Month(), 1,
		0, 0, 0, 0, dt.Location())
}

// YearStart returns a a time.Time for the beginning of the year
// in UTC time.
func YearStart(dt time.Time) time.Time {
	return time.Date(
		dt.Year(), time.January, 1,
		0, 0, 0, 0, dt.Location())
}

func NextYearStart(dt time.Time) time.Time {
	return time.Date(
		dt.Year()+1, time.January, 1,
		0, 0, 0, 0, dt.Location())
}

func IsYearStart(t time.Time) bool {
	t = t.UTC()
	if t.Nanosecond() == 0 &&
		t.Second() == 0 &&
		t.Minute() == 0 &&
		t.Hour() == 0 &&
		t.Day() == 1 &&
		t.Month() == time.January {
		return true
	}
	return false
}

// QuarterEnd returns a time.Time for the end of the
// quarter by second in UTC time.
func QuarterEnd(dt time.Time) time.Time {
	qs := QuarterStart(dt.UTC())
	qn := TimeDt6AddNMonths(qs, 3)
	return time.Date(qn.Year(), qn.Month(), 0, 23, 59, 59, 0, time.UTC)
}

// YearEnd returns a a time.Time for the end of the year in UTC time.
func YearEnd(dt time.Time) time.Time {
	return time.Date(dt.UTC().Year(), time.December, 31, 23, 59, 59, 999999999, time.UTC)
}
<|endoftext|>


<commit_before>// Copyright 2015 Comcast Cable Communications Management, LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// End Copyright

package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
	"time"

	"github.com/Comcast/rulio/core"
	"github.com/Comcast/rulio/cron"
	"github.com/Comcast/rulio/service"
	"github.com/Comcast/rulio/storage/dynamodb"
	"github.com/Comcast/rulio/sys"
)

var genericFlags = flag.NewFlagSet("generic", flag.ExitOnError)
var cpuprofile = genericFlags.String("cpuprofile", "", "write cpu profile to this file")
var memProfileRate = genericFlags.Int("memprofilerate", 512*1024, "runtime.MemProfileRate")
var blockProfileRate = genericFlags.Int("blockprofilerate", 0, "runtime.SetBlockProfileRate")
var httpProfilePort = genericFlags.String("httpprofileport", "localhost:6060", "Run an HTTP server that serves profile data; 'none' to turn off")
var verbosity = genericFlags.String("verbosity", "EVERYTHING", "Logging verbosity.")

var engineFlags = flag.NewFlagSet("engine", flag.PanicOnError)
var linearState = engineFlags.Bool("linear-state", false, "linear (or indexed) state?")
var maxPending = engineFlags.Int("max-pending", 0, "max pending requests; 0 means no max")
var maxLocations = engineFlags.Int("max-locations", 1000, "Max locations")
var maxFacts = engineFlags.Int("max-facts", 1000, "Max facts per location")
var accVerbosity = engineFlags.String("acc-verbosity", "EVERYTHING", "Log accumulator verbosity.")
var storageType = engineFlags.String("storage", "mem", "storage type")
var storageConfig = engineFlags.String("storage-config", "", "storage config")

var enginePort = engineFlags.String("engine-port", ":8001", "port engine will serve")
var locationTTL = engineFlags.String("ttl", "forever", "Location TTL, a duration, 'forever', or 'never'")
var cronURL = engineFlags.String("cron-url", "", "Optional URL for external cron service")
var rulesURL = engineFlags.String("rules-url", "http://localhost:8001/", "Optional URL for external cron service to reach rules engine")
var checkState = engineFlags.Bool("check-state", false, "Whether to check for state consistency")

// Direct storage examination an manipulation
//
// This CLI exposes the complete storage API, so you can do surgery on
// locations if you really want to.
var storeFlags = flag.NewFlagSet("storage", flag.PanicOnError)
var storeType = storeFlags.String("storage", "dynamodb", "storage type")
var storeConfig = storeFlags.String("storage-config", "us-west-2:rulestest", "storage type")
var storeClose = storeFlags.Bool("close", false, "close storage")
var storeHealth = storeFlags.Bool("health", false, "check storage health")

// The following args will apply to all locations provided on the command line.
//
// Example
//
//    ./rulesys -httpprofileport none storage \
//         -storage dynamodb -storage-config us-west-1:rulesstress \
//         -get here there somewhere_else
//
var storeGet = storeFlags.Bool("get", true, "get location")
var storeRem = storeFlags.String("rem", "", "remove fact/rule with this id")
var storeDel = storeFlags.Bool("del", false, "remove location")
var storeClear = storeFlags.Bool("clear", false, "clear location")
var storeStats = storeFlags.Bool("stats", false, "get stats")
var storeAdd = storeFlags.String("add", "", "add these facts: {id1:fact1,id2:fact2,...}")

func generic(args []string) []string {
	genericFlags.Parse(args)

	{
		runtime.MemProfileRate = *memProfileRate

		if *cpuprofile != "" {
			f, err := os.Create(*cpuprofile)
			if err != nil {
				panic(err)
			}
			// service.ProfFilename = *cpuprofile
			pprof.StartCPUProfile(f)
			// defer pprof.StopCPUProfile()
		}

		if 0 < *blockProfileRate {
			runtime.SetBlockProfileRate(*blockProfileRate)
		}

	}

	{
		if *httpProfilePort != "" && *httpProfilePort != "none" {
			go func() {
				if err := http.ListenAndServe(*httpProfilePort, nil); err != nil {
					panic(err)
				}
			}()
		}
	}

	return genericFlags.Args()
}

func engine(args []string, wg *sync.WaitGroup) []string {

	engineFlags.Parse(args)

	ctx := core.NewContext("main")

	dynamodb.CheckLastUpdated = *checkState

	conf := sys.ExampleConfig()
	if *linearState {
		conf.UnindexedState = true
	}
	conf.Storage = *storageType
	conf.StorageConfig = *storageConfig

	cont := sys.ExampleSystemControl()
	cont.MaxLocations = *maxLocations

	ttl, err := sys.ParseLocationTTL(*locationTTL)
	if err != nil {
		panic(err)
	}
	cont.LocationTTL = ttl

	// ToDo: Expose this internal cron limit
	var cronner cron.Cronner
	if *cronURL == "" {
		// Here we're using a single internal cron for all
		// served locations.  That's not (according to the
		// nice comments in cron.go) ideal.  However, it's (a)
		// easy and (b) not used in production server-side
		// deployments, which will use an external cron, which
		// provides persistency.
		cr, _ := cron.NewCron(nil, time.Second, "intcron", 1000000)
		go cr.Start(ctx)
		cronner = &cron.InternalCron{Cron: cr}
	} else {
		cronner = &cron.CroltSimple{
			CroltURL: *cronURL,
			RulesURL: *rulesURL,
		}
	}

	verb, err := core.ParseVerbosity(*verbosity)
	if err != nil {
		panic(err)
	}
	locCtl := &core.Control{
		MaxFacts:  1000,
		Verbosity: verb,
	}
	cont.DefaultLocControl = locCtl

	sys, err := sys.NewSystem(ctx, *conf, *cont, cronner)
	if err != nil {
		panic(err)
	}

	locCont := &core.Control{
		MaxFacts: *maxFacts,
	}
	ctl := sys.Control()
	ctl.DefaultLocControl = locCont
	ctx.SetLogValue("app.id", "rulesys")
	core.UseCores(ctx, false)

	ctx.Verbosity = verb
	ctx.LogAccumulatorLevel = verb

	engine := &service.Service{sys, nil, nil}
	engine.Start(ctx, *rulesURL)

	serv, err := service.NewHTTPService(ctx, engine)
	if err != nil {
		panic(err)
	}
	serv.SetMaxPending(int32(*maxPending))
	wg.Add(1)
	go func() {
		err = serv.Start(ctx, *enginePort)
		if err != nil {
			panic(err)
		}
	}()

	return engineFlags.Args()
}

func storage(args []string, wg *sync.WaitGroup) []string {

	storeFlags.Parse(args)

	ctx := core.NewContext("main")
	store, err := sys.GetStorage(ctx, *storeType, *storeConfig)
	if err != nil {
		panic(err)
	}

	if *storeHealth {
		err := store.Health(ctx)
		if err != nil {
			panic(err)
		}
	}

	if *storeClose {
		err := store.Close(ctx)
		if err != nil {
			panic(err)
		}
	}

	locations := storeFlags.Args()
	if len(locations) == 0 {
		panic(errors.New("give at least one location on command line"))
	}

	for _, name := range locations {
		loc, err := core.NewLocation(ctx, name, nil, nil)
		if err != nil {
			panic(err)
		}
		ctx.SetLoc(loc)
		if *storeGet {
			pairs, err := store.Load(ctx, name)
			if err != nil {
				panic(err)
			}
			fmt.Print("{")
			for i, pair := range pairs {
				if 0 < i {
					fmt.Print(",")
				}
				fmt.Printf(`"%s":%s`, pair.K, pair.V)
			}
			fmt.Println("}")
		}
		if *storeRem != "" {
			if _, err = store.Remove(ctx, name, []byte(*storeRem)); err != nil {
				panic(err)
			}
		}
		if *storeDel {
			if err := store.Delete(ctx, name); err != nil {
				panic(err)
			}
		}
		if *storeClear {
			if _, err := store.Clear(ctx, name); err != nil {
				panic(err)
			}
		}
		if *storeStats {
			stats, err := store.GetStats(ctx, name)
			if err != nil {
				panic(err)
			}
			js, err := json.Marshal(&stats)
			if err != nil {
				panic(err)
			}
			fmt.Println(string(js))
		}
		if *storeAdd != "" {
			var facts map[string]map[string]interface{}
			if err := json.Unmarshal([]byte(*storeAdd), &facts); err != nil {
				panic(err)
			}
			for id, fact := range facts {
				js, err := json.Marshal(fact)
				if err != nil {
					panic(err)
				}
				pair := &core.Pair{[]byte(id), js}
				if err := store.Add(ctx, name, pair); err != nil {
					panic(err)
				}
			}
		}
	}

	return []string{}
}

func usage() {
	fmt.Println("generic flags:")
	genericFlags.Usage()
	fmt.Println("engine subcommand:")
	engineFlags.Usage()
	fmt.Println("storage subcommand: List locations as final args")
	storeFlags.Usage()
}

func main() {

	wg := sync.WaitGroup{}

	args := os.Args[1:]
	args = generic(args)
	if len(args) == 0 {
		usage()
		os.Exit(1)
	}
	switch args[0] {
	case "engine":
		args = engine(args[1:], &wg)
	case "storage":
		args = storage(args[1:], &wg)
	}
	wg.Wait()

	if len(args) != 0 {
		// left-over args are bad.  Freak out (even though we
		// might have started or done something previously.
		fmt.Printf("bad command line: extra args %#v\n", args)
		os.Exit(1)
	}
}
<commit_msg>Rulesys command should be reasonable when given no args<commit_after>// Copyright 2015 Comcast Cable Communications Management, LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// End Copyright

package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
	"time"

	"github.com/Comcast/rulio/core"
	"github.com/Comcast/rulio/cron"
	"github.com/Comcast/rulio/service"
	"github.com/Comcast/rulio/storage/dynamodb"
	"github.com/Comcast/rulio/sys"
)

var genericFlags = flag.NewFlagSet("generic", flag.ExitOnError)
var cpuprofile = genericFlags.String("cpuprofile", "", "write cpu profile to this file")
var memProfileRate = genericFlags.Int("memprofilerate", 512*1024, "runtime.MemProfileRate")
var blockProfileRate = genericFlags.Int("blockprofilerate", 0, "runtime.SetBlockProfileRate")
var httpProfilePort = genericFlags.String("httpprofileport", "localhost:6060", "Run an HTTP server that serves profile data; 'none' to turn off")
var verbosity = genericFlags.String("verbosity", "EVERYTHING", "Logging verbosity.")

var engineFlags = flag.NewFlagSet("engine", flag.ExitOnError)
var linearState = engineFlags.Bool("linear-state", false, "linear (or indexed) state?")
var maxPending = engineFlags.Int("max-pending", 0, "max pending requests; 0 means no max")
var maxLocations = engineFlags.Int("max-locations", 1000, "Max locations")
var maxFacts = engineFlags.Int("max-facts", 1000, "Max facts per location")
var accVerbosity = engineFlags.String("acc-verbosity", "EVERYTHING", "Log accumulator verbosity.")
var storageType = engineFlags.String("storage", "mem", "storage type")
var storageConfig = engineFlags.String("storage-config", "", "storage config")

var enginePort = engineFlags.String("engine-port", ":8001", "port engine will serve")
var locationTTL = engineFlags.String("ttl", "forever", "Location TTL, a duration, 'forever', or 'never'")
var cronURL = engineFlags.String("cron-url", "", "Optional URL for external cron service")
var rulesURL = engineFlags.String("rules-url", "http://localhost:8001/", "Optional URL for external cron service to reach rules engine")
var checkState = engineFlags.Bool("check-state", false, "Whether to check for state consistency")

// Direct storage examination an manipulation
//
// This CLI exposes the complete storage API, so you can do surgery on
// locations if you really want to.
var storeFlags = flag.NewFlagSet("storage", flag.ExitOnError)
var storeType = storeFlags.String("storage", "dynamodb", "storage type")
var storeConfig = storeFlags.String("storage-config", "us-west-2:rulestest", "storage type")
var storeClose = storeFlags.Bool("close", false, "close storage")
var storeHealth = storeFlags.Bool("health", false, "check storage health")

// The following args will apply to all locations provided on the command line.
//
// Example
//
//    ./rulesys -httpprofileport none storage \
//         -storage dynamodb -storage-config us-west-1:rulesstress \
//         -get here there somewhere_else
//
var storeGet = storeFlags.Bool("get", true, "get location")
var storeRem = storeFlags.String("rem", "", "remove fact/rule with this id")
var storeDel = storeFlags.Bool("del", false, "remove location")
var storeClear = storeFlags.Bool("clear", false, "clear location")
var storeStats = storeFlags.Bool("stats", false, "get stats")
var storeAdd = storeFlags.String("add", "", "add these facts: {id1:fact1,id2:fact2,...}")

func generic(args []string) []string {
	genericFlags.Parse(args)

	{
		runtime.MemProfileRate = *memProfileRate

		if *cpuprofile != "" {
			f, err := os.Create(*cpuprofile)
			if err != nil {
				panic(err)
			}
			// service.ProfFilename = *cpuprofile
			pprof.StartCPUProfile(f)
			// defer pprof.StopCPUProfile()
		}

		if 0 < *blockProfileRate {
			runtime.SetBlockProfileRate(*blockProfileRate)
		}

	}

	{
		if *httpProfilePort != "" && *httpProfilePort != "none" {
			go func() {
				if err := http.ListenAndServe(*httpProfilePort, nil); err != nil {
					panic(err)
				}
			}()
		}
	}

	return genericFlags.Args()
}

func engine(args []string, wg *sync.WaitGroup) []string {

	engineFlags.Parse(args)

	ctx := core.NewContext("main")

	dynamodb.CheckLastUpdated = *checkState

	conf := sys.ExampleConfig()
	if *linearState {
		conf.UnindexedState = true
	}
	conf.Storage = *storageType
	conf.StorageConfig = *storageConfig

	cont := sys.ExampleSystemControl()
	cont.MaxLocations = *maxLocations

	ttl, err := sys.ParseLocationTTL(*locationTTL)
	if err != nil {
		panic(err)
	}
	cont.LocationTTL = ttl

	// ToDo: Expose this internal cron limit
	var cronner cron.Cronner
	if *cronURL == "" {
		// Here we're using a single internal cron for all
		// served locations.  That's not (according to the
		// nice comments in cron.go) ideal.  However, it's (a)
		// easy and (b) not used in production server-side
		// deployments, which will use an external cron, which
		// provides persistency.
		cr, _ := cron.NewCron(nil, time.Second, "intcron", 1000000)
		go cr.Start(ctx)
		cronner = &cron.InternalCron{Cron: cr}
	} else {
		cronner = &cron.CroltSimple{
			CroltURL: *cronURL,
			RulesURL: *rulesURL,
		}
	}

	verb, err := core.ParseVerbosity(*verbosity)
	if err != nil {
		panic(err)
	}
	locCtl := &core.Control{
		MaxFacts:  1000,
		Verbosity: verb,
	}
	cont.DefaultLocControl = locCtl

	sys, err := sys.NewSystem(ctx, *conf, *cont, cronner)
	if err != nil {
		panic(err)
	}

	locCont := &core.Control{
		MaxFacts: *maxFacts,
	}
	ctl := sys.Control()
	ctl.DefaultLocControl = locCont
	ctx.SetLogValue("app.id", "rulesys")
	core.UseCores(ctx, false)

	ctx.Verbosity = verb
	ctx.LogAccumulatorLevel = verb

	engine := &service.Service{sys, nil, nil}
	engine.Start(ctx, *rulesURL)

	serv, err := service.NewHTTPService(ctx, engine)
	if err != nil {
		panic(err)
	}
	serv.SetMaxPending(int32(*maxPending))
	wg.Add(1)
	go func() {
		err = serv.Start(ctx, *enginePort)
		if err != nil {
			panic(err)
		}
	}()

	return engineFlags.Args()
}

func storage(args []string, wg *sync.WaitGroup) []string {

	storeFlags.Parse(args)

	ctx := core.NewContext("main")
	store, err := sys.GetStorage(ctx, *storeType, *storeConfig)
	if err != nil {
		panic(err)
	}

	if *storeHealth {
		err := store.Health(ctx)
		if err != nil {
			panic(err)
		}
	}

	if *storeClose {
		err := store.Close(ctx)
		if err != nil {
			panic(err)
		}
	}

	locations := storeFlags.Args()
	if len(locations) == 0 {
		panic(errors.New("give at least one location on command line"))
	}

	for _, name := range locations {
		loc, err := core.NewLocation(ctx, name, nil, nil)
		if err != nil {
			panic(err)
		}
		ctx.SetLoc(loc)
		if *storeGet {
			pairs, err := store.Load(ctx, name)
			if err != nil {
				panic(err)
			}
			fmt.Print("{")
			for i, pair := range pairs {
				if 0 < i {
					fmt.Print(",")
				}
				fmt.Printf(`"%s":%s`, pair.K, pair.V)
			}
			fmt.Println("}")
		}
		if *storeRem != "" {
			if _, err = store.Remove(ctx, name, []byte(*storeRem)); err != nil {
				panic(err)
			}
		}
		if *storeDel {
			if err := store.Delete(ctx, name); err != nil {
				panic(err)
			}
		}
		if *storeClear {
			if _, err := store.Clear(ctx, name); err != nil {
				panic(err)
			}
		}
		if *storeStats {
			stats, err := store.GetStats(ctx, name)
			if err != nil {
				panic(err)
			}
			js, err := json.Marshal(&stats)
			if err != nil {
				panic(err)
			}
			fmt.Println(string(js))
		}
		if *storeAdd != "" {
			var facts map[string]map[string]interface{}
			if err := json.Unmarshal([]byte(*storeAdd), &facts); err != nil {
				panic(err)
			}
			for id, fact := range facts {
				js, err := json.Marshal(fact)
				if err != nil {
					panic(err)
				}
				pair := &core.Pair{[]byte(id), js}
				if err := store.Add(ctx, name, pair); err != nil {
					panic(err)
				}
			}
		}
	}

	return []string{}
}

func usage() {
	fmt.Fprintf(os.Stderr, "\ngeneric flags:\n\n")
	genericFlags.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\nengine subcommand:\n\n")
	engineFlags.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\nstorage subcommand: List locations as final args\n\n")
	storeFlags.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n")
}

func main() {

	wg := sync.WaitGroup{}

	args := os.Args[1:]
	args = generic(args)
	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "Error: Need a subcommand (engine|storage)\n\n")
		usage()
		os.Exit(1)
	}

	switch args[0] {
	case "engine":
		args = engine(args[1:], &wg)
	case "storage":
		args = storage(args[1:], &wg)
	case "help":
		usage()
		os.Exit(0)
	default:
		fmt.Fprintf(os.Stderr, "bad subcommand '%s'\n", args[0])
		os.Exit(1)

	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package stash

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
)

// PropertyValue holds a string value alongside an
// associated PrintKey
type PropertyValue struct {
	Value    string
	PrintKey int
}

// UnmarshalJSON implements custom deserialization for this type
//
// Typically, the GGG api will return [string, int] which is very unhelpful,
// so we take care of that right here
func (v *PropertyValue) UnmarshalJSON(b []byte) error {
	var raw []interface{}

	err := json.Unmarshal(b, &raw)
	if err != nil {
		return fmt.Errorf("invalid property pair, unparseable, err=%s", err)
	}

	if len(raw) != 2 {
		return fmt.Errorf("invalid property pair, (len()==%d)!=2", len(raw))
	}

	var ok bool
	v.Value, ok = raw[0].(string)
	if !ok {
		return fmt.Errorf("invalid property pair, first element not string")
	}

	// We have to use the widest possible type here and narrow...
	var broad float64
	broad, ok = raw[1].(float64)
	if !ok {
		return fmt.Errorf("invalid property pair, second element not float64")
	}
	v.PrintKey = int(broad)

	return nil
}

// Item represents a single item found from the stash api
type Item struct {
	Verified     bool     `json:"verified"`
	Ilvl         int      `json:"ilvl"`
	Icon         string   `json:"icon"`
	League       string   `json:"league"`
	ID           string   `json:"id"`
	Name         string   `json:"name"`
	TypeLine     string   `json:"typeLine"`
	Identified   bool     `json:"identified"`
	Corrupted    bool     `json:"corrupted"`
	ImplicitMods []string `json:"implicitMods,omitempty"`
	ExplicitMods []string `json:"explicitMods,omitempty"`
	FlavourText  []string `json:"flavourText,omitempty"`
	Note         string   `json:"note,omitempty"`
	Properties   []struct {
		Name        string          `json:"name"`
		Values      []PropertyValue `json:"values"`
		DisplayMode int             `json:"displayMode"`
	} `json:"properties,omitempty"`
	UtilityMods []string `json:"utilityMods,omitempty"`
	DescrText   string   `json:"descrText,omitempty"`
}

// Response represents expected structure of a stash api call
type Response struct {
	NextChangeID string `json:"next_change_id"`
	Stashes      []struct {
		AccountName       string `json:"accountName"`
		LastCharacterName string `json:"lastCharacterName"`
		ID                string `json:"id"`
		Stash             string `json:"stash"`
		StashType         string `json:"stashType"`
		Items             []Item `json:"items"`
		Public            bool   `json:"public"`
	} `json:"stashes"`
}

// StashAPIBase is the URL the stash api is located at
const StashAPIBase string = "http://www.pathofexile.com/api/public-stash-tabs"

// TestResponseLoc is where testing data is kept
const TestResponseLoc string = "StashResponse.json"

// GetStored gets the stored testing data and returns it
func GetStored() (*Response, error) {
	f, err := os.Open(TestResponseLoc)
	if err != nil {
		return nil, fmt.Errorf("failed to open TestResponseLoc, err=%s", err)
	}
	defer f.Close()

	decoder := json.NewDecoder(f)
	var response Response
	err = decoder.Decode(&response)
	if err != nil {
		return nil, fmt.Errorf("failed to decode TestResponseLoc, err=%s", err)
	}

	return &response, nil
}

// FetchAndSetStore grabs the latest stash tab api update
// and stores it in TestResponseLoc
func FetchAndSetStore() error {
	resp, err := http.Get(StashAPIBase)
	if err != nil {
		return fmt.Errorf("failed to call stash api, err=%s", err)
	}
	defer resp.Body.Close()

	decoder := json.NewDecoder(resp.Body)
	var response Response
	err = decoder.Decode(&response)
	if err != nil {
		return fmt.Errorf("failed to decode stash tab response, err=%s", err)
	}

	serial, err := json.Marshal(response)
	if err != nil {
		return fmt.Errorf("failed to marshal stash tab response, err=%s", err)
	}

	if err = ioutil.WriteFile(TestResponseLoc, serial, 0777); err != nil {
		return fmt.Errorf("failed to save stashResult.json, err=%s", err)
	}

	return nil
}
<commit_msg>stash marshal to value matching unmarshal<commit_after>package stash

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
)

// PropertyValue holds a string value alongside an
// associated PrintKey
type PropertyValue struct {
	Value    string
	PrintKey int
}

// MarshalJSON implements custom serialization for this type
func (v *PropertyValue) MarshalJSON() ([]byte, error) {
	raw := make([]interface{}, 2)
	raw[0] = v.Value
	raw[1] = v.PrintKey

	return json.Marshal(raw)
}

// UnmarshalJSON implements custom deserialization for this type
//
// Typically, the GGG api will return [string, int] which is very unhelpful,
// so we take care of that right here
func (v *PropertyValue) UnmarshalJSON(b []byte) error {
	var raw []interface{}

	err := json.Unmarshal(b, &raw)
	if err != nil {
		return fmt.Errorf("invalid property pair, unparseable, err=%s", err)
	}

	if len(raw) != 2 {
		return fmt.Errorf("invalid property pair, (len()==%d)!=2", len(raw))
	}

	var ok bool
	v.Value, ok = raw[0].(string)
	if !ok {
		return fmt.Errorf("invalid property pair, first element not string")
	}

	// We have to use the widest possible type here and narrow...
	var broad float64
	broad, ok = raw[1].(float64)
	if !ok {
		return fmt.Errorf("invalid property pair, second element not float64")
	}
	v.PrintKey = int(broad)

	return nil
}

// Item represents a single item found from the stash api
type Item struct {
	Verified     bool     `json:"verified"`
	Ilvl         int      `json:"ilvl"`
	Icon         string   `json:"icon"`
	League       string   `json:"league"`
	ID           string   `json:"id"`
	Name         string   `json:"name"`
	TypeLine     string   `json:"typeLine"`
	Identified   bool     `json:"identified"`
	Corrupted    bool     `json:"corrupted"`
	ImplicitMods []string `json:"implicitMods,omitempty"`
	ExplicitMods []string `json:"explicitMods,omitempty"`
	FlavourText  []string `json:"flavourText,omitempty"`
	Note         string   `json:"note,omitempty"`
	Properties   []struct {
		Name        string          `json:"name"`
		Values      []PropertyValue `json:"values"`
		DisplayMode int             `json:"displayMode"`
	} `json:"properties,omitempty"`
	UtilityMods []string `json:"utilityMods,omitempty"`
	DescrText   string   `json:"descrText,omitempty"`
}

// Response represents expected structure of a stash api call
type Response struct {
	NextChangeID string `json:"next_change_id"`
	Stashes      []struct {
		AccountName       string `json:"accountName"`
		LastCharacterName string `json:"lastCharacterName"`
		ID                string `json:"id"`
		Stash             string `json:"stash"`
		StashType         string `json:"stashType"`
		Items             []Item `json:"items"`
		Public            bool   `json:"public"`
	} `json:"stashes"`
}

// StashAPIBase is the URL the stash api is located at
const StashAPIBase string = "http://www.pathofexile.com/api/public-stash-tabs"

// TestResponseLoc is where testing data is kept
const TestResponseLoc string = "StashResponse.json"

// GetStored gets the stored testing data and returns it
func GetStored() (*Response, error) {
	f, err := os.Open(TestResponseLoc)
	if err != nil {
		return nil, fmt.Errorf("failed to open TestResponseLoc, err=%s", err)
	}
	defer f.Close()

	decoder := json.NewDecoder(f)
	var response Response
	err = decoder.Decode(&response)
	if err != nil {
		return nil, fmt.Errorf("failed to decode TestResponseLoc, err=%s", err)
	}

	return &response, nil
}

// FetchAndSetStore grabs the latest stash tab api update
// and stores it in TestResponseLoc
func FetchAndSetStore() error {
	resp, err := http.Get(StashAPIBase)
	if err != nil {
		return fmt.Errorf("failed to call stash api, err=%s", err)
	}
	defer resp.Body.Close()

	decoder := json.NewDecoder(resp.Body)
	var response Response
	err = decoder.Decode(&response)
	if err != nil {
		return fmt.Errorf("failed to decode stash tab response, err=%s", err)
	}

	serial, err := json.Marshal(response)
	if err != nil {
		return fmt.Errorf("failed to marshal stash tab response, err=%s", err)
	}

	if err = ioutil.WriteFile(TestResponseLoc, serial, 0777); err != nil {
		return fmt.Errorf("failed to save stashResult.json, err=%s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package state

import (
	"github.com/hashicorp/terraform/terraform"
)

// State is the collection of all state interfaces.
type State interface {
	StateReader
	StateWriter
	StateRefresher
	StatePersister
}

// StateReader is the interface for things that can return a state. Retrieving
// the state here must not error. Loading the state fresh (an operation that
// can likely error) should be implemented by RefreshState. If a state hasn't
// been loaded yet, it is okay for State to return nil.
type StateReader interface {
	State() *terraform.State
}

// StateWriter is the interface that must be implemented by something that
// can write a state. Writing the state can be cached or in-memory, as
// full persistence should be implemented by StatePersister.
type StateWriter interface {
	WriteState(*terraform.State) error
}

// StateRefresher is the interface that is implemented by something that
// can load a state. This might be refreshing it from a remote location or
// it might simply be reloading it from disk.
type StateRefresher interface {
	RefreshState() error
}

// StatePersister is implemented to truly persist a state. Whereas StateWriter
// is allowed to perhaps be caching in memory, PersistState must write the
// state to some durable storage.
type StatePersister interface {
	PersistState() error
}
<commit_msg>add state.Locker interface<commit_after>package state

import (
	"github.com/hashicorp/terraform/terraform"
)

// State is the collection of all state interfaces.
type State interface {
	StateReader
	StateWriter
	StateRefresher
	StatePersister
}

// StateReader is the interface for things that can return a state. Retrieving
// the state here must not error. Loading the state fresh (an operation that
// can likely error) should be implemented by RefreshState. If a state hasn't
// been loaded yet, it is okay for State to return nil.
type StateReader interface {
	State() *terraform.State
}

// StateWriter is the interface that must be implemented by something that
// can write a state. Writing the state can be cached or in-memory, as
// full persistence should be implemented by StatePersister.
type StateWriter interface {
	WriteState(*terraform.State) error
}

// StateRefresher is the interface that is implemented by something that
// can load a state. This might be refreshing it from a remote location or
// it might simply be reloading it from disk.
type StateRefresher interface {
	RefreshState() error
}

// StatePersister is implemented to truly persist a state. Whereas StateWriter
// is allowed to perhaps be caching in memory, PersistState must write the
// state to some durable storage.
type StatePersister interface {
	PersistState() error
}

// Locker is implemented to lock state during command execution.
type Locker interface {
	Lock(reason string) error
	Unlock() error
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"syscall"
)

// A connection to a particular server over a particular protocol (HTTP or
// HTTPS).
type Conn interface {
	// Call the server with the supplied request, returning a response if and
	// only if a response was received from the server. (That is, a 500 error
	// from the server will be returned here as a response with a nil error).
	SendRequest(r *Request) (*Response, error)
}

// Return a connection to the supplied endpoint, based on its scheme and host
// fields.
func NewConn(endpoint *url.URL) (c Conn, err error) {
	switch endpoint.Scheme {
	case "http", "https":
	default:
		err = fmt.Errorf("Unsupported scheme: %s", endpoint.Scheme)
		return
	}

	c = &conn{endpoint}
	return NewRetryingConn(c)
}

type conn struct {
	endpoint *url.URL
}

func makeRawQuery(r *Request) string {
	values := url.Values{}
	for key, val := range r.Parameters {
		values.Set(key, val)
	}

	return values.Encode()
}

func (c *conn) SendRequest(r *Request) (resp *Response, err error) {
	// Create an appropriate URL.
	url := url.URL{
		Scheme:   c.endpoint.Scheme,
		Host:     c.endpoint.Host,
		Path:     r.Path,
		RawQuery: makeRawQuery(r),
	}

	urlStr := url.String()

	// Create a request to the system HTTP library.
	sysReq, err := http.NewRequest(r.Verb, urlStr, bytes.NewBuffer(r.Body))
	if err != nil {
		err = &Error{"http.NewRequest", err}
		return
	}

	// Copy headers.
	for key, val := range r.Headers {
		sysReq.Header.Set(key, val)
	}

	// Call the system HTTP library.
	sysResp, err := http.DefaultClient.Do(sysReq)
	if err != nil {
		// TODO(jacobsa): Remove this logging once it has yielded useful results
		// for investigating this issue:
		//
		//     https://github.com/jacobsa/comeback/issues/11
		//
		log.Println(
			"http.DefaultClient.Do:",
			reflect.TypeOf(err),
			reflect.ValueOf(err),
		)

		if opErr, ok := err.(*net.OpError); ok {
			log.Println("Op:        ", opErr.Op)
			log.Println("Net:       ", opErr.Net)
			log.Println("Addr:      ", opErr.Addr)
			log.Println("Err:       ", opErr.Err)
			log.Println("Temporary: ", opErr.Temporary())
			log.Println("Timeout:   ", opErr.Timeout())

			if errno, ok := opErr.Err.(syscall.Errno); ok {
				log.Printf("Errno: %u\n", errno)
				log.Printf("EPIPE: %u\n", syscall.EPIPE)
			}
		}

		err = &Error{"http.DefaultClient.Do", err}
		return
	}

	// Convert the response.
	resp = &Response{
		StatusCode: sysResp.StatusCode,
	}

	if resp.Body, err = ioutil.ReadAll(sysResp.Body); err != nil {
		err = &Error{"ioutil.ReadAll", err}
		return
	}

	sysResp.Body.Close()

	return
}
<commit_msg>Fixed a bug where sysResp.Body would sometimes not be closed.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"syscall"
)

// A connection to a particular server over a particular protocol (HTTP or
// HTTPS).
type Conn interface {
	// Call the server with the supplied request, returning a response if and
	// only if a response was received from the server. (That is, a 500 error
	// from the server will be returned here as a response with a nil error).
	SendRequest(r *Request) (*Response, error)
}

// Return a connection to the supplied endpoint, based on its scheme and host
// fields.
func NewConn(endpoint *url.URL) (c Conn, err error) {
	switch endpoint.Scheme {
	case "http", "https":
	default:
		err = fmt.Errorf("Unsupported scheme: %s", endpoint.Scheme)
		return
	}

	c = &conn{endpoint}
	return NewRetryingConn(c)
}

type conn struct {
	endpoint *url.URL
}

func makeRawQuery(r *Request) string {
	values := url.Values{}
	for key, val := range r.Parameters {
		values.Set(key, val)
	}

	return values.Encode()
}

func (c *conn) SendRequest(r *Request) (resp *Response, err error) {
	// Create an appropriate URL.
	url := url.URL{
		Scheme:   c.endpoint.Scheme,
		Host:     c.endpoint.Host,
		Path:     r.Path,
		RawQuery: makeRawQuery(r),
	}

	urlStr := url.String()

	// Create a request to the system HTTP library.
	sysReq, err := http.NewRequest(r.Verb, urlStr, bytes.NewBuffer(r.Body))
	if err != nil {
		err = &Error{"http.NewRequest", err}
		return
	}

	// Copy headers.
	for key, val := range r.Headers {
		sysReq.Header.Set(key, val)
	}

	// Call the system HTTP library.
	sysResp, err := http.DefaultClient.Do(sysReq)
	if err != nil {
		// TODO(jacobsa): Remove this logging once it has yielded useful results
		// for investigating this issue:
		//
		//     https://github.com/jacobsa/comeback/issues/11
		//
		log.Println(
			"http.DefaultClient.Do:",
			reflect.TypeOf(err),
			reflect.ValueOf(err),
		)

		if opErr, ok := err.(*net.OpError); ok {
			log.Println("Op:        ", opErr.Op)
			log.Println("Net:       ", opErr.Net)
			log.Println("Addr:      ", opErr.Addr)
			log.Println("Err:       ", opErr.Err)
			log.Println("Temporary: ", opErr.Temporary())
			log.Println("Timeout:   ", opErr.Timeout())

			if errno, ok := opErr.Err.(syscall.Errno); ok {
				log.Printf("Errno: %u\n", errno)
				log.Printf("EPIPE: %u\n", syscall.EPIPE)
			}
		}

		err = &Error{"http.DefaultClient.Do", err}
		return
	}

	// Make sure the body reader is closed no matter how we exit.
	defer sysResp.Body.Close()

	// Convert the response.
	resp = &Response{
		StatusCode: sysResp.StatusCode,
	}

	if resp.Body, err = ioutil.ReadAll(sysResp.Body); err != nil {
		err = &Error{"ioutil.ReadAll", err}
		return
	}

	return
}
<|endoftext|>