<commit_before>package agent

import (
	"fmt"
	"net"
	"net/url"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/drone/drone/build"
	"github.com/drone/drone/model"
	"github.com/drone/drone/version"
	"github.com/drone/drone/yaml"
	"github.com/drone/drone/yaml/expander"
	"github.com/drone/drone/yaml/transform"
)

type Logger interface {
	Write(*build.Line)
}

type Agent struct {
	Update    UpdateFunc
	Logger    LoggerFunc
	Engine    build.Engine
	Timeout   time.Duration
	Platform  string
	Namespace string
	Disable   []string
	Escalate  []string
	Netrc     []string
	Local     string
	Pull      bool
}

func (a *Agent) Poll() error {

	// logrus.Infof("Starting build %s/%s#%d.%d",
	// 	payload.Repo.Owner, payload.Repo.Name, payload.Build.Number, payload.Job.Number)
	//
	//
	// 	logrus.Infof("Finished build %s/%s#%d.%d",
	// 		payload.Repo.Owner, payload.Repo.Name, payload.Build.Number, payload.Job.Number)

	return nil
}

func (a *Agent) Run(payload *model.Work, cancel <-chan bool) error {

	payload.Job.Status = model.StatusRunning
	payload.Job.Started = time.Now().Unix()

	spec, err := a.prep(payload)
	if err != nil {
		payload.Job.Error = err.Error()
		payload.Job.ExitCode = 255
		payload.Job.Finished = payload.Job.Started
		payload.Job.Status = model.StatusError
		a.Update(payload)
		return err
	}
	a.Update(payload)
	err = a.exec(spec, payload, cancel)

	if err != nil {
		payload.Job.ExitCode = 255
		payload.Job.Error = err.Error()
	}
	if exitErr, ok := err.(*build.ExitError); ok {
		payload.Job.ExitCode = exitErr.Code
		payload.Job.Error = "" // exit errors are already written to the log
	}

	payload.Job.Finished = time.Now().Unix()

	switch payload.Job.ExitCode {
	case 128, 130, 137:
		payload.Job.Status = model.StatusKilled
	case 0:
		payload.Job.Status = model.StatusSuccess
	default:
		payload.Job.Status = model.StatusFailure
	}

	a.Update(payload)

	return err
}

func (a *Agent) prep(w *model.Work) (*yaml.Config, error) {

	envs := toEnv(w)
	w.Yaml = expander.ExpandString(w.Yaml, envs)

	// append secrets when verified or when a secret does not require
	// verification
	var secrets []*model.Secret
	for _, secret := range w.Secrets {
		if w.Verified || secret.SkipVerify {
			secrets = append(secrets, secret)
		}
	}

	// inject the netrc file into the clone plugin if the repository is
	// private and requires authentication.
	if w.Repo.IsPrivate {
		secrets = append(secrets, &model.Secret{
			Name:   "DRONE_NETRC_USERNAME",
			Value:  w.Netrc.Login,
			Images: []string{"*"},
			Events: []string{"*"},
		})
		secrets = append(secrets, &model.Secret{
			Name:   "DRONE_NETRC_PASSWORD",
			Value:  w.Netrc.Password,
			Images: []string{"*"},
			Events: []string{"*"},
		})
		secrets = append(secrets, &model.Secret{
			Name:   "DRONE_NETRC_MACHINE",
			Value:  w.Netrc.Machine,
			Images: []string{"*"},
			Events: []string{"*"},
		})
	}

	conf, err := yaml.ParseString(w.Yaml)
	if err != nil {
		return nil, err
	}

	src := "src"
	if url, _ := url.Parse(w.Repo.Link); url != nil {
		host, _, err := net.SplitHostPort(url.Host)
		if err == nil {
			url.Host = host
		}
		src = filepath.Join(src, url.Host, url.Path)
	}

	transform.Clone(conf, w.Repo.Kind)
	transform.Environ(conf, envs)
	transform.DefaultFilter(conf)
	if w.BuildLast != nil {
		transform.ChangeFilter(conf, w.BuildLast.Status)
	}

	transform.ImageSecrets(conf, secrets, w.Build.Event)
	transform.Identifier(conf)
	transform.WorkspaceTransform(conf, "/drone", src)

	if err := transform.Check(conf, w.Repo.IsTrusted); err != nil {
		return nil, err
	}

	transform.CommandTransform(conf)
	transform.ImagePull(conf, a.Pull)
	transform.ImageTag(conf)
	if err := transform.ImageEscalate(conf, a.Escalate); err != nil {
		return nil, err
	}
	transform.PluginParams(conf)

	if a.Local != "" {
		transform.PluginDisable(conf, a.Disable)
		transform.ImageVolume(conf, []string{a.Local + ":" + conf.Workspace.Path})
	}

	transform.Pod(conf, a.Platform)

	return conf, nil
}

func (a *Agent) exec(spec *yaml.Config, payload *model.Work, cancel <-chan bool) error {

	conf := build.Config{
		Engine: a.Engine,
		Buffer: 500,
	}

	pipeline := conf.Pipeline(spec)
	defer pipeline.Teardown()

	// setup the build environment
	if err := pipeline.Setup(); err != nil {
		return err
	}

	replacer := NewSecretReplacer(payload.Secrets)
	timeout := time.After(time.Duration(payload.Repo.Timeout) * time.Minute)

	for {
		select {
		case <-pipeline.Done():
			return pipeline.Err()
		case <-cancel:
			pipeline.Stop()
			return fmt.Errorf("termination request received, build cancelled")
		case <-timeout:
			pipeline.Stop()
			return fmt.Errorf("maximum time limit exceeded, build cancelled")
		case <-time.After(a.Timeout):
			pipeline.Stop()
			return fmt.Errorf("terminal inactive for %v, build cancelled", a.Timeout)
		case <-pipeline.Next():

			// TODO(bradrydzewski) this entire block of code should probably get
			// encapsulated in the pipeline.
			status := model.StatusSuccess
			if pipeline.Err() != nil {
				status = model.StatusFailure
			}
			// updates the build status passed into each container. I realize this is
			// a bit out of place and will work to resolve.
			pipeline.Head().Environment["DRONE_BUILD_STATUS"] = status

			if !pipeline.Head().Constraints.Match(
				a.Platform,
				payload.Build.Deploy,
				payload.Build.Event,
				payload.Build.Branch,
				status, payload.Job.Environment) { // TODO: fix this whole section

				pipeline.Skip()
			} else {
				pipeline.Exec()
			}
		case line := <-pipeline.Pipe():
			line.Out = replacer.Replace(line.Out)
			a.Logger(line)
		}
	}
}

func toEnv(w *model.Work) map[string]string {
	envs := map[string]string{
		"CI":                         "drone",
		"DRONE":                      "true",
		"DRONE_ARCH":                 "linux/amd64",
		"DRONE_REPO":                 w.Repo.FullName,
		"DRONE_REPO_SCM":             w.Repo.Kind,
		"DRONE_REPO_OWNER":           w.Repo.Owner,
		"DRONE_REPO_NAME":            w.Repo.Name,
		"DRONE_REPO_LINK":            w.Repo.Link,
		"DRONE_REPO_AVATAR":          w.Repo.Avatar,
		"DRONE_REPO_BRANCH":          w.Repo.Branch,
		"DRONE_REPO_PRIVATE":         fmt.Sprintf("%v", w.Repo.IsPrivate),
		"DRONE_REPO_TRUSTED":         fmt.Sprintf("%v", w.Repo.IsTrusted),
		"DRONE_REMOTE_URL":           w.Repo.Clone,
		"DRONE_COMMIT_SHA":           w.Build.Commit,
		"DRONE_COMMIT_REF":           w.Build.Ref,
		"DRONE_COMMIT_BRANCH":        w.Build.Branch,
		"DRONE_COMMIT_LINK":          w.Build.Link,
		"DRONE_COMMIT_MESSAGE":       w.Build.Message,
		"DRONE_COMMIT_AUTHOR":        w.Build.Author,
		"DRONE_COMMIT_AUTHOR_EMAIL":  w.Build.Email,
		"DRONE_COMMIT_AUTHOR_AVATAR": w.Build.Avatar,
		"DRONE_BUILD_NUMBER":         fmt.Sprintf("%d", w.Build.Number),
		"DRONE_BUILD_EVENT":          w.Build.Event,
		"DRONE_BUILD_STATUS":         w.Build.Status,
		"DRONE_BUILD_LINK":           fmt.Sprintf("%s/%s/%d", w.System.Link, w.Repo.FullName, w.Build.Number),
		"DRONE_BUILD_CREATED":        fmt.Sprintf("%d", w.Build.Created),
		"DRONE_BUILD_STARTED":        fmt.Sprintf("%d", w.Build.Started),
		"DRONE_BUILD_FINISHED":       fmt.Sprintf("%d", w.Build.Finished),
		"DRONE_JOB_NUMBER":           fmt.Sprintf("%d", w.Job.Number),
		"DRONE_JOB_STATUS":           w.Job.Status,
		"DRONE_JOB_ERROR":            w.Job.Error,
		"DRONE_JOB_EXIT_CODE":        fmt.Sprintf("%d", w.Job.ExitCode),
		"DRONE_JOB_STARTED":          fmt.Sprintf("%d", w.Job.Started),
		"DRONE_JOB_FINISHED":         fmt.Sprintf("%d", w.Job.Finished),
		"DRONE_YAML_VERIFIED":        fmt.Sprintf("%v", w.Verified),
		"DRONE_YAML_SIGNED":          fmt.Sprintf("%v", w.Signed),
		"DRONE_BRANCH":               w.Build.Branch,
		"DRONE_COMMIT":               w.Build.Commit,
		"DRONE_VERSION":              version.Version,
	}

	if w.Build.Event == model.EventTag {
		envs["DRONE_TAG"] = strings.TrimPrefix(w.Build.Ref, "refs/tags/")
	}
	if w.Build.Event == model.EventPull {
		envs["DRONE_PULL_REQUEST"] = pullRegexp.FindString(w.Build.Ref)
	}
	if w.Build.Event == model.EventDeploy {
		envs["DRONE_DEPLOY_TO"] = w.Build.Deploy
	}

	if w.BuildLast != nil {
		envs["DRONE_PREV_BUILD_STATUS"] = w.BuildLast.Status
		envs["DRONE_PREV_BUILD_NUMBER"] = fmt.Sprintf("%v", w.BuildLast.Number)
		envs["DRONE_PREV_COMMIT_SHA"] = w.BuildLast.Commit
	}

	// inject matrix values as environment variables
	for key, val := range w.Job.Environment {
		envs[key] = val
	}
	return envs
}

var pullRegexp = regexp.MustCompile("\\d+")
<commit_msg>adding refspec to agent env<commit_after>package agent

import (
	"fmt"
	"net"
	"net/url"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/drone/drone/build"
	"github.com/drone/drone/model"
	"github.com/drone/drone/version"
	"github.com/drone/drone/yaml"
	"github.com/drone/drone/yaml/expander"
	"github.com/drone/drone/yaml/transform"
)

type Logger interface {
	Write(*build.Line)
}

type Agent struct {
	Update    UpdateFunc
	Logger    LoggerFunc
	Engine    build.Engine
	Timeout   time.Duration
	Platform  string
	Namespace string
	Disable   []string
	Escalate  []string
	Netrc     []string
	Local     string
	Pull      bool
}

func (a *Agent) Poll() error {

	// logrus.Infof("Starting build %s/%s#%d.%d",
	// 	payload.Repo.Owner, payload.Repo.Name, payload.Build.Number, payload.Job.Number)
	//
	//
	// 	logrus.Infof("Finished build %s/%s#%d.%d",
	// 		payload.Repo.Owner, payload.Repo.Name, payload.Build.Number, payload.Job.Number)

	return nil
}

func (a *Agent) Run(payload *model.Work, cancel <-chan bool) error {

	payload.Job.Status = model.StatusRunning
	payload.Job.Started = time.Now().Unix()

	spec, err := a.prep(payload)
	if err != nil {
		payload.Job.Error = err.Error()
		payload.Job.ExitCode = 255
		payload.Job.Finished = payload.Job.Started
		payload.Job.Status = model.StatusError
		a.Update(payload)
		return err
	}
	a.Update(payload)
	err = a.exec(spec, payload, cancel)

	if err != nil {
		payload.Job.ExitCode = 255
		payload.Job.Error = err.Error()
	}
	if exitErr, ok := err.(*build.ExitError); ok {
		payload.Job.ExitCode = exitErr.Code
		payload.Job.Error = "" // exit errors are already written to the log
	}

	payload.Job.Finished = time.Now().Unix()

	switch payload.Job.ExitCode {
	case 128, 130, 137:
		payload.Job.Status = model.StatusKilled
	case 0:
		payload.Job.Status = model.StatusSuccess
	default:
		payload.Job.Status = model.StatusFailure
	}

	a.Update(payload)

	return err
}

func (a *Agent) prep(w *model.Work) (*yaml.Config, error) {

	envs := toEnv(w)
	w.Yaml = expander.ExpandString(w.Yaml, envs)

	// append secrets when verified or when a secret does not require
	// verification
	var secrets []*model.Secret
	for _, secret := range w.Secrets {
		if w.Verified || secret.SkipVerify {
			secrets = append(secrets, secret)
		}
	}

	// inject the netrc file into the clone plugin if the repository is
	// private and requires authentication.
	if w.Repo.IsPrivate {
		secrets = append(secrets, &model.Secret{
			Name:   "DRONE_NETRC_USERNAME",
			Value:  w.Netrc.Login,
			Images: []string{"*"},
			Events: []string{"*"},
		})
		secrets = append(secrets, &model.Secret{
			Name:   "DRONE_NETRC_PASSWORD",
			Value:  w.Netrc.Password,
			Images: []string{"*"},
			Events: []string{"*"},
		})
		secrets = append(secrets, &model.Secret{
			Name:   "DRONE_NETRC_MACHINE",
			Value:  w.Netrc.Machine,
			Images: []string{"*"},
			Events: []string{"*"},
		})
	}

	conf, err := yaml.ParseString(w.Yaml)
	if err != nil {
		return nil, err
	}

	src := "src"
	if url, _ := url.Parse(w.Repo.Link); url != nil {
		host, _, err := net.SplitHostPort(url.Host)
		if err == nil {
			url.Host = host
		}
		src = filepath.Join(src, url.Host, url.Path)
	}

	transform.Clone(conf, w.Repo.Kind)
	transform.Environ(conf, envs)
	transform.DefaultFilter(conf)
	if w.BuildLast != nil {
		transform.ChangeFilter(conf, w.BuildLast.Status)
	}

	transform.ImageSecrets(conf, secrets, w.Build.Event)
	transform.Identifier(conf)
	transform.WorkspaceTransform(conf, "/drone", src)

	if err := transform.Check(conf, w.Repo.IsTrusted); err != nil {
		return nil, err
	}

	transform.CommandTransform(conf)
	transform.ImagePull(conf, a.Pull)
	transform.ImageTag(conf)
	if err := transform.ImageEscalate(conf, a.Escalate); err != nil {
		return nil, err
	}
	transform.PluginParams(conf)

	if a.Local != "" {
		transform.PluginDisable(conf, a.Disable)
		transform.ImageVolume(conf, []string{a.Local + ":" + conf.Workspace.Path})
	}

	transform.Pod(conf, a.Platform)

	return conf, nil
}

func (a *Agent) exec(spec *yaml.Config, payload *model.Work, cancel <-chan bool) error {

	conf := build.Config{
		Engine: a.Engine,
		Buffer: 500,
	}

	pipeline := conf.Pipeline(spec)
	defer pipeline.Teardown()

	// setup the build environment
	if err := pipeline.Setup(); err != nil {
		return err
	}

	replacer := NewSecretReplacer(payload.Secrets)
	timeout := time.After(time.Duration(payload.Repo.Timeout) * time.Minute)

	for {
		select {
		case <-pipeline.Done():
			return pipeline.Err()
		case <-cancel:
			pipeline.Stop()
			return fmt.Errorf("termination request received, build cancelled")
		case <-timeout:
			pipeline.Stop()
			return fmt.Errorf("maximum time limit exceeded, build cancelled")
		case <-time.After(a.Timeout):
			pipeline.Stop()
			return fmt.Errorf("terminal inactive for %v, build cancelled", a.Timeout)
		case <-pipeline.Next():

			// TODO(bradrydzewski) this entire block of code should probably get
			// encapsulated in the pipeline.
			status := model.StatusSuccess
			if pipeline.Err() != nil {
				status = model.StatusFailure
			}
			// updates the build status passed into each container. I realize this is
			// a bit out of place and will work to resolve.
			pipeline.Head().Environment["DRONE_BUILD_STATUS"] = status

			if !pipeline.Head().Constraints.Match(
				a.Platform,
				payload.Build.Deploy,
				payload.Build.Event,
				payload.Build.Branch,
				status, payload.Job.Environment) { // TODO: fix this whole section

				pipeline.Skip()
			} else {
				pipeline.Exec()
			}
		case line := <-pipeline.Pipe():
			line.Out = replacer.Replace(line.Out)
			a.Logger(line)
		}
	}
}

func toEnv(w *model.Work) map[string]string {
	envs := map[string]string{
		"CI":                         "drone",
		"DRONE":                      "true",
		"DRONE_ARCH":                 "linux/amd64",
		"DRONE_REPO":                 w.Repo.FullName,
		"DRONE_REPO_SCM":             w.Repo.Kind,
		"DRONE_REPO_OWNER":           w.Repo.Owner,
		"DRONE_REPO_NAME":            w.Repo.Name,
		"DRONE_REPO_LINK":            w.Repo.Link,
		"DRONE_REPO_AVATAR":          w.Repo.Avatar,
		"DRONE_REPO_BRANCH":          w.Repo.Branch,
		"DRONE_REPO_PRIVATE":         fmt.Sprintf("%v", w.Repo.IsPrivate),
		"DRONE_REPO_TRUSTED":         fmt.Sprintf("%v", w.Repo.IsTrusted),
		"DRONE_REMOTE_URL":           w.Repo.Clone,
		"DRONE_COMMIT_SHA":           w.Build.Commit,
		"DRONE_COMMIT_REF":           w.Build.Ref,
		"DRONE_COMMIT_REFSPEC":       w.Build.Refspec,
		"DRONE_COMMIT_BRANCH":        w.Build.Branch,
		"DRONE_COMMIT_LINK":          w.Build.Link,
		"DRONE_COMMIT_MESSAGE":       w.Build.Message,
		"DRONE_COMMIT_AUTHOR":        w.Build.Author,
		"DRONE_COMMIT_AUTHOR_EMAIL":  w.Build.Email,
		"DRONE_COMMIT_AUTHOR_AVATAR": w.Build.Avatar,
		"DRONE_BUILD_NUMBER":         fmt.Sprintf("%d", w.Build.Number),
		"DRONE_BUILD_EVENT":          w.Build.Event,
		"DRONE_BUILD_STATUS":         w.Build.Status,
		"DRONE_BUILD_LINK":           fmt.Sprintf("%s/%s/%d", w.System.Link, w.Repo.FullName, w.Build.Number),
		"DRONE_BUILD_CREATED":        fmt.Sprintf("%d", w.Build.Created),
		"DRONE_BUILD_STARTED":        fmt.Sprintf("%d", w.Build.Started),
		"DRONE_BUILD_FINISHED":       fmt.Sprintf("%d", w.Build.Finished),
		"DRONE_JOB_NUMBER":           fmt.Sprintf("%d", w.Job.Number),
		"DRONE_JOB_STATUS":           w.Job.Status,
		"DRONE_JOB_ERROR":            w.Job.Error,
		"DRONE_JOB_EXIT_CODE":        fmt.Sprintf("%d", w.Job.ExitCode),
		"DRONE_JOB_STARTED":          fmt.Sprintf("%d", w.Job.Started),
		"DRONE_JOB_FINISHED":         fmt.Sprintf("%d", w.Job.Finished),
		"DRONE_YAML_VERIFIED":        fmt.Sprintf("%v", w.Verified),
		"DRONE_YAML_SIGNED":          fmt.Sprintf("%v", w.Signed),
		"DRONE_BRANCH":               w.Build.Branch,
		"DRONE_COMMIT":               w.Build.Commit,
		"DRONE_VERSION":              version.Version,
	}

	if w.Build.Event == model.EventTag {
		envs["DRONE_TAG"] = strings.TrimPrefix(w.Build.Ref, "refs/tags/")
	}
	if w.Build.Event == model.EventPull {
		envs["DRONE_PULL_REQUEST"] = pullRegexp.FindString(w.Build.Ref)
	}
	if w.Build.Event == model.EventDeploy {
		envs["DRONE_DEPLOY_TO"] = w.Build.Deploy
	}

	if w.BuildLast != nil {
		envs["DRONE_PREV_BUILD_STATUS"] = w.BuildLast.Status
		envs["DRONE_PREV_BUILD_NUMBER"] = fmt.Sprintf("%v", w.BuildLast.Number)
		envs["DRONE_PREV_COMMIT_SHA"] = w.BuildLast.Commit
	}

	// inject matrix values as environment variables
	for key, val := range w.Job.Environment {
		envs[key] = val
	}
	return envs
}

var pullRegexp = regexp.MustCompile("\\d+")
<|endoftext|>


<commit_before>package agent

import (
	"context"
	"net/url"
	"time"

	"github.com/remeh/smartwitter/log"
	"github.com/remeh/smartwitter/twitter"
)

// GetTweets launches a crawler session.
// TODO(remy): better doc
func GetTweets(ctx context.Context) {
	for {
		after := time.After(time.Second * 3)

		// ----------------------

		select {
		case <-after:
			log.Debug("GetTweets is starting.")
			if err := getTweets(ctx); err != nil {
				log.Error("while running GetTweets:", err)
			}
			log.Debug("GetTweets is ending.")
		case <-ctx.Done():
			log.Debug("GetTweets canceled.")
			return
		}
	}
}

// ----------------------

func getTweets(ctx context.Context) error {
	v := url.Values{
		"tweet_mode":  []string{"extended"},
		"lang":        []string{"en"},
		"count":       []string{"50"},
		"result_type": []string{"recent"},
	}
	sr, err := twitter.GetApi().GetSearch("golang", v)
	if err != nil {
		return err
	}

	for _, s := range sr.Statuses {
		now := time.Now()

		// create this tweet and twitter user
		// ----------------------

		// tweet

		t := twitter.TweetFromTweet(s, now, []string{"golang"})

		// twitter user

		tu := twitter.TwitterUserFromTweet(s, now)

		// upsert
		// ----------------------

		if err := twitter.TwitterUserDAO().Upsert(tu); err != nil {
			return log.Err("getTweets: upsert TwitterUser:", err)
		}

		if err := twitter.TweetDAO().Upsert(t); err != nil {
			return log.Err("getTweets: upsert Tweet:", err)
			return err
		}

		log.Debug("stored tweet:", tu.Name, t.Text)

		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}
	}

	return nil
}
<commit_msg>agent: tweet: do not store retweets.<commit_after>package agent

import (
	"context"
	"net/url"
	"strings"
	"time"

	"github.com/remeh/smartwitter/log"
	"github.com/remeh/smartwitter/twitter"
)

// GetTweets launches a crawler session.
// TODO(remy): better doc
func GetTweets(ctx context.Context) {
	for {
		after := time.After(time.Second * 3)

		// ----------------------

		select {
		case <-after:
			log.Debug("GetTweets is starting.")
			if err := getTweets(ctx); err != nil {
				log.Error("while running GetTweets:", err)
			}
			log.Debug("GetTweets is ending.")
		case <-ctx.Done():
			log.Debug("GetTweets canceled.")
			return
		}
	}
}

// ----------------------

func getTweets(ctx context.Context) error {
	v := url.Values{
		"tweet_mode":  []string{"extended"},
		"lang":        []string{"en"},
		"count":       []string{"50"},
		"result_type": []string{"recent"},
	}
	sr, err := twitter.GetApi().GetSearch("golang", v)
	if err != nil {
		return err
	}

	now := time.Now()
	for _, s := range sr.Statuses {
		// atm, we want to ignore the retweets to
		// not fill up the database with retweets.
		if strings.Contains(s.FullText, "RT @") {
			log.Debug("ignoring tweet:", s.User.Name, s.FullText)
			continue
		}

		// create this tweet and twitter user
		// ----------------------

		// tweet

		t := twitter.TweetFromTweet(s, now, []string{"golang"})

		// twitter user

		tu := twitter.TwitterUserFromTweet(s, now)

		// upsert
		// ----------------------

		if err := twitter.TwitterUserDAO().Upsert(tu); err != nil {
			return log.Err("getTweets: upsert TwitterUser:", err)
		}

		if err := twitter.TweetDAO().Upsert(t); err != nil {
			return log.Err("getTweets: upsert Tweet:", err)
			return err
		}

		log.Debug("stored tweet:", tu.Name, t.Text)

		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
)

func main() {
	flag.Parse()
	configPath := flag.Arg(0)
	_, err := os.Stat(configPath)
	if err != nil && os.IsNotExist(err) {
		log.Fatal("ERROR: Could not find configuration file")
	}

	rules, err := ParseConfig(configPath)
	if err != nil {
		log.Fatalf("ERROR: Could not parse config file: %s", err)
	}
	log.Printf("Successfully loaded configuration file. Number of rules: %d", len(rules))

	dirs := make([]string, 0)
	for _, rule := range rules {
		dirs = append(dirs, rule.Path)
	}

	watcher, err := WatchDirs(dirs)
	if err != nil {
		log.Println("ERROR: Could not start watching directories")
		log.Fatal(err)
	}

	defer func() {
		err = watcher.Stop()
		if err != nil {
			log.Fatal("ERROR: Did not shut down cleanly")
		}
	}()

	queue := Manage(watcher.Events, rules)

	log.Println("Watchgopher is now ready process file events")

	for cmd := range queue {
		cmd.Stdout = os.Stdout
		err = cmd.Run()
		if err != nil {
			log.Printf("%s, ARGS: %s -- ERROR: %s\n", cmd.Path, cmd.Args[1:], err)
			continue
		}
		log.Printf("%s, ARGS: %s -- SUCCESS\n", cmd.Path, cmd.Args[1:])
	}
}
<commit_msg>Use pipes to log commands STDOUT and STDERR<commit_after>package main

import (
	"bufio"
	"flag"
	"io"
	"log"
	"os"
	"path"
)

func main() {
	flag.Parse()
	configPath := flag.Arg(0)
	_, err := os.Stat(configPath)
	if err != nil && os.IsNotExist(err) {
		log.Fatal("ERROR: Could not find configuration file")
	}

	rules, err := ParseConfig(configPath)
	if err != nil {
		log.Fatalf("ERROR: Could not parse config file: %s", err)
	}
	log.Printf("Successfully loaded configuration file. Number of rules: %d", len(rules))

	dirs := make([]string, 0)
	for _, rule := range rules {
		dirs = append(dirs, rule.Path)
	}

	watcher, err := WatchDirs(dirs)
	if err != nil {
		log.Println("ERROR: Could not start watching directories")
		log.Fatal(err)
	}

	defer func() {
		err = watcher.Stop()
		if err != nil {
			log.Fatal("ERROR: Did not shut down cleanly")
		}
	}()

	queue := Manage(watcher.Events, rules)

	log.Println("Watchgopher is now ready process file events")

	for cmd := range queue {
		outp, err := cmd.StdoutPipe()
		if err != nil {
			log.Printf("%s, ARGS: %s -- ERROR: %s\n", cmd.Path, cmd.Args[1:], err)
			continue
		}

		errp, err := cmd.StderrPipe()
		if err != nil {
			log.Printf("%s, ARGS: %s -- ERROR: %s\n", cmd.Path, cmd.Args[1:], err)
			continue
		}

		_, filename := path.Split(cmd.Path)

		if err = cmd.Start(); err != nil {
			log.Printf("%s, ARGS: %s -- ERROR: %s\n", cmd.Path, cmd.Args[1:], err)
			continue
		}

		// @TODO: This writes only after the pipes are closed. Would be better
		// to stream here.
		go pipeToLog(filename, "STDOUT", outp)
		go pipeToLog(filename, "STDERR", errp)

		err = cmd.Wait()
		if err != nil {
			log.Printf("%s, ARGS: %s -- ERROR: %s\n", cmd.Path, cmd.Args[1:], err)
			continue
		}
		log.Printf("%s, ARGS: %s -- SUCCESS\n", cmd.Path, cmd.Args[1:])
	}
}

func pipeToLog(filename, prefix string, pipe io.ReadCloser) {
	reader := bufio.NewReader(pipe)
	for {
		line, err := reader.ReadBytes('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Printf("[%s %s] Reading Error: %s", filename, prefix, err)
		}
		log.Printf("[%s %s] %s", filename, prefix, line)
	}
}
<|endoftext|>


<commit_before>package api

import (
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"strconv"

	"github.com/mdlayher/wavepipe/data"

	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/unrolled/render"
)

// ArtistsResponse represents the JSON response for /api/artists
type ArtistsResponse struct {
	Error   *Error        `json:"error"`
	Artists []data.Artist `json:"artists"`
	Albums  []data.Album  `json:"albums"`
	Songs   []data.Song   `json:"songs"`
}

// GetArtists retrieves one or more artists from wavepipe, and returns a HTTP status and JSON
func GetArtists(res http.ResponseWriter, req *http.Request) {
	// Retrieve render
	r := context.Get(req, CtxRender).(*render.Render)

	// Output struct for artists request
	out := ArtistsResponse{}

	// List of artists to return
	artists := make([]data.Artist, 0)

	// Check API version
	if version, ok := mux.Vars(req)["version"]; ok {
		// Check if this API call is supported in the advertised version
		if !apiVersionSet.Has(version) {
			r.JSON(res, 400, errRes(400, "unsupported API version: "+version))
			return
		}
	}

	// Check for an ID parameter
	if pID, ok := mux.Vars(req)["id"]; ok {
		// Verify valid integer ID
		id, err := strconv.Atoi(pID)
		if err != nil {
			r.JSON(res, 400, errRes(400, "invalid integer artist ID"))
			return
		}

		// Load the artist
		artist := new(data.Artist)
		artist.ID = id
		if err := artist.Load(); err != nil {
			// Check for invalid ID
			if err == sql.ErrNoRows {
				r.JSON(res, 404, errRes(404, "artist ID not found"))
				return
			}

			// All other errors
			log.Println(err)
			r.JSON(res, 500, serverErr)
			return
		}

		// On single artist, load the albums for this artist
		albums, err := data.DB.AlbumsForArtist(artist.ID)
		if err != nil {
			log.Println(err)
			r.JSON(res, 500, serverErr)
			return
		}

		// Add albums to output
		out.Albums = albums

		// Get songs for this artist if parameter is true
		if req.URL.Query().Get("songs") == "true" {
			// If requested, on single artist, load the songs for this artist
			songs, err := data.DB.SongsForArtist(artist.ID)
			if err != nil {
				log.Println(err)
				r.JSON(res, 500, serverErr)
				return
			}

			// Add songs to output
			out.Songs = songs
		}

		// Add artist to slice
		artists = append(artists, *artist)
	} else {
		// Check for a limit parameter
		if pLimit := req.URL.Query().Get("limit"); pLimit != "" {
			// Split limit into two integers
			var offset int
			var count int
			if n, err := fmt.Sscanf(pLimit, "%d,%d", &offset, &count); n < 2 || err != nil {
				r.JSON(res, 400, errRes(400, "invalid comma-separated integer pair for limit"))
				return
			}

			// Retrieve limited subset of artists
			tempArtists, err := data.DB.LimitArtists(offset, count)
			if err != nil {
				log.Println(err)
				r.JSON(res, 500, serverErr)
				return
			}

			// Copy artists into the output slice
			artists = tempArtists
		} else {
			// Retrieve all artists
			tempArtists, err := data.DB.AllArtists()
			if err != nil {
				log.Println(err)
				r.JSON(res, 500, serverErr)
				return
			}

			// Copy artists into the output slice
			artists = tempArtists
		}
	}

	// Build response
	out.Error = nil
	out.Artists = artists

	// HTTP 200 OK with JSON
	r.JSON(res, 200, out)
	return
}
<commit_msg>api/artists: refactor and cleanup<commit_after>package api

import (
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"strconv"

	"github.com/mdlayher/wavepipe/data"

	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/unrolled/render"
)

// ArtistsResponse represents the JSON response for the Artists API.
type ArtistsResponse struct {
	Error   *Error        `json:"error"`
	Artists []data.Artist `json:"artists"`
	Albums  []data.Album  `json:"albums"`
	Songs   []data.Song   `json:"songs"`
}

// GetArtists retrieves one or more artists from wavepipe, and returns a HTTP status and JSON.
// It can be used to fetch a single artist, a limited subset of artists, or all artists, depending
// on the request parameters.
func GetArtists(w http.ResponseWriter, r *http.Request) {
	// Retrieve render
	ren := context.Get(r, CtxRender).(*render.Render)

	// Output struct for artists request
	out := ArtistsResponse{}

	// Check API version
	if version, ok := mux.Vars(r)["version"]; ok {
		// Check if this API call is supported in the advertised version
		if !apiVersionSet.Has(version) {
			ren.JSON(w, 400, errRes(400, "unsupported API version: "+version))
			return
		}
	}

	// Check for an ID parameter
	if pID, ok := mux.Vars(r)["id"]; ok {
		// Verify valid integer ID
		id, err := strconv.Atoi(pID)
		if err != nil {
			ren.JSON(w, 400, errRes(400, "invalid integer artist ID"))
			return
		}

		// Load the artist
		artist := &data.Artist{ID: id}
		if err := artist.Load(); err != nil {
			// Check for invalid ID
			if err == sql.ErrNoRows {
				ren.JSON(w, 404, errRes(404, "artist ID not found"))
				return
			}

			// All other errors
			log.Println(err)
			ren.JSON(w, 500, serverErr)
			return
		}

		// Store artist in slice
		out.Artists = []data.Artist{*artist}

		// On single artist, load the albums for this artist
		albums, err := data.DB.AlbumsForArtist(artist.ID)
		if err != nil {
			log.Println(err)
			ren.JSON(w, 500, serverErr)
			return
		}

		// Add albums to output
		out.Albums = albums

		// Get songs for this artist if parameter is true
		if r.URL.Query().Get("songs") == "true" {
			// If requested, on single artist, load the songs for this artist
			songs, err := data.DB.SongsForArtist(artist.ID)
			if err != nil {
				log.Println(err)
				ren.JSON(w, 500, serverErr)
				return
			}

			// Add songs to output
			out.Songs = songs
		}

		// HTTP 200 OK with JSON
		ren.JSON(w, 200, out)
		return
	}

	// Check for a limit parameter
	if pLimit := r.URL.Query().Get("limit"); pLimit != "" {
		// Split limit into two integers
		var offset int
		var count int
		if n, err := fmt.Sscanf(pLimit, "%d,%d", &offset, &count); n < 2 || err != nil {
			ren.JSON(w, 400, errRes(400, "invalid comma-separated integer pair for limit"))
			return
		}

		// Retrieve limited subset of artists
		artists, err := data.DB.LimitArtists(offset, count)
		if err != nil {
			log.Println(err)
			ren.JSON(w, 500, serverErr)
			return
		}

		// HTTP 200 OK with JSON
		out.Artists = artists
		ren.JSON(w, 200, out)
		return
	}

	// If no other case, retrieve all artists
	artists, err := data.DB.AllArtists()
	if err != nil {
		log.Println(err)
		ren.JSON(w, 500, serverErr)
		return
	}

	// HTTP 200 OK with JSON
	out.Artists = artists
	ren.JSON(w, 200, out)
	return
}
<|endoftext|>


<commit_before>package integration_test

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/cloudfoundry/libbuildpack/cutlass"
	"github.com/cloudfoundry/libbuildpack/packager"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("CF NodeJS Buildpack", func() {
	var app *cutlass.App
	AfterEach(func() {
		if app != nil {
			app.Destroy()
		}
		app = nil
	})

	Context("when specifying a range for the nodeJS version in the package.json", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "node_version_range"))
		})

		It("resolves to a nodeJS version successfully", func() {
			PushAppAndConfirm(app)

			Expect(app.Stdout.String()).To(MatchRegexp("Installing node 4\\.\\d+\\.\\d+"))
			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
		})
	})

	Context("when specifying a version 6 for the nodeJS version in the package.json", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "node_version_6"))
		})

		It("resolves to a nodeJS version successfully", func() {
			PushAppAndConfirm(app)

			Expect(app.Stdout.String()).To(MatchRegexp("Installing node 6\\.\\d+\\.\\d+"))
			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))

			if ApiHasTask() {
				By("running a task", func() {
					By("can find node in the container", func() {
						command := exec.Command("cf", "run-task", app.Name, "echo \"RUNNING A TASK: $(node --version)\"")
						_, err := command.Output()
						Expect(err).To(BeNil())

						Eventually(func() string {
							return app.Stdout.String()
						}, "30s").Should(MatchRegexp("RUNNING A TASK: v6\\.\\d+\\.\\d+"))
					})
				})
			}
		})
	})

	Context("when not specifying a nodeJS version in the package.json", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "without_node_version"))
		})

		It("resolves to the stable nodeJS version successfully", func() {
			PushAppAndConfirm(app)

			Expect(app.Stdout.String()).To(MatchRegexp("Installing node 4\\.\\d+\\.\\d+"))
			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
		})
	})

	Context("with an unreleased nodejs version", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "unreleased_node_version"))
		})

		It("displays a nice error messages and gracefully fails", func() {
			Expect(app.Push()).ToNot(BeNil())
			Expect(app.ConfirmBuildpack(buildpackVersion)).To(Succeed())

			Expect(app.Stdout.String()).To(ContainSubstring("Unable to install node: no match found for 9000.0.0"))
		})
	})

	Context("with an unsupported, but released, nodejs version", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "unsupported_node_version"))
		})

		It("displays a nice error messages and gracefully fails", func() {
			Expect(app.Push()).ToNot(BeNil())
			Expect(app.ConfirmBuildpack(buildpackVersion)).To(Succeed())

			Expect(app.Stdout.String()).To(ContainSubstring("Unable to install node: no match found for 4.1.1"))
		})
	})

	Context("with no Procfile and OPTIMIZE_MEMORY=true", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "simple_app"))
			app.SetEnv("OPTIMIZE_MEMORY", "true")
		})

		It("is running with autosized max_old_space_size", func() {
			PushAppAndConfirm(app)
			Expect(app.GetBody("/")).To(ContainSubstring("MaxOldSpace: 96")) // 128 * 75%
		})
	})

	Context("with no Procfile and OPTIMIZE_MEMORY is unset", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "simple_app"))
		})

		It("is running with autosized max_old_space_size", func() {
			PushAppAndConfirm(app)
			Expect(app.GetBody("/")).To(ContainSubstring("MaxOldSpace: undefined"))
		})
	})

	It("with an app that has vendored dependencies", func() {
		app = cutlass.New(filepath.Join(bpDir, "fixtures", "vendored_dependencies"))
		PushAppAndConfirm(app)

		By("does not output protip that recommends user vendors dependencies", func() {
			Expect(app.Stdout.String()).ToNot(MatchRegexp("PRO TIP:(.*) It is recommended to vendor the application's Node.js dependencies"))
		})

		if !cutlass.Cached {
			By("with an uncached buildpack", func() {
				By("successfully deploys and includes the dependencies", func() {
					Expect(app.GetBody("/")).To(ContainSubstring("0000000005"))
					Expect(app.Stdout.String()).To(ContainSubstring("Download [https://"))
				})
			})
		}

		if cutlass.Cached {
			By("with a cached buildpack", func() {
				By("deploys without hitting the internet", func() {
					Expect(app.GetBody("/")).To(ContainSubstring("0000000005"))
					Expect(app.Stdout.String()).To(ContainSubstring("Copy [/tmp/buildpacks/"))
				})
			})
		}
	})

	Context("with an app that has vendored dependencies", func() {
		BeforeEach(func() {
			if !cutlass.Cached {
				Skip("cached tests")
			}
		})

		It("does not call out over the internet", func() {
			bpFile, err := packager.Package(bpDir, packager.CacheDir, fmt.Sprintf("%s.%s", buildpackVersion, "notraffic1"), cutlass.Cached)
			Expect(err).To(BeNil())
			defer os.Remove(bpFile)

			traffic, err := cutlass.InternetTraffic(
				bpDir,
				"fixtures/vendored_dependencies",
				bpFile,
				[]string{},
			)
			Expect(err).To(BeNil())
			Expect(traffic).To(HaveLen(0))
		})
	})

	Context("with an app with a yarn.lock file", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "with_yarn"))
		})

		It("successfully deploys and vendors the dependencies via yarn", func() {
			PushAppAndConfirm(app)
			Expect(app.Stdout.String()).To(ContainSubstring("Running yarn in online mode"))

			Expect(filepath.Join(app.Path, "node_modules")).ToNot(BeADirectory())
			Expect(app.Files("app")).To(ContainElement("app/node_modules"))

			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
		})

		AssertUsesProxyDuringStagingIfPresent("with_yarn")
	})

	Context("with an app with a yarn.lock and vendored dependencies", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "with_yarn_vendored"))
		})

		It("deploys without hitting the internet", func() {
			PushAppAndConfirm(app)
			Expect(app.Stdout.String()).To(ContainSubstring("Running yarn in offline mode"))
			Expect(app.GetBody("/microtime")).To(MatchRegexp("native time: \\d+\\.\\d+"))
		})

		It("does not call out over the internet", func() {
			if !cutlass.Cached {
				Skip("cached tests")
			}
			bpFile, err := packager.Package(bpDir, packager.CacheDir, fmt.Sprintf("%s.%s", buildpackVersion, "notraffic2"), cutlass.Cached)
			Expect(err).To(BeNil())
			defer os.Remove(bpFile)

			traffic, err := cutlass.InternetTraffic(
				bpDir,
				"fixtures/with_yarn_vendored",
				bpFile,
				[]string{},
			)
			Expect(err).To(BeNil())
			Expect(traffic).To(HaveLen(0))
		})
	})

	Context("with an app with an out of date yarn.lock", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "out_of_date_yarn_lock"))
		})

		It("warns that yarn.lock is out of date", func() {
			PushAppAndConfirm(app)
			Expect(app.Stdout.String()).To(ContainSubstring("yarn.lock is outdated"))
		})
	})

	Context("with an app with pre and post scripts", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "pre_post_commands"))
		})

		It("runs the scripts through npm run", func() {
			PushAppAndConfirm(app)
			Expect(app.Stdout.String()).To(ContainSubstring("Running heroku-prebuild (npm)"))
			Expect(app.Stdout.String()).To(ContainSubstring("Running heroku-postbuild (npm)"))
			Expect(app.GetBody("/")).To(ContainSubstring("Text: Hello Buildpacks Team"))
			Expect(app.GetBody("/")).To(ContainSubstring("Text: Goodbye Buildpacks Team"))
		})
	})

	Context("with an app with no vendored dependencies", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "no_vendored_dependencies"))
		})

		It("successfully deploys and vendors the dependencies", func() {
			PushAppAndConfirm(app)

			Expect(filepath.Join(app.Path, "node_modules")).ToNot(BeADirectory())
			Expect(app.Files("app")).To(ContainElement("app/node_modules"))

			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))

			By("outputs protip that recommends user vendors dependencies", func() {
				Expect(app.Stdout.String()).To(MatchRegexp("PRO TIP:(.*) It is recommended to vendor the application's Node.js dependencies"))
			})
		})

		AssertUsesProxyDuringStagingIfPresent("no_vendored_dependencies")
	})

	Context("with an incomplete node_modules directory", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "incomplete_node_modules"))
		})

		It("downloads missing dependencies from package.json", func() {
			PushAppAndConfirm(app)
			Expect(filepath.Join(app.Path, "node_modules", "hashish")).ToNot(BeADirectory())
			Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/hashish"))
			Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/express"))
		})
	})

	Context("with an incomplete package.json", func() {
		var tmpDir string
		BeforeEach(func() {
			var err error
			tmpDir, err = cutlass.CopyFixture(filepath.Join(bpDir, "fixtures", "incomplete_package_json"))
			Expect(err).To(BeNil())
			app = cutlass.New(tmpDir)
		})
		AfterEach(func() { os.RemoveAll(tmpDir) })

		It("does not overwrite the vendored modules not listed in package.json", func() {
			PushAppAndConfirm(app)
			Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/leftpad"))
			Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/hashish"))
		})
	})

	PContext("with a cached buildpack in an air gapped environment", func() {
		BeforeEach(func() {
			if !cutlass.Cached {
				Skip("cached tests")
			}
		})
		// TODO :cached tag only
		//     before(:each) do
		//       `cf unbind-staging-security-group public_networks`
		//       `cf unbind-staging-security-group dns`
		//     end
		//
		//     after(:each) do
		//       `cf bind-staging-security-group public_networks`
		//       `cf bind-staging-security-group dns`
		//     end
		//
		//     context 'with no npm version specified' do
		//       let (:app_name) { 'airgapped_no_npm_version' }
		//
		//       subject(:app) do
		//         Machete.deploy_app(app_name)
		//       end
		//
		//       it 'is running with the default version of npm' do
		//         expect(app).to be_running
		//         expect(app).not_to have_internet_traffic
		//
		//         default_version = YAML.load_file(File.join(File.dirname(__FILE__), '..', '..', 'manifest.yml'))['default_versions'].find { |a| a['name'] == 'node' }['version']
		//         expect(app).to have_logged /Installing node #{default_version}/
		//         expect(app).to have_logged("Using default npm version")
		//       end
		//     end
		//
		//     context 'with invalid npm version specified' do
		//       let (:app_name) { 'airgapped_invalid_npm_version' }
		//
		//       it 'is not running and prints an error message' do
		//         expect(app).not_to be_running
		//         expect(app).to have_logged("We're unable to download the version of npm")
		//       end
		//     end
	})

	Describe("NODE_HOME and NODE_ENV", func() {
		BeforeEach(func() {
			if !cutlass.Cached {
				Skip("cached tests")
			}
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "logenv"))
		})

		It("sets the NODE_HOME to correct value", func() {
			PushAppAndConfirm(app)
			Expect(app.Stdout.String()).To(MatchRegexp("NODE_HOME=\\S*/0/node"))

			body, err := app.GetBody("/")
			Expect(err).To(BeNil())
			Expect(body).To(MatchRegexp(`"NODE_HOME":"[^"]*/0/node"`))
			Expect(body).To(ContainSubstring(`"NODE_ENV":"production"`))
			Expect(body).To(ContainSubstring(`"MEMORY_AVAILABLE":"128"`))
		})
	})
})
<commit_msg>Simplify integration test setup<commit_after>package integration_test

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/cloudfoundry/libbuildpack/cutlass"
	"github.com/cloudfoundry/libbuildpack/packager"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("CF NodeJS Buildpack", func() {
	var app *cutlass.App
	AfterEach(func() {
		if app != nil {
			app.Destroy()
		}
		app = nil
	})

	Context("when specifying a range for the nodeJS version in the package.json", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "node_version_range"))
		})

		It("resolves to a nodeJS version successfully", func() {
			PushAppAndConfirm(app)

			Expect(app.Stdout.String()).To(MatchRegexp("Installing node 4\\.\\d+\\.\\d+"))
			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
		})
	})

	Context("when specifying a version 6 for the nodeJS version in the package.json", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "node_version_6"))
		})

		It("resolves to a nodeJS version successfully", func() {
			PushAppAndConfirm(app)

			Expect(app.Stdout.String()).To(MatchRegexp("Installing node 6\\.\\d+\\.\\d+"))
			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))

			if ApiHasTask() {
				By("running a task", func() {
					By("can find node in the container", func() {
						command := exec.Command("cf", "run-task", app.Name, "echo \"RUNNING A TASK: $(node --version)\"")
						_, err := command.Output()
						Expect(err).To(BeNil())

						Eventually(func() string {
							return app.Stdout.String()
						}, "30s").Should(MatchRegexp("RUNNING A TASK: v6\\.\\d+\\.\\d+"))
					})
				})
			}
		})
	})

	Context("when not specifying a nodeJS version in the package.json", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "without_node_version"))
		})

		It("resolves to the stable nodeJS version successfully", func() {
			PushAppAndConfirm(app)

			Expect(app.Stdout.String()).To(MatchRegexp("Installing node 4\\.\\d+\\.\\d+"))
			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
		})
	})

	Context("with an unreleased nodejs version", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "unreleased_node_version"))
		})

		It("displays a nice error messages and gracefully fails", func() {
			Expect(app.Push()).ToNot(BeNil())
			Expect(app.ConfirmBuildpack(buildpackVersion)).To(Succeed())

			Expect(app.Stdout.String()).To(ContainSubstring("Unable to install node: no match found for 9000.0.0"))
		})
	})

	Context("with an unsupported, but released, nodejs version", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "unsupported_node_version"))
		})

		It("displays a nice error messages and gracefully fails", func() {
			Expect(app.Push()).ToNot(BeNil())
			Expect(app.ConfirmBuildpack(buildpackVersion)).To(Succeed())

			Expect(app.Stdout.String()).To(ContainSubstring("Unable to install node: no match found for 4.1.1"))
		})
	})

	Context("with no Procfile and OPTIMIZE_MEMORY=true", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "simple_app"))
			app.SetEnv("OPTIMIZE_MEMORY", "true")
		})

		It("is running with autosized max_old_space_size", func() {
			PushAppAndConfirm(app)
			Expect(app.GetBody("/")).To(ContainSubstring("MaxOldSpace: 96")) // 128 * 75%
		})
	})

	Context("with no Procfile and OPTIMIZE_MEMORY is unset", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "simple_app"))
		})

		It("is running with autosized max_old_space_size", func() {
			PushAppAndConfirm(app)
			Expect(app.GetBody("/")).To(ContainSubstring("MaxOldSpace: undefined"))
		})
	})

	It("with an app that has vendored dependencies", func() {
		app = cutlass.New(filepath.Join(bpDir, "fixtures", "vendored_dependencies"))
		PushAppAndConfirm(app)

		By("does not output protip that recommends user vendors dependencies", func() {
			Expect(app.Stdout.String()).ToNot(MatchRegexp("PRO TIP:(.*) It is recommended to vendor the application's Node.js dependencies"))
		})

		if !cutlass.Cached {
			By("with an uncached buildpack", func() {
				By("successfully deploys and includes the dependencies", func() {
					Expect(app.GetBody("/")).To(ContainSubstring("0000000005"))
					Expect(app.Stdout.String()).To(ContainSubstring("Download [https://"))
				})
			})
		}

		if cutlass.Cached {
			By("with a cached buildpack", func() {
				By("deploys without hitting the internet", func() {
					Expect(app.GetBody("/")).To(ContainSubstring("0000000005"))
					Expect(app.Stdout.String()).To(ContainSubstring("Copy [/tmp/buildpacks/"))
				})
			})
		}
	})

	Context("with an app that has vendored dependencies", func() {
		BeforeEach(func() {
			if !cutlass.Cached {
				Skip("cached tests")
			}
		})

		It("does not call out over the internet", func() {
			bpFile, err := packager.Package(bpDir, packager.CacheDir, fmt.Sprintf("%s.%s", buildpackVersion, "notraffic1"), cutlass.Cached)
			Expect(err).To(BeNil())
			defer os.Remove(bpFile)

			traffic, err := cutlass.InternetTraffic(
				bpDir,
				"fixtures/vendored_dependencies",
				bpFile,
				[]string{},
			)
			Expect(err).To(BeNil())
			Expect(traffic).To(HaveLen(0))
		})
	})

	Context("with an app with a yarn.lock file", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "with_yarn"))
		})

		It("successfully deploys and vendors the dependencies via yarn", func() {
			PushAppAndConfirm(app)
			Expect(app.Stdout.String()).To(ContainSubstring("Running yarn in online mode"))

			Expect(filepath.Join(app.Path, "node_modules")).ToNot(BeADirectory())
			Expect(app.Files("app")).To(ContainElement("app/node_modules"))

			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))
		})

		AssertUsesProxyDuringStagingIfPresent("with_yarn")
	})

	Context("with an app with a yarn.lock and vendored dependencies", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "with_yarn_vendored"))
		})

		It("deploys without hitting the internet", func() {
			PushAppAndConfirm(app)
			Expect(app.Stdout.String()).To(ContainSubstring("Running yarn in offline mode"))
			Expect(app.GetBody("/microtime")).To(MatchRegexp("native time: \\d+\\.\\d+"))
		})

		It("does not call out over the internet", func() {
			if !cutlass.Cached {
				Skip("cached tests")
			}
			bpFile, err := packager.Package(bpDir, packager.CacheDir, fmt.Sprintf("%s.%s", buildpackVersion, "notraffic2"), cutlass.Cached)
			Expect(err).To(BeNil())
			defer os.Remove(bpFile)

			traffic, err := cutlass.InternetTraffic(
				bpDir,
				"fixtures/with_yarn_vendored",
				bpFile,
				[]string{},
			)
			Expect(err).To(BeNil())
			Expect(traffic).To(HaveLen(0))
		})
	})

	Context("with an app with an out of date yarn.lock", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "out_of_date_yarn_lock"))
		})

		It("warns that yarn.lock is out of date", func() {
			PushAppAndConfirm(app)
			Expect(app.Stdout.String()).To(ContainSubstring("yarn.lock is outdated"))
		})
	})

	Context("with an app with pre and post scripts", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "pre_post_commands"))
		})

		It("runs the scripts through npm run", func() {
			PushAppAndConfirm(app)
			Expect(app.Stdout.String()).To(ContainSubstring("Running heroku-prebuild (npm)"))
			Expect(app.Stdout.String()).To(ContainSubstring("Running heroku-postbuild (npm)"))
			Expect(app.GetBody("/")).To(ContainSubstring("Text: Hello Buildpacks Team"))
			Expect(app.GetBody("/")).To(ContainSubstring("Text: Goodbye Buildpacks Team"))
		})
	})

	Context("with an app with no vendored dependencies", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "no_vendored_dependencies"))
		})

		It("successfully deploys and vendors the dependencies", func() {
			PushAppAndConfirm(app)

			Expect(filepath.Join(app.Path, "node_modules")).ToNot(BeADirectory())
			Expect(app.Files("app")).To(ContainElement("app/node_modules"))

			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World!"))

			By("outputs protip that recommends user vendors dependencies", func() {
				Expect(app.Stdout.String()).To(MatchRegexp("PRO TIP:(.*) It is recommended to vendor the application's Node.js dependencies"))
			})
		})

		AssertUsesProxyDuringStagingIfPresent("no_vendored_dependencies")
	})

	Context("with an incomplete node_modules directory", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "incomplete_node_modules"))
		})

		It("downloads missing dependencies from package.json", func() {
			PushAppAndConfirm(app)
			Expect(filepath.Join(app.Path, "node_modules", "hashish")).ToNot(BeADirectory())
			Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/hashish"))
			Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/express"))
		})
	})

	Context("with an incomplete package.json", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "incomplete_package_json"))
		})

		It("does not overwrite the vendored modules not listed in package.json", func() {
			PushAppAndConfirm(app)
			Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/leftpad"))
			Expect(app.Files("app/node_modules")).To(ContainElement("app/node_modules/hashish"))
		})
	})

	PContext("with a cached buildpack in an air gapped environment", func() {
		BeforeEach(func() {
			if !cutlass.Cached {
				Skip("cached tests")
			}
		})
		// TODO :cached tag only
		//     before(:each) do
		//       `cf unbind-staging-security-group public_networks`
		//       `cf unbind-staging-security-group dns`
		//     end
		//
		//     after(:each) do
		//       `cf bind-staging-security-group public_networks`
		//       `cf bind-staging-security-group dns`
		//     end
		//
		//     context 'with no npm version specified' do
		//       let (:app_name) { 'airgapped_no_npm_version' }
		//
		//       subject(:app) do
		//         Machete.deploy_app(app_name)
		//       end
		//
		//       it 'is running with the default version of npm' do
		//         expect(app).to be_running
		//         expect(app).not_to have_internet_traffic
		//
		//         default_version = YAML.load_file(File.join(File.dirname(__FILE__), '..', '..', 'manifest.yml'))['default_versions'].find { |a| a['name'] == 'node' }['version']
		//         expect(app).to have_logged /Installing node #{default_version}/
		//         expect(app).to have_logged("Using default npm version")
		//       end
		//     end
		//
		//     context 'with invalid npm version specified' do
		//       let (:app_name) { 'airgapped_invalid_npm_version' }
		//
		//       it 'is not running and prints an error message' do
		//         expect(app).not_to be_running
		//         expect(app).to have_logged("We're unable to download the version of npm")
		//       end
		//     end
	})

	Describe("NODE_HOME and NODE_ENV", func() {
		BeforeEach(func() {
			if !cutlass.Cached {
				Skip("cached tests")
			}
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "logenv"))
		})

		It("sets the NODE_HOME to correct value", func() {
			PushAppAndConfirm(app)
			Expect(app.Stdout.String()).To(MatchRegexp("NODE_HOME=\\S*/0/node"))

			body, err := app.GetBody("/")
			Expect(err).To(BeNil())
			Expect(body).To(MatchRegexp(`"NODE_HOME":"[^"]*/0/node"`))
			Expect(body).To(ContainSubstring(`"NODE_ENV":"production"`))
			Expect(body).To(ContainSubstring(`"MEMORY_AVAILABLE":"128"`))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright (c) 2015, Peter Mrekaj. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE.txt file.

package ptypes

// parityTable is a parity cache for all 16-bit non-negative integers.
var parityTable = initParityTable()

// initParityTable computes and returns parities for all 16-bit non-negative integers.
func initParityTable() (pt [1 << 16]uint16) {
	for i := 0; i < len(pt); i++ {
		pt[i] = Parity(uint64(i))
	}
	return pt
}

// Parity returns 1 if the number of bits set to 1 in x is odd, otherwise O.
// The time complexity is O(log(n)) where n is the word size.
// The space complexity is O(1).
func Parity(x uint64) (p uint16) {
	x ^= x >> 32
	x ^= x >> 16
	x ^= x >> 8
	x ^= x >> 4
	x ^= x >> 2
	x ^= x >> 1
	return uint16(x & 1)
}

// ParityAlt returns 1 if the number of bits set to 1 in x is odd, otherwise O.
// The time complexity is O(k) where k is the number of bits in x set to 1.
// The space complexity is O(1).
func ParityAlt(x uint64) (p uint16) {
	for x != 0 {
		p ^= 1
		x &= (x - 1)
	}
	return p
}

// ParityLookup returns 1 if the number of bits set to 1 in x is odd, otherwise O.
// The time complexity is O(n/l) where n is the word size and l is the width
// of a word of cache key. The space complexity is O(1) beyond the 1<<l space
// is needed to cache precomputed results, which is constant.
func ParityLookup(x uint64) uint16 {
	return parityTable[(x>>48)&0xffff] ^ parityTable[(x>>32)&0xffff] ^ parityTable[(x>>16)&0xffff] ^ parityTable[x&0xffff]
}
<commit_msg>Change the way how parity table is initialized into more readable way<commit_after>// Copyright (c) 2015, Peter Mrekaj. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE.txt file.

package ptypes

// pt is a parity cache for all 16-bit non-negative integers.
var pt [1 << 16]uint16

// initialize parity table pt.
func init() {
	for i := 0; i < len(pt); i++ {
		pt[i] = Parity(uint64(i))
	}
}

// Parity returns 1 if the number of bits set to 1 in x is odd, otherwise O.
// The time complexity is O(log(n)) where n is the word size.
// The space complexity is O(1).
func Parity(x uint64) (p uint16) {
	x ^= x >> 32
	x ^= x >> 16
	x ^= x >> 8
	x ^= x >> 4
	x ^= x >> 2
	x ^= x >> 1
	return uint16(x & 1)
}

// ParityAlt returns 1 if the number of bits set to 1 in x is odd, otherwise O.
// The time complexity is O(k) where k is the number of bits in x set to 1.
// The space complexity is O(1).
func ParityAlt(x uint64) (p uint16) {
	for x != 0 {
		p ^= 1
		x &= x - 1
	}
	return p
}

// ParityLookup returns 1 if the number of bits set to 1 in x is odd, otherwise O.
// The time complexity is O(n/l) where n is the word size and l is the width
// of a word of cache key. The space complexity is O(1) beyond the 1<<l space
// is needed to cache precomputed results, which is constant.
func ParityLookup(x uint64) uint16 {
	return pt[(x>>48)&0xffff] ^ pt[(x>>32)&0xffff] ^ pt[(x>>16)&0xffff] ^ pt[x&0xffff]
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

var (
	parseSizeRegex *regexp.Regexp
)

// Utility method to determine if a file/dir exists
func FileOrDirExists(path string) (exists bool, isDir bool) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, false
	} else {
		return true, fi.IsDir()
	}
}

// Utility method to determine if a file (NOT dir) exists
func FileExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && !isDir
}

// Utility method to determine if a dir (NOT file) exists
func DirExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && isDir
}

// Utility method to determine if a file/dir exists and is of a specific size
func FileExistsAndIsOfSize(path string, sz int64) bool {
	fi, err := os.Stat(path)

	if err != nil && os.IsNotExist(err) {
		return false
	}

	return fi.Size() == sz
}

// Parse a string representing a size into a number of bytes
// supports m/mb = megabytes, g/gb = gigabytes etc (case insensitive)
func ParseSize(str string) (int64, error) {
	if parseSizeRegex == nil {
		parseSizeRegex = regexp.MustCompile(`(?i)^\s*([\d\.]+)\s*([KMGTP]?B?)\s*$`)
	}

	if match := parseSizeRegex.FindStringSubmatch(str); match != nil {
		value, err := strconv.ParseFloat(match[1], 32)
		if err != nil {
			return 0, err
		}
		strUnits := strings.ToUpper(match[2])
		switch strUnits {
		case "KB", "K":
			return int64(value * (1 << 10)), nil
		case "MB", "M":
			return int64(value * (1 << 20)), nil
		case "GB", "G":
			return int64(value * (1 << 30)), nil
		case "TB", "T":
			return int64(value * (1 << 40)), nil
		case "PB", "P":
			return int64(value * (1 << 50)), nil
		default:
			return int64(value), nil

		}

	} else {
		return 0, errors.New(fmt.Sprintf("Invalid size: %v", str))
	}

}

// Format a number of bytes into a display format
func FormatSize(sz int64) string {

	switch {
	case sz >= (1 << 50):
		return fmt.Sprintf("%.3gPB", float32(sz)/float32(1<<50))
	case sz >= (1 << 40):
		return fmt.Sprintf("%.3gTB", float32(sz)/float32(1<<40))
	case sz >= (1 << 30):
		return fmt.Sprintf("%.3gGB", float32(sz)/float32(1<<30))
	case sz >= (1 << 20):
		return fmt.Sprintf("%.3gMB", float32(sz)/float32(1<<20))
	case sz >= (1 << 10):
		return fmt.Sprintf("%.3gKB", float32(sz)/float32(1<<10))
	default:
		return fmt.Sprintf("%d", sz)
	}
}

// Format a bytes per second transfer rate into a display format
func FormatTransferRate(bytesPerSecond int64) string {
	switch {
	case bytesPerSecond >= (1 << 50): // yeah, right ;)
		return fmt.Sprintf("%.3gPB/s", float32(bytesPerSecond)/float32(1<<50))
	case bytesPerSecond >= (1 << 40):
		return fmt.Sprintf("%.3gTB/s", float32(bytesPerSecond)/float32(1<<40))
	case bytesPerSecond >= (1 << 30):
		return fmt.Sprintf("%.3gGB/s", float32(bytesPerSecond)/float32(1<<30))
	case bytesPerSecond >= (1 << 20):
		return fmt.Sprintf("%.3gMB/s", float32(bytesPerSecond)/float32(1<<20))
	case bytesPerSecond >= (1 << 10):
		return fmt.Sprintf("%.3gKB/s", float32(bytesPerSecond)/float32(1<<10))
	default:
		return fmt.Sprintf("%dBytes/s", bytesPerSecond)
	}
}

// Calculates transfer rates by averaging over n samples
type TransferRateCalculator struct {
	numSamples      int
	samples         []int64 // bytesPerSecond samples
	sampleInsertIdx int
}

func NewTransferRateCalculator(numSamples int) *TransferRateCalculator {
	return &TransferRateCalculator{numSamples, make([]int64, numSamples), 0}
}
func (t *TransferRateCalculator) AddSample(bytesPerSecond int64) {
	t.samples[t.sampleInsertIdx] = bytesPerSecond
	t.sampleInsertIdx = (t.sampleInsertIdx + 1) % t.numSamples
}
func (t *TransferRateCalculator) Average() int64 {
	var sum int64
	for _, s := range t.samples {
		sum += s
	}
	return sum / int64(t.numSamples)
}

// Search a sorted slice of strings for a specific string
// Returns boolean for if found, and either location or insertion point
func StringBinarySearch(sortedSlice []string, searchTerm string) (bool, int) {
	// Convenience method to easily provide boolean of whether to insert or not
	idx := sort.SearchStrings(sortedSlice, searchTerm)
	found := idx < len(sortedSlice) && sortedSlice[idx] == searchTerm
	return found, idx
}

// Overwrite the current line in the console (e.g. for progressive update)
// Requires the previous line length so that it can clear it with spaces
func OverwriteConsoleLine(newString string, lastLineLength int, iobuf io.Writer) {
	if len(newString) < lastLineLength {
		fmt.Fprintf(iobuf, "\r%v%v", newString, strings.Repeat(" ", lastLineLength-len(newString)))
	} else {
		fmt.Fprintf(iobuf, "\r%v", newString)
	}

}

// Remove duplicates from a slice of strings (in place)
// Linear to logarithmic time, doesn't change the ordering of the slice
// allocates/frees a new map of up to the size of the slice though
func StringRemoveDuplicates(s *[]string) {
	if s == nil || *s == nil {
		return
	}
	uniques := NewStringSet()
	insertidx := 0
	for _, x := range *s {
		if !uniques.Contains(x) {
			uniques.Add(x)
			(*s)[insertidx] = x // could do this only when x != insertidx but prob wasteful compare
			insertidx++
		}
	}
	// If any were eliminated it will now be shorter
	*s = (*s)[:insertidx]
}
<commit_msg>Matching function for include/exclude paths<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

var (
	parseSizeRegex *regexp.Regexp
)

// Utility method to determine if a file/dir exists
func FileOrDirExists(path string) (exists bool, isDir bool) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, false
	} else {
		return true, fi.IsDir()
	}
}

// Utility method to determine if a file (NOT dir) exists
func FileExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && !isDir
}

// Utility method to determine if a dir (NOT file) exists
func DirExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && isDir
}

// Utility method to determine if a file/dir exists and is of a specific size
func FileExistsAndIsOfSize(path string, sz int64) bool {
	fi, err := os.Stat(path)

	if err != nil && os.IsNotExist(err) {
		return false
	}

	return fi.Size() == sz
}

// Parse a string representing a size into a number of bytes
// supports m/mb = megabytes, g/gb = gigabytes etc (case insensitive)
func ParseSize(str string) (int64, error) {
	if parseSizeRegex == nil {
		parseSizeRegex = regexp.MustCompile(`(?i)^\s*([\d\.]+)\s*([KMGTP]?B?)\s*$`)
	}

	if match := parseSizeRegex.FindStringSubmatch(str); match != nil {
		value, err := strconv.ParseFloat(match[1], 32)
		if err != nil {
			return 0, err
		}
		strUnits := strings.ToUpper(match[2])
		switch strUnits {
		case "KB", "K":
			return int64(value * (1 << 10)), nil
		case "MB", "M":
			return int64(value * (1 << 20)), nil
		case "GB", "G":
			return int64(value * (1 << 30)), nil
		case "TB", "T":
			return int64(value * (1 << 40)), nil
		case "PB", "P":
			return int64(value * (1 << 50)), nil
		default:
			return int64(value), nil

		}

	} else {
		return 0, errors.New(fmt.Sprintf("Invalid size: %v", str))
	}

}

// Format a number of bytes into a display format
func FormatSize(sz int64) string {

	switch {
	case sz >= (1 << 50):
		return fmt.Sprintf("%.3gPB", float32(sz)/float32(1<<50))
	case sz >= (1 << 40):
		return fmt.Sprintf("%.3gTB", float32(sz)/float32(1<<40))
	case sz >= (1 << 30):
		return fmt.Sprintf("%.3gGB", float32(sz)/float32(1<<30))
	case sz >= (1 << 20):
		return fmt.Sprintf("%.3gMB", float32(sz)/float32(1<<20))
	case sz >= (1 << 10):
		return fmt.Sprintf("%.3gKB", float32(sz)/float32(1<<10))
	default:
		return fmt.Sprintf("%d", sz)
	}
}

// Format a bytes per second transfer rate into a display format
func FormatTransferRate(bytesPerSecond int64) string {
	switch {
	case bytesPerSecond >= (1 << 50): // yeah, right ;)
		return fmt.Sprintf("%.3gPB/s", float32(bytesPerSecond)/float32(1<<50))
	case bytesPerSecond >= (1 << 40):
		return fmt.Sprintf("%.3gTB/s", float32(bytesPerSecond)/float32(1<<40))
	case bytesPerSecond >= (1 << 30):
		return fmt.Sprintf("%.3gGB/s", float32(bytesPerSecond)/float32(1<<30))
	case bytesPerSecond >= (1 << 20):
		return fmt.Sprintf("%.3gMB/s", float32(bytesPerSecond)/float32(1<<20))
	case bytesPerSecond >= (1 << 10):
		return fmt.Sprintf("%.3gKB/s", float32(bytesPerSecond)/float32(1<<10))
	default:
		return fmt.Sprintf("%dBytes/s", bytesPerSecond)
	}
}

// Calculates transfer rates by averaging over n samples
type TransferRateCalculator struct {
	numSamples      int
	samples         []int64 // bytesPerSecond samples
	sampleInsertIdx int
}

func NewTransferRateCalculator(numSamples int) *TransferRateCalculator {
	return &TransferRateCalculator{numSamples, make([]int64, numSamples), 0}
}
func (t *TransferRateCalculator) AddSample(bytesPerSecond int64) {
	t.samples[t.sampleInsertIdx] = bytesPerSecond
	t.sampleInsertIdx = (t.sampleInsertIdx + 1) % t.numSamples
}
func (t *TransferRateCalculator) Average() int64 {
	var sum int64
	for _, s := range t.samples {
		sum += s
	}
	return sum / int64(t.numSamples)
}

// Search a sorted slice of strings for a specific string
// Returns boolean for if found, and either location or insertion point
func StringBinarySearch(sortedSlice []string, searchTerm string) (bool, int) {
	// Convenience method to easily provide boolean of whether to insert or not
	idx := sort.SearchStrings(sortedSlice, searchTerm)
	found := idx < len(sortedSlice) && sortedSlice[idx] == searchTerm
	return found, idx
}

// Overwrite the current line in the console (e.g. for progressive update)
// Requires the previous line length so that it can clear it with spaces
func OverwriteConsoleLine(newString string, lastLineLength int, iobuf io.Writer) {
	if len(newString) < lastLineLength {
		fmt.Fprintf(iobuf, "\r%v%v", newString, strings.Repeat(" ", lastLineLength-len(newString)))
	} else {
		fmt.Fprintf(iobuf, "\r%v", newString)
	}

}

// Remove duplicates from a slice of strings (in place)
// Linear to logarithmic time, doesn't change the ordering of the slice
// allocates/frees a new map of up to the size of the slice though
func StringRemoveDuplicates(s *[]string) {
	if s == nil || *s == nil {
		return
	}
	uniques := NewStringSet()
	insertidx := 0
	for _, x := range *s {
		if !uniques.Contains(x) {
			uniques.Add(x)
			(*s)[insertidx] = x // could do this only when x != insertidx but prob wasteful compare
			insertidx++
		}
	}
	// If any were eliminated it will now be shorter
	*s = (*s)[:insertidx]
}

// Return whether a given filename passes the include / exclude path filters
// Only paths that are in includePaths and outside excludePaths are passed
// If includePaths is empty that filter always passes and the same with excludePaths
// Both path lists support wildcard matches
func FilenamePassesIncludeExcludeFilter(filename string, includePaths, excludePaths []string) bool {
	if len(includePaths) == 0 && len(excludePaths) == 0 {
		return true
	}

	if len(includePaths) > 0 {
		matched := false
		for _, inc := range includePaths {
			matched, _ = filepath.Match(inc, filename)
			if !matched {
				// Also support matching a parent directory without a wildcard
				if strings.HasPrefix(filename, inc+string(filepath.Separator)) {
					matched = true
				}
			}
			if matched {
				break
			}

		}
		if !matched {
			return false
		}
	}

	if len(excludePaths) > 0 {
		for _, ex := range includePaths {
			matched, _ := filepath.Match(ex, filename)
			if matched {
				return false
			}
			// Also support matching a parent directory without a wildcard
			if strings.HasPrefix(filename, ex+string(filepath.Separator)) {
				return false
			}

		}
	}

	return true

}
<|endoftext|>


<commit_before>package urest

import (
	"net/http"
	"strconv"
	"strings"
)

const (
	CONTENT_TYPE_JSON = "application/json; charset=utf-8"
)

func IsSafeRequest(r *http.Request) bool {
	return r.Method == "GET" || r.Method == "HEAD" || r.Method == "OPTIONS" || r.Method == "TRACE"
}

func ReportError(w http.ResponseWriter, err error) {
	errorCodes := []int{
		http.StatusBadRequest,
		http.StatusUnauthorized,
		http.StatusPaymentRequired,
		http.StatusForbidden,
		http.StatusNotFound,
		http.StatusMethodNotAllowed,
		http.StatusNotAcceptable,
		http.StatusProxyAuthRequired,
		http.StatusRequestTimeout,
		http.StatusConflict,
		http.StatusGone,
		http.StatusLengthRequired,
		http.StatusPreconditionFailed,
		http.StatusRequestEntityTooLarge,
		http.StatusRequestURITooLong,
		http.StatusUnsupportedMediaType,
		http.StatusRequestedRangeNotSatisfiable,
		http.StatusExpectationFailed,
		http.StatusTeapot,

		http.StatusInternalServerError,
		http.StatusNotImplemented,
		http.StatusBadGateway,
		http.StatusServiceUnavailable,
		http.StatusGatewayTimeout,
		http.StatusHTTPVersionNotSupported,
	}

	e := err.Error()
	for _, code := range errorCodes {
		if strings.HasPrefix(e, strconv.Itoa(code)+" ") {
			http.Error(w, e, code)
			return
		}
	}
	http.Error(w, e, http.StatusBadRequest)
}

func FeatureFlagPresent(r *http.Request, featureFlagHeader string, featureFlag string) bool {
	return strings.Contains(r.Header.Get(featureFlagHeader), featureFlag)
}
<commit_msg>urest: import more error codes from net/http<commit_after>package urest

import (
	"net/http"
	"strconv"
	"strings"
)

const (
	CONTENT_TYPE_JSON = "application/json; charset=utf-8"
)

func IsSafeRequest(r *http.Request) bool {
	return r.Method == "GET" || r.Method == "HEAD" || r.Method == "OPTIONS" || r.Method == "TRACE"
}

func ReportError(w http.ResponseWriter, err error) {
	errorCodes := []int{
		http.StatusContinue,
		http.StatusSwitchingProtocols,

		http.StatusOK,
		http.StatusCreated,
		http.StatusAccepted,
		http.StatusNonAuthoritativeInfo,
		http.StatusNoContent,
		http.StatusResetContent,
		http.StatusPartialContent,

		http.StatusMultipleChoices,
		http.StatusMovedPermanently,
		http.StatusFound,
		http.StatusSeeOther,
		http.StatusNotModified,
		http.StatusUseProxy,
		http.StatusTemporaryRedirect,

		http.StatusBadRequest,
		http.StatusUnauthorized,
		http.StatusPaymentRequired,
		http.StatusForbidden,
		http.StatusNotFound,
		http.StatusMethodNotAllowed,
		http.StatusNotAcceptable,
		http.StatusProxyAuthRequired,
		http.StatusRequestTimeout,
		http.StatusConflict,
		http.StatusGone,
		http.StatusLengthRequired,
		http.StatusPreconditionFailed,
		http.StatusRequestEntityTooLarge,
		http.StatusRequestURITooLong,
		http.StatusUnsupportedMediaType,
		http.StatusRequestedRangeNotSatisfiable,
		http.StatusExpectationFailed,
		http.StatusTeapot,

		http.StatusInternalServerError,
		http.StatusNotImplemented,
		http.StatusBadGateway,
		http.StatusServiceUnavailable,
		http.StatusGatewayTimeout,
		http.StatusHTTPVersionNotSupported,
	}

	e := err.Error()
	for _, code := range errorCodes {
		if strings.HasPrefix(e, strconv.Itoa(code)+" ") {
			http.Error(w, e, code)
			return
		}
	}
	http.Error(w, e, http.StatusBadRequest)
}

func FeatureFlagPresent(r *http.Request, featureFlagHeader string, featureFlag string) bool {
	return strings.Contains(r.Header.Get(featureFlagHeader), featureFlag)
}
<|endoftext|>


<commit_before>package celeriac

import (
	"encoding/json"
	"time"

	// Package dependencies
	"github.com/nu7hatch/gouuid"
)

/*
Task is a representation of a Celery task
*/
type Task struct {
	// TaskName is the name of the task
	TaskName string

	// ID is the task UUID
	ID string

	// Args are task arguments (optional)
	Args []string

	// KWArgs are keyword arguments (optional)
	KWArgs map[string]interface{}

	// Retries is a number of retries to perform if an error occurs (optional)
	Retries int

	// ETA is the estimated completion time (optional)
	ETA time.Time

	// Expires is the time when this task will expire (optional)
	Expires time.Time
}

/*
NewTask is a factory function that creates and returns a pointer to a new task object
*/
func NewTask(taskName string, args []string, kwargs map[string]interface{}) (*Task, error) {
	newTaskID, err := uuid.NewV4()
	if err != nil {
		return nil, err
	}

	if args == nil {
		args = []string{}
	}

	newTask := Task{
		TaskName: taskName,
		ID:       newTaskID.String(),
		Args:     args,
		KWArgs:   kwargs,
		Retries:  0,
	}

	return &newTask, nil
}

/*
MarshalJSON marshals a Task object into a json bytes array

Time properties are converted to UTC and formatted in ISO8601
*/
func (task *Task) MarshalJSON() ([]byte, error) {
	type _jsonTask struct {
		TaskName string                 `json:"task"`
		ID       string                 `json:"id"`
		Args     []string               `json:"args, omitempty"`
		KWArgs   map[string]interface{} `json:"kwargs, omitempty"`
		Retries  int                    `json:"retries, omitempty"`
		ETA      string                 `json:"eta, omitempty"`
		Expires  string                 `json:"expires, omitempty"`
	}

	out := _jsonTask{
		TaskName: task.TaskName,
		ID:       task.ID,
		Args:     task.Args,
		KWArgs:   task.KWArgs,
		Retries:  task.Retries,
	}

	// Convert time properties to UTC, and format as ISO8601
	if !task.ETA.IsZero() {
		out.ETA = task.ETA.UTC().Format(ConstTimeFormat)
	}

	if !task.Expires.IsZero() {
		out.Expires = task.Expires.UTC().Format(ConstTimeFormat)
	}

	return json.Marshal(out)
}
<commit_msg>Added and updated factory functions for creating a task to allow specifying a custom task id.<commit_after>package celeriac

import (
	"encoding/json"
	"time"

	// Package dependencies
	"github.com/nu7hatch/gouuid"
)

/*
Task is a representation of a Celery task
*/
type Task struct {
	// TaskName is the name of the task
	TaskName string

	// ID is the task UUID
	ID string

	// Args are task arguments (optional)
	Args []string

	// KWArgs are keyword arguments (optional)
	KWArgs map[string]interface{}

	// Retries is a number of retries to perform if an error occurs (optional)
	Retries int

	// ETA is the estimated completion time (optional)
	ETA time.Time

	// Expires is the time when this task will expire (optional)
	Expires time.Time
}

/*
NewTask is a factory function that creates and returns a pointer to a new task object
*/
func NewTask(taskName string, args []string, kwargs map[string]interface{}) (*Task, error) {

	taskUUID, err := uuid.NewV4()
	if err != nil {
		return nil, err
	}

	newTask, err := NewTaskWithID(taskUUID.String(), taskName, args, kwargs)
	if err != nil {
		return nil, err
	}

	return newTask, nil
}

/*
NewTaskWithID is a factory function that creates and returns a pointer to a new task object, allowing caller to
specify the task ID.
*/
func NewTaskWithID(taskID string, taskName string, args []string, kwargs map[string]interface{}) (*Task, error) {

	newTaskID := taskID

	if newTaskID == "" {
		taskUUID, err := uuid.NewV4()
		if err != nil {
			return nil, err
		}

		newTaskID = taskUUID.String()
	}

	if args == nil {
		args = []string{}
	}

	newTask := Task{
		TaskName: taskName,
		ID:       newTaskID,
		Args:     args,
		KWArgs:   kwargs,
		Retries:  0,
	}

	return &newTask, nil
}

/*
MarshalJSON marshals a Task object into a json bytes array

Time properties are converted to UTC and formatted in ISO8601
*/
func (task *Task) MarshalJSON() ([]byte, error) {
	type _jsonTask struct {
		TaskName string                 `json:"task"`
		ID       string                 `json:"id"`
		Args     []string               `json:"args, omitempty"`
		KWArgs   map[string]interface{} `json:"kwargs, omitempty"`
		Retries  int                    `json:"retries, omitempty"`
		ETA      string                 `json:"eta, omitempty"`
		Expires  string                 `json:"expires, omitempty"`
	}

	out := _jsonTask{
		TaskName: task.TaskName,
		ID:       task.ID,
		Args:     task.Args,
		KWArgs:   task.KWArgs,
		Retries:  task.Retries,
	}

	// Convert time properties to UTC, and format as ISO8601
	if !task.ETA.IsZero() {
		out.ETA = task.ETA.UTC().Format(ConstTimeFormat)
	}

	if !task.Expires.IsZero() {
		out.Expires = task.Expires.UTC().Format(ConstTimeFormat)
	}

	return json.Marshal(out)
}
<|endoftext|>


<commit_before>package task

import (
	"context"
	"fmt"
	"io"
	"os"
	"sync/atomic"

	"github.com/go-task/task/v2/internal/compiler"
	compilerv1 "github.com/go-task/task/v2/internal/compiler/v1"
	compilerv2 "github.com/go-task/task/v2/internal/compiler/v2"
	"github.com/go-task/task/v2/internal/execext"
	"github.com/go-task/task/v2/internal/logger"
	"github.com/go-task/task/v2/internal/output"
	"github.com/go-task/task/v2/internal/taskfile"
	"github.com/go-task/task/v2/internal/taskfile/read"
	"github.com/go-task/task/v2/internal/taskfile/version"

	"github.com/Masterminds/semver"
	"golang.org/x/sync/errgroup"
)

const (
	// MaximumTaskCall is the max number of times a task can be called.
	// This exists to prevent infinite loops on cyclic dependencies
	MaximumTaskCall = 100
)

// Executor executes a Taskfile
type Executor struct {
	Taskfile *taskfile.Taskfile
	Dir      string
	Force    bool
	Watch    bool
	Verbose  bool
	Silent   bool
	Dry      bool
	Details  bool

	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer

	Logger      *logger.Logger
	Compiler    compiler.Compiler
	Output      output.Output
	OutputStyle string

	taskvars taskfile.Vars

	taskCallCount map[string]*int32
}

// Run runs Task
func (e *Executor) Run(ctx context.Context, calls ...taskfile.Call) error {
	// check if given tasks exist
	for _, c := range calls {
		if _, ok := e.Taskfile.Tasks[c.Task]; !ok {
			// FIXME: move to the main package
			e.PrintTasksHelp()
			return &taskNotFoundError{taskName: c.Task}
		}
	}

	if e.Details {
		task := calls[0].Task

		if e.Taskfile.Tasks[task].Details == "" {
			e.Logger.Errf("task: There is no detailed description for task: %s", task)
			return nil
		}

		e.Logger.Outf(e.Taskfile.Tasks[task].Details)
		return nil
	}

	if e.Watch {
		return e.watchTasks(calls...)
	}

	for _, c := range calls {
		if err := e.RunTask(ctx, c); err != nil {
			return err
		}
	}
	return nil
}

// Setup setups Executor's internal state
func (e *Executor) Setup() error {
	var err error
	e.Taskfile, err = read.Taskfile(e.Dir)
	if err != nil {
		return err
	}
	e.taskvars, err = read.Taskvars(e.Dir)
	if err != nil {
		return err
	}

	v, err := semver.NewConstraint(e.Taskfile.Version)
	if err != nil {
		return fmt.Errorf(`task: could not parse taskfile version "%s": %v`, e.Taskfile.Version, err)
	}

	if e.Stdin == nil {
		e.Stdin = os.Stdin
	}
	if e.Stdout == nil {
		e.Stdout = os.Stdout
	}
	if e.Stderr == nil {
		e.Stderr = os.Stderr
	}
	e.Logger = &logger.Logger{
		Stdout:  e.Stdout,
		Stderr:  e.Stderr,
		Verbose: e.Verbose,
	}
	switch {
	case version.IsV1(v):
		e.Compiler = &compilerv1.CompilerV1{
			Dir:    e.Dir,
			Vars:   e.taskvars,
			Logger: e.Logger,
		}
	case version.IsV2(v), version.IsV21(v), version.IsV22(v):
		e.Compiler = &compilerv2.CompilerV2{
			Dir:          e.Dir,
			Taskvars:     e.taskvars,
			TaskfileVars: e.Taskfile.Vars,
			Expansions:   e.Taskfile.Expansions,
			Logger:       e.Logger,
		}
	case version.IsV23(v):
		return fmt.Errorf(`task: Taskfile versions greater than v2.3 not implemented in the version of Task`)
	}

	if !version.IsV21(v) && e.Taskfile.Output != "" {
		return fmt.Errorf(`task: Taskfile option "output" is only available starting on Taskfile version v2.1`)
	}
	if !version.IsV22(v) && len(e.Taskfile.Includes) > 0 {
		return fmt.Errorf(`task: Including Taskfiles is only available starting on Taskfile version v2.2`)
	}
	if e.OutputStyle != "" {
		e.Taskfile.Output = e.OutputStyle
	}
	switch e.Taskfile.Output {
	case "", "interleaved":
		e.Output = output.Interleaved{}
	case "group":
		e.Output = output.Group{}
	case "prefixed":
		e.Output = output.Prefixed{}
	default:
		return fmt.Errorf(`task: output option "%s" not recognized`, e.Taskfile.Output)
	}

	if !version.IsV21(v) {
		err := fmt.Errorf(`task: Taskfile option "ignore_error" is only available starting on Taskfile version v2.1`)

		for _, task := range e.Taskfile.Tasks {
			if task.IgnoreError {
				return err
			}
			for _, cmd := range task.Cmds {
				if cmd.IgnoreError {
					return err
				}
			}
		}
	}

	e.taskCallCount = make(map[string]*int32, len(e.Taskfile.Tasks))
	for k := range e.Taskfile.Tasks {
		e.taskCallCount[k] = new(int32)
	}
	return nil
}

// RunTask runs a task by its name
func (e *Executor) RunTask(ctx context.Context, call taskfile.Call) error {
	t, err := e.CompiledTask(call)
	if err != nil {
		return err
	}
	if !e.Watch && atomic.AddInt32(e.taskCallCount[call.Task], 1) >= MaximumTaskCall {
		return &MaximumTaskCallExceededError{task: call.Task}
	}

	if err := e.runDeps(ctx, t); err != nil {
		return err
	}

	if !e.Force {
		upToDate, err := e.isTaskUpToDate(ctx, t)
		if err != nil {
			return err
		}
		if upToDate {
			if !e.Silent {
				e.Logger.Errf(`task: Task "%s" is up to date`, t.Task)
			}
			return nil
		}
	}

	for i := range t.Cmds {
		if err := e.runCommand(ctx, t, call, i); err != nil {
			if err2 := e.statusOnError(t); err2 != nil {
				e.Logger.VerboseErrf("task: error cleaning status on error: %v", err2)
			}

			if execext.IsExitError(err) && t.IgnoreError {
				e.Logger.VerboseErrf("task: task error ignored: %v", err)
				continue
			}

			return &taskRunError{t.Task, err}
		}
	}
	return nil
}

func (e *Executor) runDeps(ctx context.Context, t *taskfile.Task) error {
	g, ctx := errgroup.WithContext(ctx)

	for _, d := range t.Deps {
		d := d

		g.Go(func() error {
			return e.RunTask(ctx, taskfile.Call{Task: d.Task, Vars: d.Vars})
		})
	}

	return g.Wait()
}

func (e *Executor) runCommand(ctx context.Context, t *taskfile.Task, call taskfile.Call, i int) error {
	cmd := t.Cmds[i]

	switch {
	case cmd.Task != "":
		return e.RunTask(ctx, taskfile.Call{Task: cmd.Task, Vars: cmd.Vars})
	case cmd.Cmd != "":
		if e.Verbose || (!cmd.Silent && !t.Silent && !e.Silent) {
			e.Logger.Errf(cmd.Cmd)
		}

		if e.Dry {
			return nil
		}

		stdOut := e.Output.WrapWriter(e.Stdout, t.Prefix)
		stdErr := e.Output.WrapWriter(e.Stderr, t.Prefix)
		defer stdOut.Close()
		defer stdErr.Close()

		err := execext.RunCommand(ctx, &execext.RunCommandOptions{
			Command: cmd.Cmd,
			Dir:     t.Dir,
			Env:     getEnviron(t),
			Stdin:   e.Stdin,
			Stdout:  stdOut,
			Stderr:  stdErr,
		})
		if execext.IsExitError(err) && cmd.IgnoreError {
			e.Logger.VerboseErrf("task: command error ignored: %v", err)
			return nil
		}
		return err
	default:
		return nil
	}
}

func getEnviron(t *taskfile.Task) []string {
	if t.Env == nil {
		return nil
	}

	environ := os.Environ()
	for k, v := range t.Env.ToStringMap() {
		environ = append(environ, fmt.Sprintf("%s=%s", k, v))
	}
	return environ
}
<commit_msg>refacotring<commit_after>package task

import (
	"context"
	"fmt"
	"io"
	"os"
	"sync/atomic"

	"github.com/go-task/task/v2/internal/compiler"
	compilerv1 "github.com/go-task/task/v2/internal/compiler/v1"
	compilerv2 "github.com/go-task/task/v2/internal/compiler/v2"
	"github.com/go-task/task/v2/internal/execext"
	"github.com/go-task/task/v2/internal/logger"
	"github.com/go-task/task/v2/internal/output"
	"github.com/go-task/task/v2/internal/taskfile"
	"github.com/go-task/task/v2/internal/taskfile/read"
	"github.com/go-task/task/v2/internal/taskfile/version"

	"github.com/Masterminds/semver"
	"golang.org/x/sync/errgroup"
)

const (
	// MaximumTaskCall is the max number of times a task can be called.
	// This exists to prevent infinite loops on cyclic dependencies
	MaximumTaskCall = 100
)

// Executor executes a Taskfile
type Executor struct {
	Taskfile *taskfile.Taskfile
	Dir      string
	Force    bool
	Watch    bool
	Verbose  bool
	Silent   bool
	Dry      bool
	Details  bool

	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer

	Logger      *logger.Logger
	Compiler    compiler.Compiler
	Output      output.Output
	OutputStyle string

	taskvars taskfile.Vars

	taskCallCount map[string]*int32
}

// Run runs Task
func (e *Executor) Run(ctx context.Context, calls ...taskfile.Call) error {
	// check if given tasks exist
	for _, c := range calls {
		if _, ok := e.Taskfile.Tasks[c.Task]; !ok {
			// FIXME: move to the main package
			e.PrintTasksHelp()
			return &taskNotFoundError{taskName: c.Task}
		}
	}

	if e.Details {
		task := calls[0].Task

		if e.Taskfile.Tasks[task].Details == "" {
			e.Logger.Errf("task: There is no detailed description for task: %s", task)
		} else {
			e.Logger.Outf(e.Taskfile.Tasks[task].Details)
		}
		return nil
	}

	if e.Watch {
		return e.watchTasks(calls...)
	}

	for _, c := range calls {
		if err := e.RunTask(ctx, c); err != nil {
			return err
		}
	}
	return nil
}

// Setup setups Executor's internal state
func (e *Executor) Setup() error {
	var err error
	e.Taskfile, err = read.Taskfile(e.Dir)
	if err != nil {
		return err
	}
	e.taskvars, err = read.Taskvars(e.Dir)
	if err != nil {
		return err
	}

	v, err := semver.NewConstraint(e.Taskfile.Version)
	if err != nil {
		return fmt.Errorf(`task: could not parse taskfile version "%s": %v`, e.Taskfile.Version, err)
	}

	if e.Stdin == nil {
		e.Stdin = os.Stdin
	}
	if e.Stdout == nil {
		e.Stdout = os.Stdout
	}
	if e.Stderr == nil {
		e.Stderr = os.Stderr
	}
	e.Logger = &logger.Logger{
		Stdout:  e.Stdout,
		Stderr:  e.Stderr,
		Verbose: e.Verbose,
	}
	switch {
	case version.IsV1(v):
		e.Compiler = &compilerv1.CompilerV1{
			Dir:    e.Dir,
			Vars:   e.taskvars,
			Logger: e.Logger,
		}
	case version.IsV2(v), version.IsV21(v), version.IsV22(v):
		e.Compiler = &compilerv2.CompilerV2{
			Dir:          e.Dir,
			Taskvars:     e.taskvars,
			TaskfileVars: e.Taskfile.Vars,
			Expansions:   e.Taskfile.Expansions,
			Logger:       e.Logger,
		}
	case version.IsV23(v):
		return fmt.Errorf(`task: Taskfile versions greater than v2.3 not implemented in the version of Task`)
	}

	if !version.IsV21(v) && e.Taskfile.Output != "" {
		return fmt.Errorf(`task: Taskfile option "output" is only available starting on Taskfile version v2.1`)
	}
	if !version.IsV22(v) && len(e.Taskfile.Includes) > 0 {
		return fmt.Errorf(`task: Including Taskfiles is only available starting on Taskfile version v2.2`)
	}
	if e.OutputStyle != "" {
		e.Taskfile.Output = e.OutputStyle
	}
	switch e.Taskfile.Output {
	case "", "interleaved":
		e.Output = output.Interleaved{}
	case "group":
		e.Output = output.Group{}
	case "prefixed":
		e.Output = output.Prefixed{}
	default:
		return fmt.Errorf(`task: output option "%s" not recognized`, e.Taskfile.Output)
	}

	if !version.IsV21(v) {
		err := fmt.Errorf(`task: Taskfile option "ignore_error" is only available starting on Taskfile version v2.1`)

		for _, task := range e.Taskfile.Tasks {
			if task.IgnoreError {
				return err
			}
			for _, cmd := range task.Cmds {
				if cmd.IgnoreError {
					return err
				}
			}
		}
	}

	e.taskCallCount = make(map[string]*int32, len(e.Taskfile.Tasks))
	for k := range e.Taskfile.Tasks {
		e.taskCallCount[k] = new(int32)
	}
	return nil
}

// RunTask runs a task by its name
func (e *Executor) RunTask(ctx context.Context, call taskfile.Call) error {
	t, err := e.CompiledTask(call)
	if err != nil {
		return err
	}
	if !e.Watch && atomic.AddInt32(e.taskCallCount[call.Task], 1) >= MaximumTaskCall {
		return &MaximumTaskCallExceededError{task: call.Task}
	}

	if err := e.runDeps(ctx, t); err != nil {
		return err
	}

	if !e.Force {
		upToDate, err := e.isTaskUpToDate(ctx, t)
		if err != nil {
			return err
		}
		if upToDate {
			if !e.Silent {
				e.Logger.Errf(`task: Task "%s" is up to date`, t.Task)
			}
			return nil
		}
	}

	for i := range t.Cmds {
		if err := e.runCommand(ctx, t, call, i); err != nil {
			if err2 := e.statusOnError(t); err2 != nil {
				e.Logger.VerboseErrf("task: error cleaning status on error: %v", err2)
			}

			if execext.IsExitError(err) && t.IgnoreError {
				e.Logger.VerboseErrf("task: task error ignored: %v", err)
				continue
			}

			return &taskRunError{t.Task, err}
		}
	}
	return nil
}

func (e *Executor) runDeps(ctx context.Context, t *taskfile.Task) error {
	g, ctx := errgroup.WithContext(ctx)

	for _, d := range t.Deps {
		d := d

		g.Go(func() error {
			return e.RunTask(ctx, taskfile.Call{Task: d.Task, Vars: d.Vars})
		})
	}

	return g.Wait()
}

func (e *Executor) runCommand(ctx context.Context, t *taskfile.Task, call taskfile.Call, i int) error {
	cmd := t.Cmds[i]

	switch {
	case cmd.Task != "":
		return e.RunTask(ctx, taskfile.Call{Task: cmd.Task, Vars: cmd.Vars})
	case cmd.Cmd != "":
		if e.Verbose || (!cmd.Silent && !t.Silent && !e.Silent) {
			e.Logger.Errf(cmd.Cmd)
		}

		if e.Dry {
			return nil
		}

		stdOut := e.Output.WrapWriter(e.Stdout, t.Prefix)
		stdErr := e.Output.WrapWriter(e.Stderr, t.Prefix)
		defer stdOut.Close()
		defer stdErr.Close()

		err := execext.RunCommand(ctx, &execext.RunCommandOptions{
			Command: cmd.Cmd,
			Dir:     t.Dir,
			Env:     getEnviron(t),
			Stdin:   e.Stdin,
			Stdout:  stdOut,
			Stderr:  stdErr,
		})
		if execext.IsExitError(err) && cmd.IgnoreError {
			e.Logger.VerboseErrf("task: command error ignored: %v", err)
			return nil
		}
		return err
	default:
		return nil
	}
}

func getEnviron(t *taskfile.Task) []string {
	if t.Env == nil {
		return nil
	}

	environ := os.Environ()
	for k, v := range t.Env.ToStringMap() {
		environ = append(environ, fmt.Sprintf("%s=%s", k, v))
	}
	return environ
}
<|endoftext|>


<commit_before>package manta

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/golang/protobuf/proto"
)

var debugMode, traceMode, fixturesMode bool

func init() {
	if os.Getenv("DEBUG") != "" {
		debugMode = true
	}
	if os.Getenv("TRACE") != "" {
		traceMode = true
	}
	if os.Getenv("FIXTURES") != "" {
		fixturesMode = true
	}
}

var (
	_sprintf = fmt.Sprintf
	_sdump   = spew.Sdump
)

// Convert a string to an int32
func atoi32(s string) (int32, error) {
	n, err := strconv.ParseInt(s, 0, 32)
	if err != nil {
		return 0, err
	}
	return int32(n), nil
}

// printf only if debugging
func _debugf(format string, args ...interface{}) {
	if debugMode {
		args = append([]interface{}{_caller(2)}, args...)
		fmt.Printf("%s: "+format+"\n", args...)
	}
}

// printf only if tracing
func _tracef(format string, args ...interface{}) {
	if traceMode {
		args = append([]interface{}{_caller(2)}, args...)
		fmt.Printf("%s: "+format+"\n", args...)
	}
}

// error with printf syntax
func _errorf(format string, args ...interface{}) error {
	return fmt.Errorf(format, args...)
}

// panic with printf syntax
func _panicf(format string, args ...interface{}) {
	panic(fmt.Errorf(format, args...))
}

// dump named object only if debugging
func _dump(label string, args ...interface{}) {
	if debugMode {
		fmt.Printf("%s: %s", _caller(2), label)
		spew.Dump(args...)
	}
}

// dumps a given byte buffer to the given fixture filename
func _dump_fixture(filename string, buf []byte) {
	fmt.Printf("writing fixture %s...\n", filename)
	if err := ioutil.WriteFile("./fixtures/"+filename, buf, 0644); err != nil {
		panic(err)
	}
}

// reads a byte buffer from the given fixture filename
func _read_fixture(filename string) []byte {
	buf, err := ioutil.ReadFile("./fixtures/" + filename)
	if err != nil {
		panic(err)
	}
	return buf
}

// marshal a proto.Message to bytes
func _proto_marshal(obj proto.Message) []byte {
	buf, err := proto.Marshal(obj)
	if err != nil {
		panic(err)
	}
	return buf
}

// marshal an interface{} to JSON bytes
func _json_marshal(obj interface{}) []byte {
	buf, err := json.MarshalIndent(obj, "", "  ")
	if err != nil {
		panic(err)
	}
	return buf
}

// Returns the name of the calling function
func _caller(n int) string {
	if pc, _, _, ok := runtime.Caller(n); ok {
		fns := strings.Split(runtime.FuncForPC(pc).Name(), "/")
		return fns[len(fns)-1]
	}

	return "unknown"
}

// Compares string with prefix
func hasPrefix(s, prefix string) bool {
	return len(s) >= len(prefix) && s[:len(prefix)] == prefix
}

// Prints value after checking for nil
func saveReturnInt32(v *int32) int32 {
	if v == nil {
		return 0
	} else {
		return *v
	}
}

// Prints value after checking for nil
func saveReturnFloat32(v *float32, def interface{}) interface{} {
	if v == nil {
		return def
	} else {
		return *v
	}
}

func log2(n int) int {
	return int(math.Log(float64(n))/math.Log(2)) + 1
}
<commit_msg>Added debugLevel, testLevel and _debugfl function<commit_after>package manta

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/golang/protobuf/proto"
)

var debugMode, traceMode, fixturesMode bool
var debugLevel, testLevel uint // Test level refers to the inline-test level which runs additional checks on the data

func init() {
	if os.Getenv("DEBUG") != "" {
		debugMode = true
	}
	if os.Getenv("TRACE") != "" {
		traceMode = true
	}
	if os.Getenv("FIXTURES") != "" {
		fixturesMode = true
	}
}

var (
	_sprintf = fmt.Sprintf
	_sdump   = spew.Sdump
)

// Convert a string to an int32
func atoi32(s string) (int32, error) {
	n, err := strconv.ParseInt(s, 0, 32)
	if err != nil {
		return 0, err
	}
	return int32(n), nil
}

// printf with debug level
func _debugfl(level uint, format string, args ...interface{}) {
	if level <= debugLevel {
		args = append([]interface{}{_caller(2)}, args...)
		fmt.Printf("%s: "+format+"\n", args...)
	}
}

// printf only if debugging
func _debugf(format string, args ...interface{}) {
	if debugMode {
		args = append([]interface{}{_caller(2)}, args...)
		fmt.Printf("%s: "+format+"\n", args...)
	}
}

// printf only if tracing
func _tracef(format string, args ...interface{}) {
	if traceMode {
		args = append([]interface{}{_caller(2)}, args...)
		fmt.Printf("%s: "+format+"\n", args...)
	}
}

// error with printf syntax
func _errorf(format string, args ...interface{}) error {
	return fmt.Errorf(format, args...)
}

// panic with printf syntax
func _panicf(format string, args ...interface{}) {
	panic(fmt.Errorf(format, args...))
}

// dump named object only if debugging
func _dump(label string, args ...interface{}) {
	if debugMode {
		fmt.Printf("%s: %s", _caller(2), label)
		spew.Dump(args...)
	}
}

// dumps a given byte buffer to the given fixture filename
func _dump_fixture(filename string, buf []byte) {
	fmt.Printf("writing fixture %s...\n", filename)
	if err := ioutil.WriteFile("./fixtures/"+filename, buf, 0644); err != nil {
		panic(err)
	}
}

// reads a byte buffer from the given fixture filename
func _read_fixture(filename string) []byte {
	buf, err := ioutil.ReadFile("./fixtures/" + filename)
	if err != nil {
		panic(err)
	}
	return buf
}

// marshal a proto.Message to bytes
func _proto_marshal(obj proto.Message) []byte {
	buf, err := proto.Marshal(obj)
	if err != nil {
		panic(err)
	}
	return buf
}

// marshal an interface{} to JSON bytes
func _json_marshal(obj interface{}) []byte {
	buf, err := json.MarshalIndent(obj, "", "  ")
	if err != nil {
		panic(err)
	}
	return buf
}

// Returns the name of the calling function
func _caller(n int) string {
	if pc, _, _, ok := runtime.Caller(n); ok {
		fns := strings.Split(runtime.FuncForPC(pc).Name(), "/")
		return fns[len(fns)-1]
	}

	return "unknown"
}

// Compares string with prefix
func hasPrefix(s, prefix string) bool {
	return len(s) >= len(prefix) && s[:len(prefix)] == prefix
}

// Prints value after checking for nil
func saveReturnInt32(v *int32) int32 {
	if v == nil {
		return 0
	} else {
		return *v
	}
}

// Prints value after checking for nil
func saveReturnFloat32(v *float32, def interface{}) interface{} {
	if v == nil {
		return def
	} else {
		return *v
	}
}

func log2(n int) int {
	return int(math.Log(float64(n))/math.Log(2)) + 1
}
<|endoftext|>