<commit_before>package server

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	net2 "net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rancher/k3s/pkg/daemons/config"
	"github.com/rancher/k3s/pkg/daemons/control"
	"github.com/rancher/k3s/pkg/datadir"
	"github.com/rancher/k3s/pkg/deploy"
	"github.com/rancher/k3s/pkg/helm"
	"github.com/rancher/k3s/pkg/node"
	"github.com/rancher/k3s/pkg/rootlessports"
	"github.com/rancher/k3s/pkg/servicelb"
	"github.com/rancher/k3s/pkg/static"
	"github.com/rancher/k3s/pkg/tls"
	appsv1 "github.com/rancher/k3s/types/apis/apps/v1"
	batchv1 "github.com/rancher/k3s/types/apis/batch/v1"
	corev1 "github.com/rancher/k3s/types/apis/core/v1"
	v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1"
	rbacv1 "github.com/rancher/k3s/types/apis/rbac.authorization.k8s.io/v1"
	"github.com/rancher/norman"
	"github.com/rancher/norman/pkg/clientaccess"
	"github.com/rancher/norman/pkg/dynamiclistener"
	"github.com/rancher/norman/pkg/resolvehome"
	"github.com/rancher/norman/types"
	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/util/net"
)

func resolveDataDir(dataDir string) (string, error) {
	dataDir, err := datadir.Resolve(dataDir)
	return filepath.Join(dataDir, "server"), err
}

func StartServer(ctx context.Context, config *Config) (string, error) {

	if err := setupDataDirAndChdir(&config.ControlConfig); err != nil {
		return "", err
	}

	if err := setNoProxyEnv(&config.ControlConfig); err != nil {
		return "", err
	}

	if err := control.Server(ctx, &config.ControlConfig); err != nil {
		return "", errors.Wrap(err, "starting kubernetes")
	}

	certs, err := startNorman(ctx, config)
	if err != nil {
		return "", errors.Wrap(err, "starting tls server")
	}

	ip := net2.ParseIP(config.TLSConfig.BindAddress)
	if ip == nil {
		ip, err = net.ChooseHostInterface()
		if err != nil {
			ip = net2.ParseIP("127.0.0.1")
		}
	}
	printTokens(certs, ip.String(), &config.TLSConfig, &config.ControlConfig)

	writeKubeConfig(certs, &config.TLSConfig, config)

	return certs, nil
}

func startNorman(ctx context.Context, config *Config) (string, error) {
	var (
		err           error
		tlsServer     dynamiclistener.ServerInterface
		tlsConfig     = &config.TLSConfig
		controlConfig = &config.ControlConfig
	)

	tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) {
		if tlsServer == nil {
			return "", nil
		}
		return tlsServer.CACert()
	})

	normanConfig := &norman.Config{
		Name:       "k3s",
		KubeConfig: controlConfig.Runtime.KubeConfigSystem,
		Clients: []norman.ClientFactory{
			v1.Factory,
			appsv1.Factory,
			corev1.Factory,
			batchv1.Factory,
			rbacv1.Factory,
		},
		Schemas: []*types.Schemas{
			v1.Schemas,
		},
		CRDs: map[*types.APIVersion][]string{
			&v1.APIVersion: {
				v1.ListenerConfigGroupVersionKind.Kind,
				v1.AddonGroupVersionKind.Kind,
				v1.HelmChartGroupVersionKind.Kind,
			},
		},
		IgnoredKubeConfigEnv: true,
		GlobalSetup: func(ctx context.Context) (context.Context, error) {
			tlsServer, err = tls.NewServer(ctx, v1.ClientsFrom(ctx).ListenerConfig, *tlsConfig)
			return ctx, err
		},
		DisableLeaderElection: true,
		MasterControllers: []norman.ControllerRegister{
			node.Register,
			helm.Register,
			func(ctx context.Context) error {
				return servicelb.Register(ctx, norman.GetServer(ctx).K8sClient, !config.DisableServiceLB,
					config.Rootless)
			},
			func(ctx context.Context) error {
				dataDir := filepath.Join(controlConfig.DataDir, "static")
				return static.Stage(dataDir)
			},
			func(ctx context.Context) error {
				dataDir := filepath.Join(controlConfig.DataDir, "manifests")
				templateVars := map[string]string{"%{CLUSTER_DNS}%": controlConfig.ClusterDNS.String(), "%{CLUSTER_DOMAIN}%": controlConfig.ClusterDomain}
				if err := deploy.Stage(dataDir, templateVars, controlConfig.Skips); err != nil {
					return err
				}
				if err := deploy.WatchFiles(ctx, dataDir); err != nil {
					return err
				}
				return nil
			},
			func(ctx context.Context) error {
				if !config.DisableServiceLB && config.Rootless {
					return rootlessports.Register(ctx, config.TLSConfig.HTTPSPort)
				}
				return nil
			},
		},
	}

	if _, _, err = normanConfig.Build(ctx, nil); err != nil {
		return "", err
	}

	for {
		certs, err := tlsServer.CACert()
		if err != nil {
			logrus.Infof("waiting to generate CA certs")
			time.Sleep(time.Second)
			continue
		}
		return certs, nil
	}
}

func HomeKubeConfig(write, rootless bool) (string, error) {
	if write {
		if os.Getuid() == 0 && !rootless {
			return datadir.GlobalConfig, nil
		}
		return resolvehome.Resolve(datadir.HomeConfig)
	}

	if _, err := os.Stat(datadir.GlobalConfig); err == nil {
		return datadir.GlobalConfig, nil
	}

	return resolvehome.Resolve(datadir.HomeConfig)
}

func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) {
	var (
		nodeFile string
	)

	if advertiseIP == "" {
		advertiseIP = "localhost"
	}

	if len(config.Runtime.NodeToken) > 0 {
		p := filepath.Join(config.DataDir, "node-token")
		if err := writeToken(config.Runtime.NodeToken, p, certs); err == nil {
			logrus.Infof("Node token is available at %s", p)
			nodeFile = p
		}
	}

	if len(nodeFile) > 0 {
		printToken(tlsConfig.HTTPSPort, advertiseIP, "To join node to cluster:", "agent")
	}

}

func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config *Config) {
	clientToken := FormatToken(config.ControlConfig.Runtime.ClientToken, certs)
	ip := tlsConfig.BindAddress
	if ip == "" {
		ip = "localhost"
	}
	url := fmt.Sprintf("https://%s:%d", ip, tlsConfig.HTTPSPort)
	kubeConfig, err := HomeKubeConfig(true, config.Rootless)
	def := true
	if err != nil {
		kubeConfig = filepath.Join(config.ControlConfig.DataDir, "kubeconfig-k3s.yaml")
		def = false
	}

	if config.ControlConfig.KubeConfigOutput != "" {
		kubeConfig = config.ControlConfig.KubeConfigOutput
	}

	if err = clientaccess.AgentAccessInfoToKubeConfig(kubeConfig, url, clientToken); err != nil {
		logrus.Errorf("Failed to generate kubeconfig: %v", err)
	}

	if config.ControlConfig.KubeConfigMode != "" {
		mode, err := strconv.ParseInt(config.ControlConfig.KubeConfigMode, 8, 0)
		if err == nil {
			os.Chmod(kubeConfig, os.FileMode(mode))
		} else {
			logrus.Errorf("failed to set %s to mode %s: %v", kubeConfig, os.FileMode(mode), err)
		}
	} else {
		os.Chmod(kubeConfig, os.FileMode(0644))
	}

	logrus.Infof("Wrote kubeconfig %s", kubeConfig)
	if def {
		logrus.Infof("Run: %s kubectl", filepath.Base(os.Args[0]))
	}
}

func setupDataDirAndChdir(config *config.Control) error {
	var (
		err error
	)

	config.DataDir, err = resolveDataDir(config.DataDir)
	if err != nil {
		return err
	}

	dataDir := config.DataDir

	if err := os.MkdirAll(dataDir, 0700); err != nil {
		return errors.Wrapf(err, "can not mkdir %s", dataDir)
	}

	if err := os.Chdir(dataDir); err != nil {
		return errors.Wrapf(err, "can not chdir %s", dataDir)
	}

	return nil
}

func printToken(httpsPort int, advertiseIP, prefix, cmd string) {
	ip := advertiseIP
	if ip == "" {
		hostIP, err := net.ChooseHostInterface()
		if err != nil {
			logrus.Error(err)
		}
		ip = hostIP.String()
	}

	logrus.Infof("%s k3s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, cmd, ip, httpsPort)
}

func FormatToken(token string, certs string) string {
	if len(token) == 0 {
		return token
	}

	prefix := "K10"
	if len(certs) > 0 {
		digest := sha256.Sum256([]byte(certs))
		prefix = "K10" + hex.EncodeToString(digest[:]) + "::"
	}

	return prefix + token
}

func writeToken(token, file, certs string) error {
	if len(token) == 0 {
		return nil
	}

	token = FormatToken(token, certs)
	return ioutil.WriteFile(file, []byte(token+"\n"), 0600)
}

func setNoProxyEnv(config *config.Control) error {
	envList := strings.Join([]string{
		os.Getenv("NO_PROXY"),
		config.ClusterIPRange.String(),
		config.ServiceIPRange.String(),
	}, ",")
	return os.Setenv("NO_PROXY", envList)
}
<commit_msg>Create symlink for kubeconfig when --write-kubeconfig is selected<commit_after>package server

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	net2 "net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rancher/k3s/pkg/daemons/config"
	"github.com/rancher/k3s/pkg/daemons/control"
	"github.com/rancher/k3s/pkg/datadir"
	"github.com/rancher/k3s/pkg/deploy"
	"github.com/rancher/k3s/pkg/helm"
	"github.com/rancher/k3s/pkg/node"
	"github.com/rancher/k3s/pkg/rootlessports"
	"github.com/rancher/k3s/pkg/servicelb"
	"github.com/rancher/k3s/pkg/static"
	"github.com/rancher/k3s/pkg/tls"
	appsv1 "github.com/rancher/k3s/types/apis/apps/v1"
	batchv1 "github.com/rancher/k3s/types/apis/batch/v1"
	corev1 "github.com/rancher/k3s/types/apis/core/v1"
	v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1"
	rbacv1 "github.com/rancher/k3s/types/apis/rbac.authorization.k8s.io/v1"
	"github.com/rancher/norman"
	"github.com/rancher/norman/pkg/clientaccess"
	"github.com/rancher/norman/pkg/dynamiclistener"
	"github.com/rancher/norman/pkg/resolvehome"
	"github.com/rancher/norman/types"
	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/util/net"
)

func resolveDataDir(dataDir string) (string, error) {
	dataDir, err := datadir.Resolve(dataDir)
	return filepath.Join(dataDir, "server"), err
}

func StartServer(ctx context.Context, config *Config) (string, error) {

	if err := setupDataDirAndChdir(&config.ControlConfig); err != nil {
		return "", err
	}

	if err := setNoProxyEnv(&config.ControlConfig); err != nil {
		return "", err
	}

	if err := control.Server(ctx, &config.ControlConfig); err != nil {
		return "", errors.Wrap(err, "starting kubernetes")
	}

	certs, err := startNorman(ctx, config)
	if err != nil {
		return "", errors.Wrap(err, "starting tls server")
	}

	ip := net2.ParseIP(config.TLSConfig.BindAddress)
	if ip == nil {
		ip, err = net.ChooseHostInterface()
		if err != nil {
			ip = net2.ParseIP("127.0.0.1")
		}
	}
	printTokens(certs, ip.String(), &config.TLSConfig, &config.ControlConfig)

	writeKubeConfig(certs, &config.TLSConfig, config)

	return certs, nil
}

func startNorman(ctx context.Context, config *Config) (string, error) {
	var (
		err           error
		tlsServer     dynamiclistener.ServerInterface
		tlsConfig     = &config.TLSConfig
		controlConfig = &config.ControlConfig
	)

	tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) {
		if tlsServer == nil {
			return "", nil
		}
		return tlsServer.CACert()
	})

	normanConfig := &norman.Config{
		Name:       "k3s",
		KubeConfig: controlConfig.Runtime.KubeConfigSystem,
		Clients: []norman.ClientFactory{
			v1.Factory,
			appsv1.Factory,
			corev1.Factory,
			batchv1.Factory,
			rbacv1.Factory,
		},
		Schemas: []*types.Schemas{
			v1.Schemas,
		},
		CRDs: map[*types.APIVersion][]string{
			&v1.APIVersion: {
				v1.ListenerConfigGroupVersionKind.Kind,
				v1.AddonGroupVersionKind.Kind,
				v1.HelmChartGroupVersionKind.Kind,
			},
		},
		IgnoredKubeConfigEnv: true,
		GlobalSetup: func(ctx context.Context) (context.Context, error) {
			tlsServer, err = tls.NewServer(ctx, v1.ClientsFrom(ctx).ListenerConfig, *tlsConfig)
			return ctx, err
		},
		DisableLeaderElection: true,
		MasterControllers: []norman.ControllerRegister{
			node.Register,
			helm.Register,
			func(ctx context.Context) error {
				return servicelb.Register(ctx, norman.GetServer(ctx).K8sClient, !config.DisableServiceLB,
					config.Rootless)
			},
			func(ctx context.Context) error {
				dataDir := filepath.Join(controlConfig.DataDir, "static")
				return static.Stage(dataDir)
			},
			func(ctx context.Context) error {
				dataDir := filepath.Join(controlConfig.DataDir, "manifests")
				templateVars := map[string]string{"%{CLUSTER_DNS}%": controlConfig.ClusterDNS.String(), "%{CLUSTER_DOMAIN}%": controlConfig.ClusterDomain}
				if err := deploy.Stage(dataDir, templateVars, controlConfig.Skips); err != nil {
					return err
				}
				if err := deploy.WatchFiles(ctx, dataDir); err != nil {
					return err
				}
				return nil
			},
			func(ctx context.Context) error {
				if !config.DisableServiceLB && config.Rootless {
					return rootlessports.Register(ctx, config.TLSConfig.HTTPSPort)
				}
				return nil
			},
		},
	}

	if _, _, err = normanConfig.Build(ctx, nil); err != nil {
		return "", err
	}

	for {
		certs, err := tlsServer.CACert()
		if err != nil {
			logrus.Infof("waiting to generate CA certs")
			time.Sleep(time.Second)
			continue
		}
		return certs, nil
	}
}

func HomeKubeConfig(write, rootless bool) (string, error) {
	if write {
		if os.Getuid() == 0 && !rootless {
			return datadir.GlobalConfig, nil
		}
		return resolvehome.Resolve(datadir.HomeConfig)
	}

	if _, err := os.Stat(datadir.GlobalConfig); err == nil {
		return datadir.GlobalConfig, nil
	}

	return resolvehome.Resolve(datadir.HomeConfig)
}

func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) {
	var (
		nodeFile string
	)

	if advertiseIP == "" {
		advertiseIP = "localhost"
	}

	if len(config.Runtime.NodeToken) > 0 {
		p := filepath.Join(config.DataDir, "node-token")
		if err := writeToken(config.Runtime.NodeToken, p, certs); err == nil {
			logrus.Infof("Node token is available at %s", p)
			nodeFile = p
		}
	}

	if len(nodeFile) > 0 {
		printToken(tlsConfig.HTTPSPort, advertiseIP, "To join node to cluster:", "agent")
	}

}

func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config *Config) {
	clientToken := FormatToken(config.ControlConfig.Runtime.ClientToken, certs)
	ip := tlsConfig.BindAddress
	if ip == "" {
		ip = "localhost"
	}
	url := fmt.Sprintf("https://%s:%d", ip, tlsConfig.HTTPSPort)
	kubeConfig, err := HomeKubeConfig(true, config.Rootless)
	def := true
	if err != nil {
		kubeConfig = filepath.Join(config.ControlConfig.DataDir, "kubeconfig-k3s.yaml")
		def = false
	}
	kubeConfigSymlink := kubeConfig
	if config.ControlConfig.KubeConfigOutput != "" {
		kubeConfig = config.ControlConfig.KubeConfigOutput
	}

	if isSymlink(kubeConfigSymlink) {
		if err := os.Remove(kubeConfigSymlink); err != nil {
			logrus.Errorf("failed to remove kubeconfig symlink")
		}
	}

	if err = clientaccess.AgentAccessInfoToKubeConfig(kubeConfig, url, clientToken); err != nil {
		logrus.Errorf("Failed to generate kubeconfig: %v", err)
	}

	if config.ControlConfig.KubeConfigMode != "" {
		mode, err := strconv.ParseInt(config.ControlConfig.KubeConfigMode, 8, 0)
		if err == nil {
			os.Chmod(kubeConfig, os.FileMode(mode))
		} else {
			logrus.Errorf("failed to set %s to mode %s: %v", kubeConfig, os.FileMode(mode), err)
		}
	} else {
		os.Chmod(kubeConfig, os.FileMode(0644))
	}

	if kubeConfigSymlink != kubeConfig {
		if err := writeConfigSymlink(kubeConfig, kubeConfigSymlink); err != nil {
			logrus.Errorf("failed to write kubeconfig symlink: %v", err)
		}
	}

	logrus.Infof("Wrote kubeconfig %s", kubeConfig)
	if def {
		logrus.Infof("Run: %s kubectl", filepath.Base(os.Args[0]))
	}
}

func setupDataDirAndChdir(config *config.Control) error {
	var (
		err error
	)

	config.DataDir, err = resolveDataDir(config.DataDir)
	if err != nil {
		return err
	}

	dataDir := config.DataDir

	if err := os.MkdirAll(dataDir, 0700); err != nil {
		return errors.Wrapf(err, "can not mkdir %s", dataDir)
	}

	if err := os.Chdir(dataDir); err != nil {
		return errors.Wrapf(err, "can not chdir %s", dataDir)
	}

	return nil
}

func printToken(httpsPort int, advertiseIP, prefix, cmd string) {
	ip := advertiseIP
	if ip == "" {
		hostIP, err := net.ChooseHostInterface()
		if err != nil {
			logrus.Error(err)
		}
		ip = hostIP.String()
	}

	logrus.Infof("%s k3s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, cmd, ip, httpsPort)
}

func FormatToken(token string, certs string) string {
	if len(token) == 0 {
		return token
	}

	prefix := "K10"
	if len(certs) > 0 {
		digest := sha256.Sum256([]byte(certs))
		prefix = "K10" + hex.EncodeToString(digest[:]) + "::"
	}

	return prefix + token
}

func writeToken(token, file, certs string) error {
	if len(token) == 0 {
		return nil
	}

	token = FormatToken(token, certs)
	return ioutil.WriteFile(file, []byte(token+"\n"), 0600)
}

func setNoProxyEnv(config *config.Control) error {
	envList := strings.Join([]string{
		os.Getenv("NO_PROXY"),
		config.ClusterIPRange.String(),
		config.ServiceIPRange.String(),
	}, ",")
	return os.Setenv("NO_PROXY", envList)
}

func writeConfigSymlink(kubeconfig, kubeconfigSymlink string) error {
	if err := os.Remove(kubeconfigSymlink); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to remove %s file: %v", kubeconfigSymlink, err)
	}
	if err := os.MkdirAll(filepath.Dir(kubeconfigSymlink), 0755); err != nil {
		return fmt.Errorf("failed to create path for symlink: %v", err)
	}
	if err := os.Symlink(kubeconfig, kubeconfigSymlink); err != nil {
		return fmt.Errorf("failed to create symlink: %v", err)
	}
	return nil
}

func isSymlink(config string) bool {
	if fi, err := os.Lstat(config); err == nil && (fi.Mode()&os.ModeSymlink == os.ModeSymlink) {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package stream

import (
	"strings"

	"nimona.io/pkg/crypto"
	"nimona.io/pkg/object"
)

func GetAllowsKeysFromPolicies(os ...object.Object) []crypto.PublicKey {
	// TODO this currently only accepts allow actions
	pks := []crypto.PublicKey{}
	for _, o := range os {
		p := o.GetPolicy()
		for _, a := range p.Actions {
			switch strings.ToLower(a) {
			case "allow":
				for _, s := range p.Subjects {
					pks = append(pks, crypto.PublicKey(s))
				}
			}
		}
	}
	return pks
}

func GetStreamLeaves(os []object.Object) []object.Object {
	hm := map[string]bool{} // map[hash]isParent
	om := map[string]object.Object{}
	for _, o := range os {
		h := object.NewHash(o).String()
		if _, ok := hm[h]; !ok {
			hm[h] = false
		}
		for _, p := range o.GetParents() {
			hm[p.String()] = true
		}
		om[h] = o
	}

	os = []object.Object{}
	for h, isParent := range hm {
		if !isParent {
			os = append(os, om[h])
		}
	}

	return os
}
<commit_msg>fix: assume owners of all stream objects are allowed to receive updates<commit_after>package stream

import (
	"strings"

	"nimona.io/pkg/crypto"
	"nimona.io/pkg/object"
)

func GetAllowsKeysFromPolicies(os ...object.Object) []crypto.PublicKey {
	// TODO this currently only accepts allow actions
	pkm := map[crypto.PublicKey]struct{}{}
	for _, o := range os {
		for _, s := range o.GetOwners() {
			pkm[s] = struct{}{}
		}
		p := o.GetPolicy()
		for _, a := range p.Actions {
			switch strings.ToLower(a) {
			case "allow":
				for _, s := range p.Subjects {
					pkm[crypto.PublicKey(s)] = struct{}{}
				}
			}
		}
	}
	pks := []crypto.PublicKey{}
	for k := range pkm {
		pks = append(pks, k)
	}
	return pks
}

func GetStreamLeaves(os []object.Object) []object.Object {
	hm := map[string]bool{} // map[hash]isParent
	om := map[string]object.Object{}
	for _, o := range os {
		h := object.NewHash(o).String()
		if _, ok := hm[h]; !ok {
			hm[h] = false
		}
		for _, p := range o.GetParents() {
			hm[p.String()] = true
		}
		om[h] = o
	}

	os = []object.Object{}
	for h, isParent := range hm {
		if !isParent {
			os = append(os, om[h])
		}
	}

	return os
}
<|endoftext|>


<commit_before>// Name: action.go
// Desc:
//   This portion of the package manager is meant to go through each
//   PkgOperation and perform them.
package pkginstall

import (
	"fmt"
	"github.com/crockeo/dotfile-manager/pkgfile"
)

//MoveFile
//RunScript
//InstallPackage

// Performing a MoveFile operation.
func performMoveFile(moveFile pkgfile.MoveFile) error {
	return nil
}

// Performing a RunScript operation.
func performRunScript(runScript pkgfile.RunScript) error {
	return nil
}

// Performing an InstallPackage operation.
func performInstallPackage(installPackage pkgfile.InstallPackage) error {
	return nil
}

// Performing a single package operation.
func PerformPackageOperation(pkgOperation pkgfile.PkgOperation) error {
	t := pkgOperation.GetType()

	switch t {
	case pkgfile.MoveFileType:
		v, succ := pkgOperation.(pkgfile.MoveFile)
		if !succ {
			fmt.Println(":(")
		}

		return performMoveFile(v)
	case pkgfile.RunScriptType:
		v, succ := pkgOperation.(pkgfile.RunScript)
		if !succ {
			fmt.Println(":(")
		}

		return performRunScript(v)
	case pkgfile.InstallPackageType:
		v, succ := pkgOperation.(pkgfile.InstallPackage)
		if !succ {
			fmt.Println(":(")
		}

		return performInstallPackage(v)
	}

	return nil
}

// Performing a whole set of package operations (as described in a PkgFile).
func PerformPackageOperations(pkgFile pkgfile.PkgFile) error {
	var err error
	for _, v := range pkgFile.Ops {
		err = PerformPackageOperation(v)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Returned a set of errrors.<commit_after>// Name: action.go
// Desc:
//   This portion of the package manager is meant to go through each
//   PkgOperation and perform them.
package pkginstall

import (
	"errors"
	"github.com/crockeo/dotfile-manager/pkgfile"
)

var (
	MalformedPkgOperationError error = errors.New("Malformed package operation.")
)

//MoveFile
//RunScript
//InstallPackage

// Performing a MoveFile operation.
func performMoveFile(moveFile pkgfile.MoveFile) error {
	return nil
}

// Performing a RunScript operation.
func performRunScript(runScript pkgfile.RunScript) error {
	return nil
}

// Performing an InstallPackage operation.
func performInstallPackage(installPackage pkgfile.InstallPackage) error {
	return nil
}

// Performing a single package operation.
func PerformPackageOperation(pkgOperation pkgfile.PkgOperation) error {
	t := pkgOperation.GetType()

	switch t {
	case pkgfile.MoveFileType:
		v, succ := pkgOperation.(pkgfile.MoveFile)
		if !succ {
			return MalformedPkgOperationError
		}

		return performMoveFile(v)
	case pkgfile.RunScriptType:
		v, succ := pkgOperation.(pkgfile.RunScript)
		if !succ {
			return MalformedPkgOperationError
		}

		return performRunScript(v)
	case pkgfile.InstallPackageType:
		v, succ := pkgOperation.(pkgfile.InstallPackage)
		if !succ {
			return MalformedPkgOperationError
		}

		return performInstallPackage(v)
	}

	return MalformedPkgOperationError
}

// Performing a whole set of package operations (as described in a PkgFile).
func PerformPackageOperations(pkgFile pkgfile.PkgFile) error {
	var err error
	for _, v := range pkgFile.Ops {
		err = PerformPackageOperation(v)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before><commit_msg>crawler: remove todo note (make host specific)<commit_after><|endoftext|>


<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package platform

import (
	"fmt"
	"io"
	"os"
	"time"

	"path/filepath"
	"sync"

	"github.com/coreos/mantle/Godeps/_workspace/src/golang.org/x/crypto/ssh"
	"github.com/coreos/mantle/util"
)

const (
	sshRetries = 7
	sshTimeout = 1 * time.Second
)

type Machine interface {
	ID() string
	IP() string
	PrivateIP() string
	SSHSession() (*ssh.Session, error)
	SSH(cmd string) ([]byte, error)
	Destroy() error
	StartJournal() error
}

type Cluster interface {
	NewCommand(name string, arg ...string) util.Cmd
	NewMachine(config string) (Machine, error)
	Machines() []Machine
	// Points to an embedded etcd for QEMU, not sure what this
	// is going to look like for other platforms yet.
	EtcdEndpoint() string
	GetDiscoveryURL(size int) (string, error)
	Destroy() error
}

// TestCluster embedds a Cluster to provide platform independant helper
// methods.
type TestCluster struct {
	Name        string
	NativeFuncs []string
	Options     map[string]string
	Cluster
}

// run a registered NativeFunc on a remote machine
func (t *TestCluster) RunNative(funcName string, m Machine) error {
	// scp and execute kolet on remote machine
	ssh, err := m.SSHSession()
	if err != nil {
		return fmt.Errorf("kolet SSH session: %v", err)
	}
	b, err := ssh.CombinedOutput(fmt.Sprintf("./kolet run %q %q", t.Name, funcName))
	if err != nil {
		return fmt.Errorf("%s", b) // return function std output, not the exit status
	}
	return nil
}

func (t *TestCluster) ListNativeFunctions() []string {
	return t.NativeFuncs
}

// DropFile places file from localPath to ~/ on every machine in cluster
func (t *TestCluster) DropFile(localPath string) error {
	in, err := os.Open(localPath)
	if err != nil {
		return err
	}
	defer in.Close()

	for _, m := range t.Machines() {
		if _, err := in.Seek(0, 0); err != nil {
			return err
		}
		if err := transfer(in, m, localPath); err != nil {
			return err
		}
	}
	return nil
}

func transfer(in io.Reader, m Machine, localPath string) error {
	session, err := m.SSHSession()
	if err != nil {
		return fmt.Errorf("Error establishing ssh session: %v", err)
	}
	defer session.Close()

	session.Stdin = in

	// write file to fs from stdin
	_, filename := filepath.Split(localPath)
	err = session.Run(fmt.Sprintf("install -m 0755 /dev/stdin ./%s", filename))
	if err != nil {
		return err
	}

	return nil
}

func NewMachines(c Cluster, userdatas []string) ([]Machine, error) {
	var wg sync.WaitGroup

	n := len(userdatas)

	if n <= 0 {
		return nil, fmt.Errorf("must provide one or more userdatas")
	}

	mchan := make(chan Machine, n)
	errchan := make(chan error, n)

	for i := 0; i < n; i++ {
		ud := userdatas[i]
		wg.Add(1)
		go func() {
			defer wg.Done()
			m, err := c.NewMachine(ud)
			if err != nil {
				errchan <- err
			}
			if m != nil {
				mchan <- m
			}
		}()
	}

	wg.Wait()
	close(mchan)
	close(errchan)

	machs := []Machine{}

	for m := range mchan {
		machs = append(machs, m)
	}

	if firsterr, ok := <-errchan; ok {
		for _, m := range machs {
			m.Destroy()
		}
		return nil, firsterr
	}

	return machs, nil
}
<commit_msg>platform: increase ssh timeout<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package platform

import (
	"fmt"
	"io"
	"os"
	"time"

	"path/filepath"
	"sync"

	"github.com/coreos/mantle/Godeps/_workspace/src/golang.org/x/crypto/ssh"
	"github.com/coreos/mantle/util"
)

const (
	sshRetries = 10
	sshTimeout = 2 * time.Second
)

type Machine interface {
	ID() string
	IP() string
	PrivateIP() string
	SSHSession() (*ssh.Session, error)
	SSH(cmd string) ([]byte, error)
	Destroy() error
	StartJournal() error
}

type Cluster interface {
	NewCommand(name string, arg ...string) util.Cmd
	NewMachine(config string) (Machine, error)
	Machines() []Machine
	// Points to an embedded etcd for QEMU, not sure what this
	// is going to look like for other platforms yet.
	EtcdEndpoint() string
	GetDiscoveryURL(size int) (string, error)
	Destroy() error
}

// TestCluster embedds a Cluster to provide platform independant helper
// methods.
type TestCluster struct {
	Name        string
	NativeFuncs []string
	Options     map[string]string
	Cluster
}

// run a registered NativeFunc on a remote machine
func (t *TestCluster) RunNative(funcName string, m Machine) error {
	// scp and execute kolet on remote machine
	ssh, err := m.SSHSession()
	if err != nil {
		return fmt.Errorf("kolet SSH session: %v", err)
	}
	b, err := ssh.CombinedOutput(fmt.Sprintf("./kolet run %q %q", t.Name, funcName))
	if err != nil {
		return fmt.Errorf("%s", b) // return function std output, not the exit status
	}
	return nil
}

func (t *TestCluster) ListNativeFunctions() []string {
	return t.NativeFuncs
}

// DropFile places file from localPath to ~/ on every machine in cluster
func (t *TestCluster) DropFile(localPath string) error {
	in, err := os.Open(localPath)
	if err != nil {
		return err
	}
	defer in.Close()

	for _, m := range t.Machines() {
		if _, err := in.Seek(0, 0); err != nil {
			return err
		}
		if err := transfer(in, m, localPath); err != nil {
			return err
		}
	}
	return nil
}

func transfer(in io.Reader, m Machine, localPath string) error {
	session, err := m.SSHSession()
	if err != nil {
		return fmt.Errorf("Error establishing ssh session: %v", err)
	}
	defer session.Close()

	session.Stdin = in

	// write file to fs from stdin
	_, filename := filepath.Split(localPath)
	err = session.Run(fmt.Sprintf("install -m 0755 /dev/stdin ./%s", filename))
	if err != nil {
		return err
	}

	return nil
}

func NewMachines(c Cluster, userdatas []string) ([]Machine, error) {
	var wg sync.WaitGroup

	n := len(userdatas)

	if n <= 0 {
		return nil, fmt.Errorf("must provide one or more userdatas")
	}

	mchan := make(chan Machine, n)
	errchan := make(chan error, n)

	for i := 0; i < n; i++ {
		ud := userdatas[i]
		wg.Add(1)
		go func() {
			defer wg.Done()
			m, err := c.NewMachine(ud)
			if err != nil {
				errchan <- err
			}
			if m != nil {
				mchan <- m
			}
		}()
	}

	wg.Wait()
	close(mchan)
	close(errchan)

	machs := []Machine{}

	for m := range mchan {
		machs = append(machs, m)
	}

	if firsterr, ok := <-errchan; ok {
		for _, m := range machs {
			m.Destroy()
		}
		return nil, firsterr
	}

	return machs, nil
}
<|endoftext|>


<commit_before><commit_msg>Release binding value to struct pointor with reflect package<commit_after><|endoftext|>


<commit_before>package predicates

import "encoding/json"

type equals struct {
	req Request
}

func (p equals) Type() string {
	return "equals"
}

func (p equals) MarshalJSON() ([]byte, error) {
	requestBytes, _ := json.Marshal(p.req)

	requestJSON := string(requestBytes)
	equalsJSON := " { \"equals\" : " + equalsJSON + "}"

	return []byte(equalsJSON), nil
}

type EqualsBuilder struct {
	equals equals
}

func Equals() *EqualsBuilder {
	return &EqualsBuilder{equals: equals{req: Request{}}}
}

func (builder *EqualsBuilder) Path(path string) *EqualsBuilder {
	builder.equals.req.Path = path
	return builder
}

func (builder *EqualsBuilder) Method(method string) *EqualsBuilder {
	builder.equals.req.Method = method
	return builder
}

func (builder *EqualsBuilder) Header(header string, value string) *EqualsBuilder {
	if builder.equals.req.Headers == nil {
		builder.equals.req.Headers = map[string]string{}
	}
	builder.equals.req.Headers[header] = value
	return builder
}

func (builder *EqualsBuilder) Query(param string, value string) *EqualsBuilder {
	if builder.equals.req.QueryParams == nil {
		builder.equals.req.QueryParams = map[string]string{}
	}

	builder.equals.req.QueryParams[param] = value
	return builder
}

func (builder *EqualsBuilder) Body(body string) *EqualsBuilder {
	builder.equals.req.Body = body
	return builder
}

func (builder *EqualsBuilder) Build() equals {
	return builder.equals
}
<commit_msg>#MPL-816 ed fixes undefined equalsJSON error<commit_after>package predicates

import "encoding/json"

type equals struct {
	req Request
}

func (p equals) Type() string {
	return "equals"
}

func (p equals) MarshalJSON() ([]byte, error) {
	requestBytes, _ := json.Marshal(p.req)

	requestJSON := string(requestBytes)
	equalsJSON := " { \"equals\" : " + requestJSON + "}"

	return []byte(equalsJSON), nil
}

type EqualsBuilder struct {
	equals equals
}

func Equals() *EqualsBuilder {
	return &EqualsBuilder{equals: equals{req: Request{}}}
}

func (builder *EqualsBuilder) Path(path string) *EqualsBuilder {
	builder.equals.req.Path = path
	return builder
}

func (builder *EqualsBuilder) Method(method string) *EqualsBuilder {
	builder.equals.req.Method = method
	return builder
}

func (builder *EqualsBuilder) Header(header string, value string) *EqualsBuilder {
	if builder.equals.req.Headers == nil {
		builder.equals.req.Headers = map[string]string{}
	}
	builder.equals.req.Headers[header] = value
	return builder
}

func (builder *EqualsBuilder) Query(param string, value string) *EqualsBuilder {
	if builder.equals.req.QueryParams == nil {
		builder.equals.req.QueryParams = map[string]string{}
	}

	builder.equals.req.QueryParams[param] = value
	return builder
}

func (builder *EqualsBuilder) Body(body string) *EqualsBuilder {
	builder.equals.req.Body = body
	return builder
}

func (builder *EqualsBuilder) Build() equals {
	return builder.equals
}
<|endoftext|>


<commit_before>/*
Package supervisor provides supervisor trees for Go applications.

This package is a clean reimplementation of github.com/thejerf/suture, aiming
to be more Go idiomatic, thus less Erlang-like.

It is built on top of context package, with all of its advantages, namely the
possibility trickle down context-related values and cancellation signals.

TheJerf's blog post about Suture is a very good and helpful read to understand
how this package has been implemented.

http://www.jerf.org/iri/post/2930
*/
package supervisor // import "cirello.io/supervisor"

import (
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"golang.org/x/net/context"
)

// Service is the public interface expected by a Supervisor.
//
// This will be internally named after the result of fmt.Stringer, if available.
// Otherwise it will going to use an internal representation for the service
// name.
type Service interface {
	// Serve is called by a Supervisor to start the service. It expects the
	// service to honor the passed context and its lifetime. Observe
	// <-ctx.Done() and ctx.Err(). If the service is stopped by anything
	// but the Supervisor, it will get started again. Be careful with shared
	// state among restarts.
	Serve(ctx context.Context)
}

// Supervisor is the basic datastructure responsible for offering a supervisor
// tree. It implements Service, therefore it can be nested if necessary. When
// passing the Supervisor around, remind to do it as reference (&supervisor).
type Supervisor struct {
	// Name for this supervisor tree, used for logging.
	Name string

	// FailureDecay is the timespan on which the current failure count will
	// be halved.
	FailureDecay float64

	// FailureThreshold is the maximum accepted number of failures, after
	// decay adjustment, that shall trigger the back-off wait.
	FailureThreshold float64

	// Backoff is the wait duration when hit threshold.
	Backoff time.Duration

	// Log is a replaceable function used for overall logging
	Log func(interface{})

	ready sync.Once

	addedService    chan struct{}
	startedServices chan struct{}

	runningServices sync.WaitGroup

	servicesMu sync.Mutex
	services   map[string]Service

	cancelationsMu sync.Mutex
	cancelations   map[string]context.CancelFunc

	backoffMu sync.Mutex
	backoff   map[string]*backoff

	singleflight singleflight
}

func (s *Supervisor) String() string {
	return s.Name
}

func (s *Supervisor) prepare() {
	s.ready.Do(func() {
		if s.Name == "" {
			s.Name = "supervisor"
		}
		s.addedService = make(chan struct{}, 1)
		s.backoff = make(map[string]*backoff)
		s.cancelations = make(map[string]context.CancelFunc)
		s.services = make(map[string]Service)
		s.startedServices = make(chan struct{}, 1)

		if s.Log == nil {
			s.Log = func(msg interface{}) {
				log.Println(s.Name, ":", msg)
			}
		}
		if s.FailureDecay == 0 {
			s.FailureDecay = 30
		}
		if s.FailureThreshold == 0 {
			s.FailureThreshold = 5
		}
		if s.Backoff == 0 {
			s.Backoff = 15 * time.Second
		}
	})
}

// Add inserts into the Supervisor tree a new service. If the Supervisor is
// already started, it will start it automatically.
func (s *Supervisor) Add(service Service) {
	s.prepare()

	name := fmt.Sprintf("%s", service)

	s.servicesMu.Lock()
	s.backoffMu.Lock()
	s.backoff[name] = &backoff{}
	s.services[name] = service
	s.backoffMu.Unlock()
	s.servicesMu.Unlock()

	select {
	case s.addedService <- struct{}{}:
	default:
	}
}

// Remove stops the service in the Supervisor tree and remove from it.
func (s *Supervisor) Remove(name string) {
	s.prepare()

	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()
	if _, ok := s.services[name]; !ok {
		return
	}

	s.cancelationsMu.Lock()
	defer s.cancelationsMu.Unlock()
	if c, ok := s.cancelations[name]; ok {
		delete(s.cancelations, name)
		c()
	}
}

// Serve starts the Supervisor tree. It can be started only once at a time. If
// stopped (canceled), it can be restarted. It will discard context of
// successive calls to supervisor when still running.
func (s *Supervisor) Serve(ctx context.Context) {
	s.prepare()

	s.singleflight.do(func() {
		s.serve(ctx)
	})
}

// Services return a list of services
func (s *Supervisor) Services() map[string]Service {
	svclist := make(map[string]Service)
	s.servicesMu.Lock()
	for k, v := range s.services {
		svclist[k] = v
	}
	s.servicesMu.Unlock()
	return svclist
}

// Cancelations return a list of services names and their cancellation calls
func (s *Supervisor) Cancelations() map[string]context.CancelFunc {
	svclist := make(map[string]context.CancelFunc)
	s.cancelationsMu.Lock()
	for k, v := range s.cancelations {
		svclist[k] = v
	}
	s.cancelationsMu.Unlock()
	return svclist
}

func (s *Supervisor) serve(ctx context.Context) {
	select {
	case s.addedService <- struct{}{}:
	default:
	}
	go func(ctx context.Context) {
		for {
			select {
			case <-s.addedService:
				s.startServices(ctx)
				select {
				case s.startedServices <- struct{}{}:
				default:
				}

			case <-ctx.Done():
				return
			}
		}
	}(ctx)
	<-ctx.Done()

	s.runningServices.Wait()

	s.cancelationsMu.Lock()
	s.cancelations = make(map[string]context.CancelFunc)
	s.cancelationsMu.Unlock()
	return
}

func (s *Supervisor) startServices(ctx context.Context) {
	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()

	var wg sync.WaitGroup

	for name, svc := range s.services {
		s.cancelationsMu.Lock()
		_, ok := s.cancelations[name]
		if ok {
			s.cancelationsMu.Unlock()
			continue
		}

		wg.Add(1)
		go func(name string, svc Service) {
			s.runningServices.Add(1)
			wg.Done()
			for {
				retry := func() (retry bool) {
					select {
					case <-ctx.Done():
						return false
					default:
					}

					defer func() {
						if r := recover(); r != nil {
							s.Log(fmt.Sprintf("trapped panic: %v", r))
							retry = true
						}
					}()

					c, cancel := context.WithCancel(ctx)
					s.cancelationsMu.Lock()
					s.cancelations[name] = cancel
					s.cancelationsMu.Unlock()
					svc.Serve(c)

					select {
					case <-ctx.Done():
						return false
					default:
						return true
					}
				}()
				if retry {
					s.Log(fmt.Sprintf("restarting %s", name))
					s.backoffMu.Lock()
					b := s.backoff[name]
					s.backoffMu.Unlock()
					b.wait(s.FailureDecay, s.FailureThreshold, s.Backoff, func(msg interface{}) {
						s.Log(fmt.Sprintf("backoff %s: %v", name, msg))
					})
					continue
				}
				break
			}
			s.runningServices.Done()
		}(name, svc)
		s.cancelationsMu.Unlock()
	}

	wg.Wait()
}

type backoff struct {
	lastfail time.Time
	failures float64
}

func (b *backoff) wait(failureDecay float64, threshold float64, backoffDur time.Duration, log func(str interface{})) {
	if b.lastfail.IsZero() {
		b.lastfail = time.Now()
		b.failures = 1.0
	} else {
		b.failures++
		intervals := time.Since(b.lastfail).Seconds() / failureDecay
		b.failures = b.failures*math.Pow(.5, intervals) + 1
	}

	if b.failures > threshold {
		log(backoffDur)
		time.Sleep(backoffDur)
	}
}

// Based from groupcache's singleflight
type singleflight struct {
	mu     sync.Mutex
	flying bool
}

func (g *singleflight) do(fn func()) {
	g.mu.Lock()
	if g.flying {
		g.mu.Unlock()
		return
	}

	g.flying = true
	g.mu.Unlock()

	fn()

	g.mu.Lock()
	g.flying = false
	g.mu.Unlock()

	return
}
<commit_msg>improve default logger layout<commit_after>/*
Package supervisor provides supervisor trees for Go applications.

This package is a clean reimplementation of github.com/thejerf/suture, aiming
to be more Go idiomatic, thus less Erlang-like.

It is built on top of context package, with all of its advantages, namely the
possibility trickle down context-related values and cancellation signals.

TheJerf's blog post about Suture is a very good and helpful read to understand
how this package has been implemented.

http://www.jerf.org/iri/post/2930
*/
package supervisor // import "cirello.io/supervisor"

import (
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"golang.org/x/net/context"
)

// Service is the public interface expected by a Supervisor.
//
// This will be internally named after the result of fmt.Stringer, if available.
// Otherwise it will going to use an internal representation for the service
// name.
type Service interface {
	// Serve is called by a Supervisor to start the service. It expects the
	// service to honor the passed context and its lifetime. Observe
	// <-ctx.Done() and ctx.Err(). If the service is stopped by anything
	// but the Supervisor, it will get started again. Be careful with shared
	// state among restarts.
	Serve(ctx context.Context)
}

// Supervisor is the basic datastructure responsible for offering a supervisor
// tree. It implements Service, therefore it can be nested if necessary. When
// passing the Supervisor around, remind to do it as reference (&supervisor).
type Supervisor struct {
	// Name for this supervisor tree, used for logging.
	Name string

	// FailureDecay is the timespan on which the current failure count will
	// be halved.
	FailureDecay float64

	// FailureThreshold is the maximum accepted number of failures, after
	// decay adjustment, that shall trigger the back-off wait.
	FailureThreshold float64

	// Backoff is the wait duration when hit threshold.
	Backoff time.Duration

	// Log is a replaceable function used for overall logging
	Log func(interface{})

	ready sync.Once

	addedService    chan struct{}
	startedServices chan struct{}

	runningServices sync.WaitGroup

	servicesMu sync.Mutex
	services   map[string]Service

	cancelationsMu sync.Mutex
	cancelations   map[string]context.CancelFunc

	backoffMu sync.Mutex
	backoff   map[string]*backoff

	singleflight singleflight
}

func (s *Supervisor) String() string {
	return s.Name
}

func (s *Supervisor) prepare() {
	s.ready.Do(func() {
		if s.Name == "" {
			s.Name = "supervisor"
		}
		s.addedService = make(chan struct{}, 1)
		s.backoff = make(map[string]*backoff)
		s.cancelations = make(map[string]context.CancelFunc)
		s.services = make(map[string]Service)
		s.startedServices = make(chan struct{}, 1)

		if s.Log == nil {
			s.Log = func(msg interface{}) {
				log.Printf("%s: %v", s.Name, msg)
			}
		}
		if s.FailureDecay == 0 {
			s.FailureDecay = 30
		}
		if s.FailureThreshold == 0 {
			s.FailureThreshold = 5
		}
		if s.Backoff == 0 {
			s.Backoff = 15 * time.Second
		}
	})
}

// Add inserts into the Supervisor tree a new service. If the Supervisor is
// already started, it will start it automatically.
func (s *Supervisor) Add(service Service) {
	s.prepare()

	name := fmt.Sprintf("%s", service)

	s.servicesMu.Lock()
	s.backoffMu.Lock()
	s.backoff[name] = &backoff{}
	s.services[name] = service
	s.backoffMu.Unlock()
	s.servicesMu.Unlock()

	select {
	case s.addedService <- struct{}{}:
	default:
	}
}

// Remove stops the service in the Supervisor tree and remove from it.
func (s *Supervisor) Remove(name string) {
	s.prepare()

	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()
	if _, ok := s.services[name]; !ok {
		return
	}

	s.cancelationsMu.Lock()
	defer s.cancelationsMu.Unlock()
	if c, ok := s.cancelations[name]; ok {
		delete(s.cancelations, name)
		c()
	}
}

// Serve starts the Supervisor tree. It can be started only once at a time. If
// stopped (canceled), it can be restarted. It will discard context of
// successive calls to supervisor when still running.
func (s *Supervisor) Serve(ctx context.Context) {
	s.prepare()

	s.singleflight.do(func() {
		s.serve(ctx)
	})
}

// Services return a list of services
func (s *Supervisor) Services() map[string]Service {
	svclist := make(map[string]Service)
	s.servicesMu.Lock()
	for k, v := range s.services {
		svclist[k] = v
	}
	s.servicesMu.Unlock()
	return svclist
}

// Cancelations return a list of services names and their cancellation calls
func (s *Supervisor) Cancelations() map[string]context.CancelFunc {
	svclist := make(map[string]context.CancelFunc)
	s.cancelationsMu.Lock()
	for k, v := range s.cancelations {
		svclist[k] = v
	}
	s.cancelationsMu.Unlock()
	return svclist
}

func (s *Supervisor) serve(ctx context.Context) {
	select {
	case s.addedService <- struct{}{}:
	default:
	}
	go func(ctx context.Context) {
		for {
			select {
			case <-s.addedService:
				s.startServices(ctx)
				select {
				case s.startedServices <- struct{}{}:
				default:
				}

			case <-ctx.Done():
				return
			}
		}
	}(ctx)
	<-ctx.Done()

	s.runningServices.Wait()

	s.cancelationsMu.Lock()
	s.cancelations = make(map[string]context.CancelFunc)
	s.cancelationsMu.Unlock()
	return
}

func (s *Supervisor) startServices(ctx context.Context) {
	s.servicesMu.Lock()
	defer s.servicesMu.Unlock()

	var wg sync.WaitGroup

	for name, svc := range s.services {
		s.cancelationsMu.Lock()
		_, ok := s.cancelations[name]
		if ok {
			s.cancelationsMu.Unlock()
			continue
		}

		wg.Add(1)
		go func(name string, svc Service) {
			s.runningServices.Add(1)
			wg.Done()
			for {
				retry := func() (retry bool) {
					select {
					case <-ctx.Done():
						return false
					default:
					}

					defer func() {
						if r := recover(); r != nil {
							s.Log(fmt.Sprintf("trapped panic: %v", r))
							retry = true
						}
					}()

					c, cancel := context.WithCancel(ctx)
					s.cancelationsMu.Lock()
					s.cancelations[name] = cancel
					s.cancelationsMu.Unlock()
					svc.Serve(c)

					select {
					case <-ctx.Done():
						return false
					default:
						return true
					}
				}()
				if retry {
					s.Log(fmt.Sprintf("restarting %s", name))
					s.backoffMu.Lock()
					b := s.backoff[name]
					s.backoffMu.Unlock()
					b.wait(s.FailureDecay, s.FailureThreshold, s.Backoff, func(msg interface{}) {
						s.Log(fmt.Sprintf("backoff %s: %v", name, msg))
					})
					continue
				}
				break
			}
			s.runningServices.Done()
		}(name, svc)
		s.cancelationsMu.Unlock()
	}

	wg.Wait()
}

type backoff struct {
	lastfail time.Time
	failures float64
}

func (b *backoff) wait(failureDecay float64, threshold float64, backoffDur time.Duration, log func(str interface{})) {
	if b.lastfail.IsZero() {
		b.lastfail = time.Now()
		b.failures = 1.0
	} else {
		b.failures++
		intervals := time.Since(b.lastfail).Seconds() / failureDecay
		b.failures = b.failures*math.Pow(.5, intervals) + 1
	}

	if b.failures > threshold {
		log(backoffDur)
		time.Sleep(backoffDur)
	}
}

// Based from groupcache's singleflight
type singleflight struct {
	mu     sync.Mutex
	flying bool
}

func (g *singleflight) do(fn func()) {
	g.mu.Lock()
	if g.flying {
		g.mu.Unlock()
		return
	}

	g.flying = true
	g.mu.Unlock()

	fn()

	g.mu.Lock()
	g.flying = false
	g.mu.Unlock()

	return
}
<|endoftext|>


<commit_before>// Copyright © 2017 ben dewan <benj.dewan@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package progress

import (
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"sync"
	"time"
)

// ProgressBars is the primary interface into the package.
// codebeat:disable[TOO_MANY_IVARS]
type ProgressBars struct {
	// Writer can be set to direct where progress bars are printed
	// os.Stdout is used by default
	Writer io.Writer

	// Width specifies the overall width of the progress bars display.
	// 80 is used by default
	Width int

	// RefreshRate controls the frequency of updates being pushed to the
	// screen.
	// 1 second is the default
	RefreshRate time.Duration

	// internal fields
	bars     []*progressBar
	started  bool
	stopChan chan struct{}
	lock     *sync.RWMutex
}

// codebeat:enable[TOO_MANY_IVARS]

const (
	stateRunning  = "running"
	stateDone     = "done"
	stateFailed   = "failed"
	stateFinished = "finished"
)

const bar = "░"

type progressBar struct {
	action string
	name   string
	state  string
}

// New provisions a new ProgressBars struct with default values
func New() *ProgressBars {
	return &ProgressBars{
		Writer:      os.Stdout,
		Width:       80,
		RefreshRate: time.Second,
		bars:        [](*progressBar){},
		stopChan:    make(chan struct{}, 1),
		lock:        &sync.RWMutex{},
	}
}

// AddBar adds a progress bar to the ProgressBars struct. The action and name
// are used to print status information when the progress bars are started,
// Because this package is designed to work in non-tty settings where
// re-painting screens doesn't work we cannot dynamically add new progress
// bars, so this method panics if the ProgressBars is currently running when
// invoked.
func (p *ProgressBars) AddBar(action, name string) *ProgressBars {
	if p.started {
		panic("Progress bars cannot be added while running")
	}
	p.bars = append(p.bars, &progressBar{
		action: action,
		name:   name,
		state:  stateRunning,
	})
	return p
}

// Start prints a state header defining the progress bars and then begins
// printing the bars themselves. It will continue printing until every bar
// is finished or Stop() is called.
func (p *ProgressBars) Start() {
	p.started = true
	p.printHeader()

	go func() {
		for {
			select {
			case <-p.stopChan:
				return
			default:
				p.draw()
				time.Sleep(p.RefreshRate)
			}
		}
	}()

}

// Done terminates a single progress bar by name in a successful state
func (p *ProgressBars) Done(barName string) {
	p.changeState(barName, stateDone)
}

// Done terminates a single progress bar by name in a failure state
func (p *ProgressBars) Error(barName string) {
	p.changeState(barName, stateFailed)
}

// Stop ends the printing of all progress bars.
func (p *ProgressBars) Stop() {
	p.stopChan <- struct{}{}
	p.started = false
}

func (p *ProgressBars) changeState(name, state string) {
	for _, bar := range p.bars {
		if bar.name != name {
			continue
		}
		p.lock.Lock()
		bar.state = state
		p.lock.Unlock()
		return
	}
}

func (p *ProgressBars) draw() {
	line := strings.Join(p.statuses(), " ")
	if len(strings.Trim(line, " ")) == 0 { // everything has finished. Stop drawing
		p.Stop()
		return
	}

	fprintln(p.Writer, line)
}

func (p *ProgressBars) statuses() []string {
	statuses := []string{}
	width := p.barWidth()
	p.lock.Lock()
	for _, bar := range p.bars {
		statuses = append(statuses, bar.statusString(width))
	}
	p.lock.Unlock()
	return statuses
}

func (bar *progressBar) statusString(width int) string {
	switch bar.state {
	case stateRunning:
		return strings.Repeat("░", width)
	case stateDone:
		bar.state = stateFinished
		return center("DONE", width)
	case stateFailed:
		bar.state = stateFinished
		return center("ERROR", width)
	case stateFinished:
		return strings.Repeat(" ", width)
	default:
		log.Panicf("Unknown progress bar state: %s", bar.state)
	}
	return "Unreachable"
}

func (p *ProgressBars) printHeader() {
	fprintln(p.Writer, strings.Join(p.barHeaders(), " "))
}

func (p *ProgressBars) barHeaders() []string {
	width := p.barWidth()
	if maxWidth(p.bars) < width {
		return completeHeaders(p.bars, width)
	} else if maxNameWidth(p.bars) < width {
		return namedHeaders(p.bars, width, p.Writer)
	}
	return numberedHeaders(p.bars, width, p.Writer)
}

func maxWidth(bars []*progressBar) int {
	max := 0
	for _, bar := range bars {
		str := fmt.Sprintf("%s '%s'", bar.action, bar.name)
		if len(str) > max {
			max = len(str)
		}
	}
	return max
}

func maxNameWidth(bars []*progressBar) int {
	max := 0
	for _, bar := range bars {
		if len(bar.name) > max {
			max = len(bar.name)
		}
	}
	return max
}

func completeHeaders(bars []*progressBar, barWidth int) []string {
	barHeaders := []string{}
	for _, bar := range bars {
		barHeaders = append(barHeaders,
			center(fmt.Sprintf("%s '%s'", bar.action, bar.name),
				barWidth))
	}
	return barHeaders
}

func namedHeaders(bars []*progressBar, barWidth int, w io.Writer) []string {
	barHeaders := []string{}
	for _, bar := range bars {
		fprintf(w, "%s '%s'\n", bar.action, bar.name)
		barHeaders = append(barHeaders, center(bar.name, barWidth))
	}
	return barHeaders
}

func numberedHeaders(bars []*progressBar, barWidth int, w io.Writer) []string {
	barHeaders := []string{}
	for i, bar := range bars {
		num := fmt.Sprintf("%02d", i)
		fprintf(w, "%02d - %s '%s'\n", i, bar.action, bar.name)
		barHeaders = append(barHeaders, center(num, barWidth))
	}
	return barHeaders
}

func (p *ProgressBars) barWidth() int {
	return (p.Width - len(p.bars)) / len(p.bars)
}

func center(str string, width int) string {
	if len(str) > width {
		return strings.Join(strings.SplitAfter(str, "")[:width], "")
	}
	div := (width - len(str)) / 2

	return strings.Repeat(" ", div) + str + strings.Repeat(" ", (width-div-len(str)))
}

func fprintln(w io.Writer, a ...interface{}) {
	if _, err := fmt.Fprintln(w, a...); err != nil {
		panic(err)
	}
}

func fprintf(w io.Writer, format string, a ...interface{}) {
	if _, err := fmt.Fprintf(w, format, a...); err != nil {
		panic(err)
	}
}
<commit_msg>Draw a final line when ProgressBars.Stop() is called.<commit_after>// Copyright © 2017 ben dewan <benj.dewan@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package progress

import (
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"sync"
	"time"
)

// ProgressBars is the primary interface into the package.
// codebeat:disable[TOO_MANY_IVARS]
type ProgressBars struct {
	// Writer can be set to direct where progress bars are printed
	// os.Stdout is used by default
	Writer io.Writer

	// Width specifies the overall width of the progress bars display.
	// 80 is used by default
	Width int

	// RefreshRate controls the frequency of updates being pushed to the
	// screen.
	// 1 second is the default
	RefreshRate time.Duration

	// internal fields
	bars     []*progressBar
	started  bool
	stopChan chan struct{}
	lock     *sync.RWMutex
}

// codebeat:enable[TOO_MANY_IVARS]

const (
	stateRunning  = "running"
	stateDone     = "done"
	stateFailed   = "failed"
	stateFinished = "finished"
)

const bar = "░"

type progressBar struct {
	action string
	name   string
	state  string
}

// New provisions a new ProgressBars struct with default values
func New() *ProgressBars {
	return &ProgressBars{
		Writer:      os.Stdout,
		Width:       80,
		RefreshRate: time.Second,
		bars:        [](*progressBar){},
		stopChan:    make(chan struct{}, 1),
		lock:        &sync.RWMutex{},
	}
}

// AddBar adds a progress bar to the ProgressBars struct. The action and name
// are used to print status information when the progress bars are started,
// Because this package is designed to work in non-tty settings where
// re-painting screens doesn't work we cannot dynamically add new progress
// bars, so this method panics if the ProgressBars is currently running when
// invoked.
func (p *ProgressBars) AddBar(action, name string) *ProgressBars {
	if p.started {
		panic("Progress bars cannot be added while running")
	}
	p.bars = append(p.bars, &progressBar{
		action: action,
		name:   name,
		state:  stateRunning,
	})
	return p
}

// Start prints a state header defining the progress bars and then begins
// printing the bars themselves. It will continue printing until every bar
// is finished or Stop() is called.
func (p *ProgressBars) Start() {
	p.started = true
	p.printHeader()

	go func() {
		for {
			select {
			case <-p.stopChan:
				return
			default:
				p.draw()
				time.Sleep(p.RefreshRate)
			}
		}
	}()

}

// Done terminates a single progress bar by name in a successful state
func (p *ProgressBars) Done(barName string) {
	p.changeState(barName, stateDone)
}

// Done terminates a single progress bar by name in a failure state
func (p *ProgressBars) Error(barName string) {
	p.changeState(barName, stateFailed)
}

// Stop ends the printing of all progress bars. Because Stop()
// is often called immediately after a call to Done() or Error()
// we do a final status update before actually doing a stop to flush
// those messages
func (p *ProgressBars) Stop() {
	p.draw()
	p.stopChan <- struct{}{}
	p.started = false
}

func (p *ProgressBars) changeState(name, state string) {
	for _, bar := range p.bars {
		if bar.name != name {
			continue
		}
		p.lock.Lock()
		bar.state = state
		p.lock.Unlock()
		return
	}
}

func (p *ProgressBars) draw() {
	line := strings.Join(p.statuses(), " ")
	if len(strings.Trim(line, " ")) == 0 { // everything has finished. Stop drawing
		p.Stop()
		return
	}

	fprintln(p.Writer, line)
}

func (p *ProgressBars) statuses() []string {
	statuses := []string{}
	width := p.barWidth()
	p.lock.Lock()
	for _, bar := range p.bars {
		statuses = append(statuses, bar.statusString(width))
	}
	p.lock.Unlock()
	return statuses
}

func (bar *progressBar) statusString(width int) string {
	switch bar.state {
	case stateRunning:
		return strings.Repeat("░", width)
	case stateDone:
		bar.state = stateFinished
		return center("DONE", width)
	case stateFailed:
		bar.state = stateFinished
		return center("ERROR", width)
	case stateFinished:
		return strings.Repeat(" ", width)
	default:
		log.Panicf("Unknown progress bar state: %s", bar.state)
	}
	return "Unreachable"
}

func (p *ProgressBars) printHeader() {
	fprintln(p.Writer, strings.Join(p.barHeaders(), " "))
}

func (p *ProgressBars) barHeaders() []string {
	width := p.barWidth()
	if maxWidth(p.bars) < width {
		return completeHeaders(p.bars, width)
	} else if maxNameWidth(p.bars) < width {
		return namedHeaders(p.bars, width, p.Writer)
	}
	return numberedHeaders(p.bars, width, p.Writer)
}

func maxWidth(bars []*progressBar) int {
	max := 0
	for _, bar := range bars {
		str := fmt.Sprintf("%s '%s'", bar.action, bar.name)
		if len(str) > max {
			max = len(str)
		}
	}
	return max
}

func maxNameWidth(bars []*progressBar) int {
	max := 0
	for _, bar := range bars {
		if len(bar.name) > max {
			max = len(bar.name)
		}
	}
	return max
}

func completeHeaders(bars []*progressBar, barWidth int) []string {
	barHeaders := []string{}
	for _, bar := range bars {
		barHeaders = append(barHeaders,
			center(fmt.Sprintf("%s '%s'", bar.action, bar.name),
				barWidth))
	}
	return barHeaders
}

func namedHeaders(bars []*progressBar, barWidth int, w io.Writer) []string {
	barHeaders := []string{}
	for _, bar := range bars {
		fprintf(w, "%s '%s'\n", bar.action, bar.name)
		barHeaders = append(barHeaders, center(bar.name, barWidth))
	}
	return barHeaders
}

func numberedHeaders(bars []*progressBar, barWidth int, w io.Writer) []string {
	barHeaders := []string{}
	for i, bar := range bars {
		num := fmt.Sprintf("%02d", i)
		fprintf(w, "%02d - %s '%s'\n", i, bar.action, bar.name)
		barHeaders = append(barHeaders, center(num, barWidth))
	}
	return barHeaders
}

func (p *ProgressBars) barWidth() int {
	return (p.Width - len(p.bars)) / len(p.bars)
}

func center(str string, width int) string {
	if len(str) > width {
		return strings.Join(strings.SplitAfter(str, "")[:width], "")
	}
	div := (width - len(str)) / 2

	return strings.Repeat(" ", div) + str + strings.Repeat(" ", (width-div-len(str)))
}

func fprintln(w io.Writer, a ...interface{}) {
	if _, err := fmt.Fprintln(w, a...); err != nil {
		panic(err)
	}
}

func fprintf(w io.Writer, format string, a ...interface{}) {
	if _, err := fmt.Fprintf(w, format, a...); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
)

// cli flags
var (
	uploadDir string
	baseUrl   string
	port      int
	host      string
)

func init() {
	flag.StringVar(&uploadDir, "uploaddir", "/var/www/oop/",
		"where to keep uploaded files")
	flag.StringVar(&baseUrl, "url", "",
		"base url content is served from")
	flag.StringVar(&host, "host", "", "host to listen on")
	flag.IntVar(&port, "port", 8080, "port to listen on")

	flag.Parse()

	if baseUrl == "" {
		log.Fatal("-url is required")
	}
}

func main() {
	http.HandleFunc("/delete", handleDelete)
	http.HandleFunc("/up", handleUpload)
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		if req.URL.Path != "/" {
			log.Printf("not found: %s", req.URL.Path)
			http.NotFound(w, req)
			return
		}

		handleIndex(w, req)
	})

	addr := fmt.Sprintf("%s:%d", host, port)

	log.Printf("starting server on %s", addr)
	http.ListenAndServe(addr, nil)
}

func handleUpload(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "not allowed", http.StatusMethodNotAllowed)
		return
	}

	file, header, err := r.FormFile("file")
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	defer file.Close()

	s := strings.Split(header.Filename, ".")
	name, ext := s[0], s[len(s)-1]

	tmpfile, err := ioutil.TempFile(uploadDir, fmt.Sprintf("*.%s", ext))
	if err != nil {
		log.Printf("create temp file: %+v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer tmpfile.Close()

	log.Printf("receiving file: %s.%s => %s", name, ext, tmpfile.Name())

	if b, err := io.Copy(tmpfile, file); err != nil {
		log.Printf("download failed: %+v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	} else {
		log.Printf("%s completed (%d bytes)", tmpfile.Name(), b)
	}

	// Tempfiles are created as 0600, which means other processes
	// won't be able to read them.
	if err := os.Chmod(tmpfile.Name(), 0644); err != nil {
		log.Printf("chmod failed: %+v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	url, _ := url.Parse(baseUrl)
	url.Path = path.Join(url.Path, path.Base(tmpfile.Name()))
	w.Write([]byte(url.String() + "\n"))
}

func handleDelete(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "not allowed", http.StatusMethodNotAllowed)
		return
	}

	panic("not implemented")
}

func handleIndex(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodGet {
		http.Error(w, "not allowed", http.StatusMethodNotAllowed)
		return
	}

	w.Header().Add("Content-Type", "text/html")
	w.Write([]byte(`
<!doctype html>

  <title>oop</title>
  <meta name="viewport"
        content="width=device-width, initial-scale=1">

oop

<form method="post" action="up" enctype="multipart/form-data">
  <input type="file" name="file">

  <input type="submit" value="Upload">
</form>
`))
}
<commit_msg>Show uploaded files on index<commit_after>package main

import (
	"flag"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
)

// cli flags
var (
	uploadDir string
	baseUrl   string
	port      int
	host      string
)

func init() {
	flag.StringVar(&uploadDir, "uploaddir", "/var/www/oop/",
		"where to keep uploaded files")
	flag.StringVar(&baseUrl, "url", "",
		"base url content is served from")
	flag.StringVar(&host, "host", "", "host to listen on")
	flag.IntVar(&port, "port", 8080, "port to listen on")

	flag.Parse()

	if baseUrl == "" {
		log.Fatal("-url is required")
	}
}

func main() {
	http.HandleFunc("/delete", handleDelete)
	http.HandleFunc("/up", handleUpload)
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		if req.URL.Path != "/" {
			log.Printf("not found: %s", req.URL.Path)
			http.NotFound(w, req)
			return
		}

		handleIndex(w, req)
	})

	addr := fmt.Sprintf("%s:%d", host, port)

	log.Printf("starting server on %s", addr)
	http.ListenAndServe(addr, nil)
}

func handleUpload(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "not allowed", http.StatusMethodNotAllowed)
		return
	}

	file, header, err := r.FormFile("file")
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	defer file.Close()

	s := strings.Split(header.Filename, ".")
	name, ext := s[0], s[len(s)-1]

	tmpfile, err := ioutil.TempFile(uploadDir, fmt.Sprintf("*.%s", ext))
	if err != nil {
		log.Printf("create temp file: %+v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer tmpfile.Close()

	log.Printf("receiving file: %s.%s => %s", name, ext, tmpfile.Name())

	if b, err := io.Copy(tmpfile, file); err != nil {
		log.Printf("download failed: %+v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	} else {
		log.Printf("%s completed (%d bytes)", tmpfile.Name(), b)
	}

	// Tempfiles are created as 0600, which means other processes
	// won't be able to read them.
	if err := os.Chmod(tmpfile.Name(), 0644); err != nil {
		log.Printf("chmod failed: %+v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Write([]byte(staticUrl(tmpfile.Name()) + "\n"))
}

// staticUrl takes a file system path (presumably within the upload
// directory, but it doesn't matter) and returns the URL that this file
// is served at.
func staticUrl(filePath string) string {
	url, _ := url.Parse(baseUrl)
	url.Path = path.Join(url.Path, path.Base(filePath))

	return url.String()
}

func handleDelete(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "not allowed", http.StatusMethodNotAllowed)
		return
	}

	panic("not implemented")
}

var (
	indexTmplStr = `
<!doctype html>

  <title>oop</title>
  <meta name="viewport" content="width=device-width, initial-scale=1">

oop

<form method="post" action="up" enctype="multipart/form-data">
  <input type="file" name="file">

  <input type="submit" value="Upload">
</form>

uploads

  {{range .}}

      {{.Timestamp}}
      {{ .Name }}
      ({{.SizeKb}} KiB)

  {{end}}

`
	indexTemplate = template.Must(template.New("index").Parse(indexTmplStr))
)

type staticFile struct {
	Timestamp string
	Name      string
	Url       string
	SizeKb    int64
}

// listFiles returns all files inside `uploadDir`.
func listFiles(dir string) ([]staticFile, error) {
	fileInfo := []os.FileInfo{}
	err := filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			fileInfo = append(fileInfo, info)
		}

		return nil
	})

	// newest files first
	sort.Slice(fileInfo, func(i, j int) bool {
		return fileInfo[i].ModTime().Unix() > fileInfo[j].ModTime().Unix()
	})

	files := make([]staticFile, len(fileInfo))
	for i, info := range fileInfo {
		files[i] = staticFile{
			Timestamp: info.ModTime().Format("2006-01-02 15:04"),
			Name:      info.Name(),
			Url:       staticUrl(info.Name()),
			SizeKb:    info.Size() / 1024,
		}

	}
	return files, err
}

func handleIndex(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodGet {
		http.Error(w, "not allowed", http.StatusMethodNotAllowed)
		return
	}

	files, err := listFiles(uploadDir)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Add("Content-Type", "text/html")
	if err := indexTemplate.Execute(w, files); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}
<|endoftext|>


<commit_before>package projects

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/ion-channel/ionic/aliases"
	"github.com/ion-channel/ionic/tags"
)

const (
	validEmailRegex = `^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,}$`
)

var (
	// ErrInvalidProject is returned when a given project does not pass the
	// standards for a project
	ErrInvalidProject = fmt.Errorf("project has invalid fields")
)

//Project is a representation of a project within the Ion Channel system
type Project struct {
	ID             *string         `json:"id,omitempty"`
	TeamID         *string         `json:"team_id,omitempty"`
	RulesetID      *string         `json:"ruleset_id,omitempty"`
	Name           *string         `json:"name,omitempty"`
	Type           *string         `json:"type,omitempty"`
	Source         *string         `json:"source,omitempty"`
	Branch         *string         `json:"branch,omitempty"`
	Description    *string         `json:"description,omitempty"`
	Active         bool            `json:"active"`
	ChatChannel    string          `json:"chat_channel"`
	CreatedAt      time.Time       `json:"created_at"`
	UpdatedAt      time.Time       `json:"updated_at"`
	DeployKey      string          `json:"deploy_key"`
	Monitor        bool            `json:"should_monitor"`
	POCName        string          `json:"poc_name"`
	POCEmail       string          `json:"poc_email"`
	Username       string          `json:"username"`
	Password       string          `json:"password"`
	KeyFingerprint string          `json:"key_fingerprint"`
	Aliases        []aliases.Alias `json:"aliases"`
	Tags           []tags.Tag      `json:"tags"`
}

// Validate takes an http client and returns a slice of fields as a string and
// an error. The fields will be a list of fields that did not pass the
// validation. An error will only be returned if any of the fields fail their
// validation.
func (p *Project) Validate(client *http.Client) (map[string]string, error) {
	invalidFields := make(map[string]string)
	var projErr error

	if p.ID == nil {
		invalidFields["id"] = "missing id"
		projErr = ErrInvalidProject
	}

	if p.TeamID == nil {
		invalidFields["team_id"] = "missing team id"
		projErr = ErrInvalidProject
	}

	if p.RulesetID == nil {
		invalidFields["ruleset_id"] = "missing ruleset id"
		projErr = ErrInvalidProject
	}

	if p.Name == nil {
		invalidFields["name"] = "missing name"
		projErr = ErrInvalidProject
	}

	if p.Type == nil {
		invalidFields["type"] = "missing type"
		projErr = ErrInvalidProject
	}

	if p.Source == nil {
		invalidFields["source"] = "missing source"
		projErr = ErrInvalidProject
	}

	if p.Branch == nil {
		invalidFields["branch"] = "missing branch"
		projErr = ErrInvalidProject
	}

	if p.Description == nil {
		invalidFields["description"] = "missing description"
		projErr = ErrInvalidProject
	}

	r := regexp.MustCompile(validEmailRegex)
	if p.POCEmail != "" && !r.MatchString(p.POCEmail) {
		invalidFields["poc_email"] = "invalid email supplied"
		projErr = ErrInvalidProject
	}

	block, rest := pem.Decode([]byte(p.DeployKey))
	if block != nil {
		pkey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
		if err != nil {
			invalidFields["deploy_key"] = "must be a valid ssh key"
			projErr = ErrInvalidProject
		} else {
			err = pkey.Validate()
			if err != nil {
				invalidFields["deploy_key"] = "must be a valid ssh key"
				projErr = ErrInvalidProject
			}
		}
	}

	if block == nil && rest != nil && string(rest) != "" {
		invalidFields["deploy_key"] = "must be a valid ssh key"
		projErr = ErrInvalidProject
	}

	if p.Type != nil {
		switch strings.ToLower(*p.Type) {
		case "artifact":
			u, err := url.Parse(*p.Source)
			if err != nil {
				invalidFields["source"] = fmt.Sprintf("source must be a valid url: %v", err.Error())
				projErr = ErrInvalidProject
			}

			if u != nil {
				res, err := client.Head(u.String())
				if err != nil {
					invalidFields["source"] = "source failed to return a response"
					projErr = ErrInvalidProject
				}

				if res != nil && res.StatusCode == http.StatusNotFound {
					invalidFields["source"] = "source returned a not found"
					projErr = ErrInvalidProject
				}
			}
		}
	}

	return invalidFields, projErr
}
<commit_msg>adding url validation<commit_after>package projects

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/ion-channel/ionic/aliases"
	"github.com/ion-channel/ionic/tags"
)

const (
	validEmailRegex  = `^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,}$`
	validGitURIRegex = `^(?:[http|ftp|gopher|mailto|mid|cid|news|nntp|prospero|telnet|rlogin|tn3270|wais|svn|git]+\+ssh\:\/\/|git\+[http][s]?:\/\/|git\@|[http|ftp|gopher|mailto|mid|cid|news|nntp|prospero|telnet|rlogin|tn3270|wais|svn|git]+[s]?:\/\/)[[^\s]\w\@]+$`
)

var (
	// ErrInvalidProject is returned when a given project does not pass the
	// standards for a project
	ErrInvalidProject = fmt.Errorf("project has invalid fields")
)

//Project is a representation of a project within the Ion Channel system
type Project struct {
	ID             *string         `json:"id,omitempty"`
	TeamID         *string         `json:"team_id,omitempty"`
	RulesetID      *string         `json:"ruleset_id,omitempty"`
	Name           *string         `json:"name,omitempty"`
	Type           *string         `json:"type,omitempty"`
	Source         *string         `json:"source,omitempty"`
	Branch         *string         `json:"branch,omitempty"`
	Description    *string         `json:"description,omitempty"`
	Active         bool            `json:"active"`
	ChatChannel    string          `json:"chat_channel"`
	CreatedAt      time.Time       `json:"created_at"`
	UpdatedAt      time.Time       `json:"updated_at"`
	DeployKey      string          `json:"deploy_key"`
	Monitor        bool            `json:"should_monitor"`
	POCName        string          `json:"poc_name"`
	POCEmail       string          `json:"poc_email"`
	Username       string          `json:"username"`
	Password       string          `json:"password"`
	KeyFingerprint string          `json:"key_fingerprint"`
	Aliases        []aliases.Alias `json:"aliases"`
	Tags           []tags.Tag      `json:"tags"`
}

// Validate takes an http client and returns a slice of fields as a string and
// an error. The fields will be a list of fields that did not pass the
// validation. An error will only be returned if any of the fields fail their
// validation.
func (p *Project) Validate(client *http.Client) (map[string]string, error) {
	invalidFields := make(map[string]string)
	var projErr error

	if p.ID == nil {
		invalidFields["id"] = "missing id"
		projErr = ErrInvalidProject
	}

	if p.TeamID == nil {
		invalidFields["team_id"] = "missing team id"
		projErr = ErrInvalidProject
	}

	if p.RulesetID == nil {
		invalidFields["ruleset_id"] = "missing ruleset id"
		projErr = ErrInvalidProject
	}

	if p.Name == nil {
		invalidFields["name"] = "missing name"
		projErr = ErrInvalidProject
	}

	if p.Type == nil {
		invalidFields["type"] = "missing type"
		projErr = ErrInvalidProject
	}

	if p.Source == nil {
		invalidFields["source"] = "missing source"
		projErr = ErrInvalidProject
	}

	if p.Branch == nil {
		invalidFields["branch"] = "missing branch"
		projErr = ErrInvalidProject
	}

	if p.Description == nil {
		invalidFields["description"] = "missing description"
		projErr = ErrInvalidProject
	}

	r := regexp.MustCompile(validEmailRegex)
	if p.POCEmail != "" && !r.MatchString(p.POCEmail) {
		invalidFields["poc_email"] = "invalid email supplied"
		projErr = ErrInvalidProject
	}

	block, rest := pem.Decode([]byte(p.DeployKey))
	if block != nil {
		pkey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
		if err != nil {
			invalidFields["deploy_key"] = "must be a valid ssh key"
			projErr = ErrInvalidProject
		} else {
			err = pkey.Validate()
			if err != nil {
				invalidFields["deploy_key"] = "must be a valid ssh key"
				projErr = ErrInvalidProject
			}
		}
	}

	if block == nil && rest != nil && string(rest) != "" {
		invalidFields["deploy_key"] = "must be a valid ssh key"
		projErr = ErrInvalidProject
	}

	if p.Type != nil {
		switch strings.ToLower(*p.Type) {
		case "artifact":
			u, err := url.Parse(*p.Source)
			if err != nil {
				invalidFields["source"] = fmt.Sprintf("source must be a valid url: %v", err.Error())
				projErr = ErrInvalidProject
			}

			if u != nil {
				res, err := client.Head(u.String())
				if err != nil {
					invalidFields["source"] = "source failed to return a response"
					projErr = ErrInvalidProject
				}

				if res != nil && res.StatusCode == http.StatusNotFound {
					invalidFields["source"] = "source returned a not found"
					projErr = ErrInvalidProject
				}
			}
		case "git", "svn":
			r := regexp.MustCompile(validGitURIRegex)
			if p.Source != nil && !r.MatchString(*p.Source) {
				invalidFields["source"] = "source must be a valid uri"
				projErr = ErrInvalidProject
			}
		}
	}

	return invalidFields, projErr
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, softwar
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package promql

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/util/teststorage"
)

func BenchmarkRangeQuery(b *testing.B) {
	storage := teststorage.New(b)
	defer storage.Close()
	opts := EngineOpts{
		Logger:        nil,
		Reg:           nil,
		MaxConcurrent: 10,
		MaxSamples:    50000000,
		Timeout:       100 * time.Second,
	}
	engine := NewEngine(opts)

	metrics := []labels.Labels{}
	metrics = append(metrics, labels.FromStrings("__name__", "a_one"))
	metrics = append(metrics, labels.FromStrings("__name__", "b_one"))
	for j := 0; j < 10; j++ {
		metrics = append(metrics, labels.FromStrings("__name__", "h_one", "le", strconv.Itoa(j)))
	}
	metrics = append(metrics, labels.FromStrings("__name__", "h_one", "le", "+Inf"))

	for i := 0; i < 10; i++ {
		metrics = append(metrics, labels.FromStrings("__name__", "a_ten", "l", strconv.Itoa(i)))
		metrics = append(metrics, labels.FromStrings("__name__", "b_ten", "l", strconv.Itoa(i)))
		for j := 0; j < 10; j++ {
			metrics = append(metrics, labels.FromStrings("__name__", "h_ten", "l", strconv.Itoa(i), "le", strconv.Itoa(j)))
		}
		metrics = append(metrics, labels.FromStrings("__name__", "h_ten", "l", strconv.Itoa(i), "le", "+Inf"))
	}

	for i := 0; i < 100; i++ {
		metrics = append(metrics, labels.FromStrings("__name__", "a_hundred", "l", strconv.Itoa(i)))
		metrics = append(metrics, labels.FromStrings("__name__", "b_hundred", "l", strconv.Itoa(i)))
		for j := 0; j < 10; j++ {
			metrics = append(metrics, labels.FromStrings("__name__", "h_hundred", "l", strconv.Itoa(i), "le", strconv.Itoa(j)))
		}
		metrics = append(metrics, labels.FromStrings("__name__", "h_hundred", "l", strconv.Itoa(i), "le", "+Inf"))
	}
	refs := make([]uint64, len(metrics))

	// A day of data plus 10k steps.
	numIntervals := 8640 + 10000

	for s := 0; s < numIntervals; s++ {
		a, err := storage.Appender()
		if err != nil {
			b.Fatal(err)
		}
		ts := int64(s * 10000) // 10s interval.
		for i, metric := range metrics {
			err := a.AddFast(metric, refs[i], ts, float64(s))
			if err != nil {
				refs[i], _ = a.Add(metric, ts, float64(s))
			}
		}
		if err := a.Commit(); err != nil {
			b.Fatal(err)
		}
	}

	type benchCase struct {
		expr  string
		steps int
	}
	cases := []benchCase{
		// Plain retrieval.
		{
			expr: "a_X",
		},
		// Simple rate.
		{
			expr: "rate(a_X[1m])",
		},
		{
			expr:  "rate(a_X[1m])",
			steps: 10000,
		},
		// Holt-Winters and long ranges.
		{
			expr: "holt_winters(a_X[1d], 0.3, 0.3)",
		},
		{
			expr: "changes(a_X[1d])",
		},
		{
			expr: "rate(a_X[1d])",
		},
		// Unary operators.
		{
			expr: "-a_X",
		},
		// Binary operators.
		{
			expr: "a_X - b_X",
		},
		{
			expr:  "a_X - b_X",
			steps: 10000,
		},
		{
			expr: "a_X and b_X{l=~'.*[0-4]$'}",
		},
		{
			expr: "a_X or b_X{l=~'.*[0-4]$'}",
		},
		{
			expr: "a_X unless b_X{l=~'.*[0-4]$'}",
		},
		// Simple functions.
		{
			expr: "abs(a_X)",
		},
		{
			expr: "label_replace(a_X, 'l2', '$1', 'l', '(.*)')",
		},
		{
			expr: "label_join(a_X, 'l2', '-', 'l', 'l')",
		},
		// Simple aggregations.
		{
			expr: "sum(a_X)",
		},
		{
			expr: "sum without (l)(h_X)",
		},
		{
			expr: "sum without (le)(h_X)",
		},
		{
			expr: "sum by (l)(h_X)",
		},
		{
			expr: "sum by (le)(h_X)",
		},
		// Combinations.
		{
			expr: "rate(a_X[1m]) + rate(b_X[1m])",
		},
		{
			expr: "sum without (l)(rate(a_X[1m]))",
		},
		{
			expr: "sum without (l)(rate(a_X[1m])) / sum without (l)(rate(b_X[1m]))",
		},
		{
			expr: "histogram_quantile(0.9, rate(h_X[5m]))",
		},
	}

	// X in an expr will be replaced by different metric sizes.
	tmp := []benchCase{}
	for _, c := range cases {
		if !strings.Contains(c.expr, "X") {
			tmp = append(tmp, c)
		} else {
			tmp = append(tmp, benchCase{expr: strings.Replace(c.expr, "X", "one", -1), steps: c.steps})
			tmp = append(tmp, benchCase{expr: strings.Replace(c.expr, "X", "ten", -1), steps: c.steps})
			tmp = append(tmp, benchCase{expr: strings.Replace(c.expr, "X", "hundred", -1), steps: c.steps})
		}
	}
	cases = tmp

	// No step will be replaced by cases with the standard step.
	tmp = []benchCase{}
	for _, c := range cases {
		if c.steps != 0 {
			tmp = append(tmp, c)
		} else {
			tmp = append(tmp, benchCase{expr: c.expr, steps: 1})
			tmp = append(tmp, benchCase{expr: c.expr, steps: 10})
			tmp = append(tmp, benchCase{expr: c.expr, steps: 100})
			tmp = append(tmp, benchCase{expr: c.expr, steps: 1000})
		}
	}
	cases = tmp
	for _, c := range cases {
		name := fmt.Sprintf("expr=%s,steps=%d", c.expr, c.steps)
		b.Run(name, func(b *testing.B) {
			b.ReportAllocs()
			for i := 0; i < b.N; i++ {
				qry, err := engine.NewRangeQuery(
					storage, c.expr,
					time.Unix(int64((numIntervals-c.steps)*10), 0),
					time.Unix(int64(numIntervals*10), 0), time.Second*10)
				if err != nil {
					b.Fatal(err)
				}
				res := qry.Exec(context.Background())
				if res.Err != nil {
					b.Fatal(res.Err)
				}
				qry.Close()
			}
		})
	}
}
<commit_msg>Add Benchmark for the PromQL Parser (#6355)<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, softwar
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package promql

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/util/teststorage"
)

func BenchmarkRangeQuery(b *testing.B) {
	storage := teststorage.New(b)
	defer storage.Close()
	opts := EngineOpts{
		Logger:        nil,
		Reg:           nil,
		MaxConcurrent: 10,
		MaxSamples:    50000000,
		Timeout:       100 * time.Second,
	}
	engine := NewEngine(opts)

	metrics := []labels.Labels{}
	metrics = append(metrics, labels.FromStrings("__name__", "a_one"))
	metrics = append(metrics, labels.FromStrings("__name__", "b_one"))
	for j := 0; j < 10; j++ {
		metrics = append(metrics, labels.FromStrings("__name__", "h_one", "le", strconv.Itoa(j)))
	}
	metrics = append(metrics, labels.FromStrings("__name__", "h_one", "le", "+Inf"))

	for i := 0; i < 10; i++ {
		metrics = append(metrics, labels.FromStrings("__name__", "a_ten", "l", strconv.Itoa(i)))
		metrics = append(metrics, labels.FromStrings("__name__", "b_ten", "l", strconv.Itoa(i)))
		for j := 0; j < 10; j++ {
			metrics = append(metrics, labels.FromStrings("__name__", "h_ten", "l", strconv.Itoa(i), "le", strconv.Itoa(j)))
		}
		metrics = append(metrics, labels.FromStrings("__name__", "h_ten", "l", strconv.Itoa(i), "le", "+Inf"))
	}

	for i := 0; i < 100; i++ {
		metrics = append(metrics, labels.FromStrings("__name__", "a_hundred", "l", strconv.Itoa(i)))
		metrics = append(metrics, labels.FromStrings("__name__", "b_hundred", "l", strconv.Itoa(i)))
		for j := 0; j < 10; j++ {
			metrics = append(metrics, labels.FromStrings("__name__", "h_hundred", "l", strconv.Itoa(i), "le", strconv.Itoa(j)))
		}
		metrics = append(metrics, labels.FromStrings("__name__", "h_hundred", "l", strconv.Itoa(i), "le", "+Inf"))
	}
	refs := make([]uint64, len(metrics))

	// A day of data plus 10k steps.
	numIntervals := 8640 + 10000

	for s := 0; s < numIntervals; s++ {
		a, err := storage.Appender()
		if err != nil {
			b.Fatal(err)
		}
		ts := int64(s * 10000) // 10s interval.
		for i, metric := range metrics {
			err := a.AddFast(metric, refs[i], ts, float64(s))
			if err != nil {
				refs[i], _ = a.Add(metric, ts, float64(s))
			}
		}
		if err := a.Commit(); err != nil {
			b.Fatal(err)
		}
	}

	type benchCase struct {
		expr  string
		steps int
	}
	cases := []benchCase{
		// Plain retrieval.
		{
			expr: "a_X",
		},
		// Simple rate.
		{
			expr: "rate(a_X[1m])",
		},
		{
			expr:  "rate(a_X[1m])",
			steps: 10000,
		},
		// Holt-Winters and long ranges.
		{
			expr: "holt_winters(a_X[1d], 0.3, 0.3)",
		},
		{
			expr: "changes(a_X[1d])",
		},
		{
			expr: "rate(a_X[1d])",
		},
		// Unary operators.
		{
			expr: "-a_X",
		},
		// Binary operators.
		{
			expr: "a_X - b_X",
		},
		{
			expr:  "a_X - b_X",
			steps: 10000,
		},
		{
			expr: "a_X and b_X{l=~'.*[0-4]$'}",
		},
		{
			expr: "a_X or b_X{l=~'.*[0-4]$'}",
		},
		{
			expr: "a_X unless b_X{l=~'.*[0-4]$'}",
		},
		// Simple functions.
		{
			expr: "abs(a_X)",
		},
		{
			expr: "label_replace(a_X, 'l2', '$1', 'l', '(.*)')",
		},
		{
			expr: "label_join(a_X, 'l2', '-', 'l', 'l')",
		},
		// Simple aggregations.
		{
			expr: "sum(a_X)",
		},
		{
			expr: "sum without (l)(h_X)",
		},
		{
			expr: "sum without (le)(h_X)",
		},
		{
			expr: "sum by (l)(h_X)",
		},
		{
			expr: "sum by (le)(h_X)",
		},
		// Combinations.
		{
			expr: "rate(a_X[1m]) + rate(b_X[1m])",
		},
		{
			expr: "sum without (l)(rate(a_X[1m]))",
		},
		{
			expr: "sum without (l)(rate(a_X[1m])) / sum without (l)(rate(b_X[1m]))",
		},
		{
			expr: "histogram_quantile(0.9, rate(h_X[5m]))",
		},
	}

	// X in an expr will be replaced by different metric sizes.
	tmp := []benchCase{}
	for _, c := range cases {
		if !strings.Contains(c.expr, "X") {
			tmp = append(tmp, c)
		} else {
			tmp = append(tmp, benchCase{expr: strings.Replace(c.expr, "X", "one", -1), steps: c.steps})
			tmp = append(tmp, benchCase{expr: strings.Replace(c.expr, "X", "ten", -1), steps: c.steps})
			tmp = append(tmp, benchCase{expr: strings.Replace(c.expr, "X", "hundred", -1), steps: c.steps})
		}
	}
	cases = tmp

	// No step will be replaced by cases with the standard step.
	tmp = []benchCase{}
	for _, c := range cases {
		if c.steps != 0 {
			tmp = append(tmp, c)
		} else {
			tmp = append(tmp, benchCase{expr: c.expr, steps: 1})
			tmp = append(tmp, benchCase{expr: c.expr, steps: 10})
			tmp = append(tmp, benchCase{expr: c.expr, steps: 100})
			tmp = append(tmp, benchCase{expr: c.expr, steps: 1000})
		}
	}
	cases = tmp
	for _, c := range cases {
		name := fmt.Sprintf("expr=%s,steps=%d", c.expr, c.steps)
		b.Run(name, func(b *testing.B) {
			b.ReportAllocs()
			for i := 0; i < b.N; i++ {
				qry, err := engine.NewRangeQuery(
					storage, c.expr,
					time.Unix(int64((numIntervals-c.steps)*10), 0),
					time.Unix(int64(numIntervals*10), 0), time.Second*10)
				if err != nil {
					b.Fatal(err)
				}
				res := qry.Exec(context.Background())
				if res.Err != nil {
					b.Fatal(res.Err)
				}
				qry.Close()
			}
		})
	}
}

func BenchmarkParser(b *testing.B) {
	cases := []string{
		"a",
		"metric",
		"1",
		"1 >= bool 1",
		"1 + 2/(3*1)",
		"foo or bar",
		"foo and bar unless baz or qux",
		"bar + on(foo) bla / on(baz, buz) group_right(test) blub",
		"foo / ignoring(test,blub) group_left(blub) bar",
		"foo - ignoring(test,blub) group_right(bar,foo) bar",
		`foo{a="b", foo!="bar", test=~"test", bar!~"baz"}`,
		`min_over_time(rate(foo{bar="baz"}[2s])[5m:])[4m:3s]`,
		"sum without(and, by, avg, count, alert, annotations)(some_metric) [30m:10s]",
	}
	errCases := []string{
		"(",
		"}",
		"1 or 1",
		"1 or on(bar) foo",
		"foo unless on(bar) group_left(baz) bar",
		"test[5d] OFFSET 10s [10m:5s]",
	}

	for _, c := range cases {
		b.Run(c, func(b *testing.B) {
			b.ReportAllocs()
			for i := 0; i < b.N; i++ {
				ParseMetric(c)
			}
		})
	}
	for _, c := range errCases {
		name := fmt.Sprintf("%s (should fail)", c)
		b.Run(name, func(b *testing.B) {
			b.ReportAllocs()
			for i := 0; i < b.N; i++ {
				ParseMetric(c)
			}
		})
	}
}
<|endoftext|>


<commit_before>package protocol

import (
	"fmt"
	"io"
)

func ReadResponse(r io.Reader, apiKey ApiKey, apiVersion int16) (correlationID int32, msg Message, err error) {
	if i := int(apiKey); i < 0 || i >= len(apiTypes) {
		err = fmt.Errorf("unsupported api key: %d", i)
		return
	}

	t := &apiTypes[apiKey]
	if t == nil {
		err = fmt.Errorf("unsupported api: %s", apiNames[apiKey])
		return
	}

	minVersion := t.minVersion()
	maxVersion := t.maxVersion()

	if apiVersion < minVersion || apiVersion > maxVersion {
		err = fmt.Errorf("unsupported %s version: v%d not in range v%d-v%d", apiKey, apiVersion, minVersion, maxVersion)
		return
	}

	d := &decoder{reader: r, remain: 4}
	size := d.readInt32()

	if err = d.err; err != nil {
		err = dontExpectEOF(err)
		return
	}

	d.remain = int(size)
	correlationID = d.readInt32()

	res := &t.responses[apiVersion-minVersion]

	if res.flexible {
		// In the flexible case, there's a tag buffer at the end of the response header
		taggedCount := int(d.readUnsignedVarInt())
		for i := 0; i < taggedCount; i++ {
			d.readUnsignedVarInt() // tagID
			size := d.readUnsignedVarInt()

			// Just throw away the values for now
			d.read(int(size))
		}
	}

	msg = res.new()
	res.decode(d, valueOf(msg))
	d.discardAll()

	if err = d.err; err != nil {
		err = dontExpectEOF(err)
	}

	return
}

func WriteResponse(w io.Writer, apiVersion int16, correlationID int32, msg Message) error {
	apiKey := msg.ApiKey()

	if i := int(apiKey); i < 0 || i >= len(apiTypes) {
		return fmt.Errorf("unsupported api key: %d", i)
	}

	t := &apiTypes[apiKey]
	if t == nil {
		return fmt.Errorf("unsupported api: %s", apiNames[apiKey])
	}

	minVersion := t.minVersion()
	maxVersion := t.maxVersion()

	if apiVersion < minVersion || apiVersion > maxVersion {
		return fmt.Errorf("unsupported %s version: v%d not in range v%d-v%d", apiKey, apiVersion, minVersion, maxVersion)
	}

	r := &t.responses[apiVersion-minVersion]
	v := valueOf(msg)
	b := newPageBuffer()
	defer b.unref()

	e := &encoder{writer: b}
	e.writeInt32(0) // placeholder for the response size
	e.writeInt32(correlationID)
	r.encode(e, v)
	err := e.err

	if err == nil {
		size := packUint32(uint32(b.Size()) - 4)
		b.WriteAt(size[:], 0)
		_, err = b.WriteTo(w)
	}

	return err
}
<commit_msg>Write tag buffer size for flixible responses. (#733)<commit_after>package protocol

import (
	"fmt"
	"io"
)

func ReadResponse(r io.Reader, apiKey ApiKey, apiVersion int16) (correlationID int32, msg Message, err error) {
	if i := int(apiKey); i < 0 || i >= len(apiTypes) {
		err = fmt.Errorf("unsupported api key: %d", i)
		return
	}

	t := &apiTypes[apiKey]
	if t == nil {
		err = fmt.Errorf("unsupported api: %s", apiNames[apiKey])
		return
	}

	minVersion := t.minVersion()
	maxVersion := t.maxVersion()

	if apiVersion < minVersion || apiVersion > maxVersion {
		err = fmt.Errorf("unsupported %s version: v%d not in range v%d-v%d", apiKey, apiVersion, minVersion, maxVersion)
		return
	}

	d := &decoder{reader: r, remain: 4}
	size := d.readInt32()

	if err = d.err; err != nil {
		err = dontExpectEOF(err)
		return
	}

	d.remain = int(size)
	correlationID = d.readInt32()

	res := &t.responses[apiVersion-minVersion]

	if res.flexible {
		// In the flexible case, there's a tag buffer at the end of the response header
		taggedCount := int(d.readUnsignedVarInt())
		for i := 0; i < taggedCount; i++ {
			d.readUnsignedVarInt() // tagID
			size := d.readUnsignedVarInt()

			// Just throw away the values for now
			d.read(int(size))
		}
	}

	msg = res.new()
	res.decode(d, valueOf(msg))
	d.discardAll()

	if err = d.err; err != nil {
		err = dontExpectEOF(err)
	}

	return
}

func WriteResponse(w io.Writer, apiVersion int16, correlationID int32, msg Message) error {
	apiKey := msg.ApiKey()

	if i := int(apiKey); i < 0 || i >= len(apiTypes) {
		return fmt.Errorf("unsupported api key: %d", i)
	}

	t := &apiTypes[apiKey]
	if t == nil {
		return fmt.Errorf("unsupported api: %s", apiNames[apiKey])
	}

	minVersion := t.minVersion()
	maxVersion := t.maxVersion()

	if apiVersion < minVersion || apiVersion > maxVersion {
		return fmt.Errorf("unsupported %s version: v%d not in range v%d-v%d", apiKey, apiVersion, minVersion, maxVersion)
	}

	r := &t.responses[apiVersion-minVersion]
	v := valueOf(msg)
	b := newPageBuffer()
	defer b.unref()

	e := &encoder{writer: b}
	e.writeInt32(0) // placeholder for the response size
	e.writeInt32(correlationID)
	if r.flexible {
		// Flexible messages use extra space for a tag buffer,
		// which begins with a size value. Since we're not writing any fields into the
		// latter, we can just write zero for now.
		//
		// See
		// https://cwiki.apache.org/confluence/display/KAFKA/KIP-482%3A+The+Kafka+Protocol+should+Support+Optional+Tagged+Fields
		// for details.
		e.writeUnsignedVarInt(0)
	}
	r.encode(e, v)
	err := e.err

	if err == nil {
		size := packUint32(uint32(b.Size()) - 4)
		b.WriteAt(size[:], 0)
		_, err = b.WriteTo(w)
	}

	return err
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// The provider package holds constants identifying known provider types.
// They have hitherto only been used for nefarious purposes; no new code
// should use them, and when old code is updated to no longer use them
// they must be deleted.
package provider

const (
	Joyent    = "joyent"
	Local     = "local"
	MAAS      = "maas"
)

// IsManual returns true iff the specified provider
// type refers to the manual provider.
func IsManual(provider string) bool {
	switch provider {
	case "null", "manual":
		return true
	default:
		return false
	}
}
<commit_msg>gofmt<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// The provider package holds constants identifying known provider types.
// They have hitherto only been used for nefarious purposes; no new code
// should use them, and when old code is updated to no longer use them
// they must be deleted.
package provider

const (
	Joyent = "joyent"
	Local  = "local"
	MAAS   = "maas"
)

// IsManual returns true iff the specified provider
// type refers to the manual provider.
func IsManual(provider string) bool {
	switch provider {
	case "null", "manual":
		return true
	default:
		return false
	}
}
<|endoftext|>