<commit_before>package main

import (
	"context"
	"fmt"
	"io"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path"
	"path/filepath"
	"time"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
	"github.com/pachyderm/pachyderm/src/server/worker"
	"google.golang.org/grpc"

	log "github.com/sirupsen/logrus"
)

// appEnv stores the environment variables that this worker needs
type appEnv struct {
	// Address of etcd, so that worker can write its own IP there for discoverh
	EtcdAddress string `env:"ETCD_PORT_2379_TCP_ADDR,required"`

	// Address for connecting to pachd (so this can download input data)
	PachdAddress string `env:"PACHD_PORT_650_TCP_ADDR,required"`

	// Prefix in etcd for all pachd-related records
	PPSPrefix string `env:"PPS_ETCD_PREFIX,required"`

	// worker gets its own IP here, via the k8s downward API. It then writes that
	// IP back to etcd so that pachd can discover it
	PPSWorkerIP string `env:"PPS_WORKER_IP,required"`

	// The name of the pipeline that this worker belongs to
	PPSPipelineName string `env:"PPS_PIPELINE_NAME,required"`

	// The ID of the commit that contains the pipeline spec.
	PPSSpecCommitID string `env:"PPS_SPEC_COMMIT,required"`

	// The name of this pod
	PodName string `env:"PPS_POD_NAME,required"`

	// The namespace in which Pachyderm is deployed
	Namespace string `env:"PPS_NAMESPACE,required"`
}

func main() {
	// Copy the contents of /pach-bin/certs into /etc/ssl/certs
	if info, err := os.Stat("/etc/ssl/certs"); err != nil || !info.IsDir() {
		os.MkdirAll("/etc/ssl/certs", 0755)
	}
	if err := filepath.Walk("/pach-bin/certs", func(inPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err // Don't try and fix any errors encountered by Walk() itself
		}
		if info.IsDir() {
			return nil // We'll just copy the children of any directories when we traverse them
		}

		// Create output file (dest)
		outRelPath, err := filepath.Rel("/pach-bin/certs", inPath)
		if err != nil {
			return fmt.Errorf("could not extract relative path: %v", err)
		}
		outPath := filepath.Join("/etc/ssl/certs", outRelPath)
		out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, info.Mode())
		if err != nil {
			return fmt.Errorf("could not create %s: %v", outPath, err)
		}
		defer out.Close()

		// Open input file (src)
		in, err := os.OpenFile(inPath, os.O_RDONLY, 0)
		if err != nil {
			return fmt.Errorf("could not read %s: %v", inPath, err)
		}
		defer in.Close()

		// Copy src -> dest
		if _, err := io.Copy(out, in); err != nil {
			return fmt.Errorf("could not copy %s to %s: %v", inPath, outPath, err)
		}
		return nil
	}); err != nil {
		panic(fmt.Sprintf("could not copy /pach-bin/certs to : %v", err))
	}
	cmdutil.Main(do, &appEnv{})
}

// getPipelineInfo gets the PipelineInfo proto describing the pipeline that this
// worker is part of.
// getPipelineInfo has the side effect of adding auth to the passed pachClient
// which is necessary to get the PipelineInfo from pfs.
func getPipelineInfo(etcdClient *etcd.Client, pachClient *client.APIClient, appEnv *appEnv) (*pps.PipelineInfo, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "pipelines", appEnv.PPSPipelineName))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 pipeline (%s), got %d: %v", appEnv.PPSPipelineName, len(resp.Kvs), resp)
	}
	var pipelinePtr pps.EtcdPipelineInfo
	if err := pipelinePtr.Unmarshal(resp.Kvs[0].Value); err != nil {
		return nil, err
	}
	pachClient.SetAuthToken(pipelinePtr.AuthToken)
	// Notice we use the SpecCommitID from our env, not from etcd. This is
	// because the value in etcd might get updated while the worker pod is
	// being created and we don't want to run the transform of one version of
	// the pipeline in the image of a different verison.
	pipelinePtr.SpecCommit.ID = appEnv.PPSSpecCommitID
	return ppsutil.GetPipelineInfo(pachClient, &pipelinePtr)
}

func do(appEnvObj interface{}) error {
	go func() {
		log.Println(http.ListenAndServe(":651", nil))
	}()

	appEnv := appEnvObj.(*appEnv)

	// Construct a client that connects to the sidecar.
	pachClient, err := client.NewFromAddress("localhost:650")
	if err != nil {
		return fmt.Errorf("error constructing pachClient: %v", err)
	}

	// Get etcd client, so we can register our IP (so pachd can discover us)
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{fmt.Sprintf("%s:2379", appEnv.EtcdAddress)},
		DialOptions: client.EtcdDialOptions(),
	})
	if err != nil {
		return fmt.Errorf("error constructing etcdClient: %v", err)
	}

	pipelineInfo, err := getPipelineInfo(etcdClient, pachClient, appEnv)
	if err != nil {
		return fmt.Errorf("error getting pipelineInfo: %v", err)
	}

	// Construct worker API server.
	workerRcName := ppsutil.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	apiServer, err := worker.NewAPIServer(pachClient, etcdClient, appEnv.PPSPrefix, pipelineInfo, appEnv.PodName, appEnv.Namespace)
	if err != nil {
		return err
	}

	// Start worker api server
	eg := errgroup.Group{}
	ready := make(chan error)
	eg.Go(func() error {
		return grpcutil.Serve(
			func(s *grpc.Server) {
				worker.RegisterWorkerServer(s, apiServer)
				close(ready)
			},
			grpcutil.ServeOptions{
				Version:    version.Version,
				MaxMsgSize: grpcutil.MaxMsgSize,
			},
			grpcutil.ServeEnv{
				GRPCPort: client.PPSWorkerPort,
			},
		)
	})

	// Wait until server is ready, then put our IP address into etcd, so pachd can
	// discover us
	<-ready
	key := path.Join(appEnv.PPSPrefix, "workers", workerRcName, appEnv.PPSWorkerIP)

	// Prepare to write "key" into etcd by creating lease -- if worker dies, our
	// IP will be removed from etcd
	ctx, cancel := context.WithTimeout(pachClient.Ctx(), 10*time.Second)
	defer cancel()
	resp, err := etcdClient.Grant(ctx, 10 /* seconds */)
	if err != nil {
		return fmt.Errorf("error granting lease: %v", err)
	}

	// keepalive forever
	if _, err := etcdClient.KeepAlive(context.Background(), resp.ID); err != nil {
		return fmt.Errorf("error with KeepAlive: %v", err)
	}

	// Actually write "key" into etcd
	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) // new ctx
	defer cancel()
	if _, err := etcdClient.Put(ctx, key, "", etcd.WithLease(resp.ID)); err != nil {
		return fmt.Errorf("error putting IP address: %v", err)
	}

	// If server ever exits, return error
	return eg.Wait()
}
<commit_msg>Fix broken error message<commit_after>package main

import (
	"context"
	"fmt"
	"io"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path"
	"path/filepath"
	"time"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
	"github.com/pachyderm/pachyderm/src/server/worker"
	"google.golang.org/grpc"

	log "github.com/sirupsen/logrus"
)

// appEnv stores the environment variables that this worker needs
type appEnv struct {
	// Address of etcd, so that worker can write its own IP there for discoverh
	EtcdAddress string `env:"ETCD_PORT_2379_TCP_ADDR,required"`

	// Address for connecting to pachd (so this can download input data)
	PachdAddress string `env:"PACHD_PORT_650_TCP_ADDR,required"`

	// Prefix in etcd for all pachd-related records
	PPSPrefix string `env:"PPS_ETCD_PREFIX,required"`

	// worker gets its own IP here, via the k8s downward API. It then writes that
	// IP back to etcd so that pachd can discover it
	PPSWorkerIP string `env:"PPS_WORKER_IP,required"`

	// The name of the pipeline that this worker belongs to
	PPSPipelineName string `env:"PPS_PIPELINE_NAME,required"`

	// The ID of the commit that contains the pipeline spec.
	PPSSpecCommitID string `env:"PPS_SPEC_COMMIT,required"`

	// The name of this pod
	PodName string `env:"PPS_POD_NAME,required"`

	// The namespace in which Pachyderm is deployed
	Namespace string `env:"PPS_NAMESPACE,required"`
}

func main() {
	// Copy the contents of /pach-bin/certs into /etc/ssl/certs
	if info, err := os.Stat("/etc/ssl/certs"); err != nil || !info.IsDir() {
		os.MkdirAll("/etc/ssl/certs", 0755)
	}
	if err := filepath.Walk("/pach-bin/certs", func(inPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err // Don't try and fix any errors encountered by Walk() itself
		}
		if info.IsDir() {
			return nil // We'll just copy the children of any directories when we traverse them
		}

		// Create output file (dest)
		outRelPath, err := filepath.Rel("/pach-bin/certs", inPath)
		if err != nil {
			return fmt.Errorf("could not extract relative path: %v", err)
		}
		outPath := filepath.Join("/etc/ssl/certs", outRelPath)
		out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, info.Mode())
		if err != nil {
			return fmt.Errorf("could not create %s: %v", outPath, err)
		}
		defer out.Close()

		// Open input file (src)
		in, err := os.OpenFile(inPath, os.O_RDONLY, 0)
		if err != nil {
			return fmt.Errorf("could not read %s: %v", inPath, err)
		}
		defer in.Close()

		// Copy src -> dest
		if _, err := io.Copy(out, in); err != nil {
			return fmt.Errorf("could not copy %s to %s: %v", inPath, outPath, err)
		}
		return nil
	}); err != nil {
		panic(fmt.Sprintf("could not copy /pach-bin/certs to /etc/ssl/certs: %v", err))
	}
	cmdutil.Main(do, &appEnv{})
}

// getPipelineInfo gets the PipelineInfo proto describing the pipeline that this
// worker is part of.
// getPipelineInfo has the side effect of adding auth to the passed pachClient
// which is necessary to get the PipelineInfo from pfs.
func getPipelineInfo(etcdClient *etcd.Client, pachClient *client.APIClient, appEnv *appEnv) (*pps.PipelineInfo, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "pipelines", appEnv.PPSPipelineName))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 pipeline (%s), got %d: %v", appEnv.PPSPipelineName, len(resp.Kvs), resp)
	}
	var pipelinePtr pps.EtcdPipelineInfo
	if err := pipelinePtr.Unmarshal(resp.Kvs[0].Value); err != nil {
		return nil, err
	}
	pachClient.SetAuthToken(pipelinePtr.AuthToken)
	// Notice we use the SpecCommitID from our env, not from etcd. This is
	// because the value in etcd might get updated while the worker pod is
	// being created and we don't want to run the transform of one version of
	// the pipeline in the image of a different verison.
	pipelinePtr.SpecCommit.ID = appEnv.PPSSpecCommitID
	return ppsutil.GetPipelineInfo(pachClient, &pipelinePtr)
}

func do(appEnvObj interface{}) error {
	go func() {
		log.Println(http.ListenAndServe(":651", nil))
	}()

	appEnv := appEnvObj.(*appEnv)

	// Construct a client that connects to the sidecar.
	pachClient, err := client.NewFromAddress("localhost:650")
	if err != nil {
		return fmt.Errorf("error constructing pachClient: %v", err)
	}

	// Get etcd client, so we can register our IP (so pachd can discover us)
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{fmt.Sprintf("%s:2379", appEnv.EtcdAddress)},
		DialOptions: client.EtcdDialOptions(),
	})
	if err != nil {
		return fmt.Errorf("error constructing etcdClient: %v", err)
	}

	pipelineInfo, err := getPipelineInfo(etcdClient, pachClient, appEnv)
	if err != nil {
		return fmt.Errorf("error getting pipelineInfo: %v", err)
	}

	// Construct worker API server.
	workerRcName := ppsutil.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	apiServer, err := worker.NewAPIServer(pachClient, etcdClient, appEnv.PPSPrefix, pipelineInfo, appEnv.PodName, appEnv.Namespace)
	if err != nil {
		return err
	}

	// Start worker api server
	eg := errgroup.Group{}
	ready := make(chan error)
	eg.Go(func() error {
		return grpcutil.Serve(
			func(s *grpc.Server) {
				worker.RegisterWorkerServer(s, apiServer)
				close(ready)
			},
			grpcutil.ServeOptions{
				Version:    version.Version,
				MaxMsgSize: grpcutil.MaxMsgSize,
			},
			grpcutil.ServeEnv{
				GRPCPort: client.PPSWorkerPort,
			},
		)
	})

	// Wait until server is ready, then put our IP address into etcd, so pachd can
	// discover us
	<-ready
	key := path.Join(appEnv.PPSPrefix, "workers", workerRcName, appEnv.PPSWorkerIP)

	// Prepare to write "key" into etcd by creating lease -- if worker dies, our
	// IP will be removed from etcd
	ctx, cancel := context.WithTimeout(pachClient.Ctx(), 10*time.Second)
	defer cancel()
	resp, err := etcdClient.Grant(ctx, 10 /* seconds */)
	if err != nil {
		return fmt.Errorf("error granting lease: %v", err)
	}

	// keepalive forever
	if _, err := etcdClient.KeepAlive(context.Background(), resp.ID); err != nil {
		return fmt.Errorf("error with KeepAlive: %v", err)
	}

	// Actually write "key" into etcd
	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) // new ctx
	defer cancel()
	if _, err := etcdClient.Put(ctx, key, "", etcd.WithLease(resp.ID)); err != nil {
		return fmt.Errorf("error putting IP address: %v", err)
	}

	// If server ever exits, return error
	return eg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path"
	"path/filepath"
	"time"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/pachyderm/pachyderm/src/client"
	debugclient "github.com/pachyderm/pachyderm/src/client/debug"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/client/version/versionpb"
	debugserver "github.com/pachyderm/pachyderm/src/server/debug/server"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	logutil "github.com/pachyderm/pachyderm/src/server/pkg/log"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/serviceenv"
	"github.com/pachyderm/pachyderm/src/server/worker"
	"google.golang.org/grpc"

	log "github.com/sirupsen/logrus"
)

func main() {
	log.SetFormatter(logutil.FormatterFunc(logutil.Pretty))

	// Copy the contents of /pach-bin/certs into /etc/ssl/certs. Don't return an
	// error (which would cause 'Walk()' to exit early) but do record if any certs
	// are known to be missing so we can inform the user
	copyErr := false
	if err := filepath.Walk("/pach-bin/certs", func(inPath string, info os.FileInfo, err error) error {
		if err != nil {
			log.Warnf("skipping \"%s\", could not stat path: %v", inPath, err)
			copyErr = true
			return nil // Don't try and fix any errors encountered by Walk() itself
		}
		if info.IsDir() {
			return nil // We'll just copy the children of any directories when we traverse them
		}

		// Open input file (src)
		in, err := os.OpenFile(inPath, os.O_RDONLY, 0)
		if err != nil {
			log.Warnf("could not read \"%s\": %v", inPath, err)
			copyErr = true
			return nil
		}
		defer in.Close()

		// Create output file (dest) and open for writing
		outRelPath, err := filepath.Rel("/pach-bin/certs", inPath)
		if err != nil {
			log.Warnf("skipping \"%s\", could not extract relative path: %v", inPath, err)
			copyErr = true
			return nil
		}
		outPath := filepath.Join("/etc/ssl/certs", outRelPath)
		outDir := filepath.Dir(outPath)
		if err := os.MkdirAll(outDir, 0755); err != nil {
			log.Warnf("skipping \"%s\", could not create directory \"%s\": %v", inPath, outDir, err)
			copyErr = true
			return nil
		}
		out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode())
		if err != nil {
			log.Warnf("skipping \"%s\", could not create output file \"%s\": %v", inPath, outPath, err)
			copyErr = true
			return nil
		}
		defer out.Close()

		// Copy src -> dest
		if _, err := io.Copy(out, in); err != nil {
			log.Warnf("could not copy \"%s\" to \"%s\": %v", inPath, outPath, err)
			copyErr = true
			return nil
		}
		return nil
	}); err != nil {
		// Should never happen
		log.Warnf("could not copy /pach-bin/certs to /etc/ssl/certs: %v", err)
	}
	if copyErr {
		log.Warnf("Errors were encountered while copying /pach-bin/certs to /etc/ssl/certs (see above--might result in subsequent SSL/TLS errors)")
	}
	cmdutil.Main(do, &serviceenv.WorkerFullConfiguration{})
}

// getPipelineInfo gets the PipelineInfo proto describing the pipeline that this
// worker is part of.
// getPipelineInfo has the side effect of adding auth to the passed pachClient
// which is necessary to get the PipelineInfo from pfs.
func getPipelineInfo(env *serviceenv.ServiceEnv) (*pps.PipelineInfo, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	resp, err := env.GetEtcdClient().Get(ctx, path.Join(env.PPSEtcdPrefix, "pipelines", env.PPSPipelineName))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 pipeline (%s), got %d: %v", env.PPSPipelineName, len(resp.Kvs), resp)
	}
	var pipelinePtr pps.EtcdPipelineInfo
	if err := pipelinePtr.Unmarshal(resp.Kvs[0].Value); err != nil {
		return nil, err
	}
	pachClient := env.GetPachClient(ctx)
	pachClient.SetAuthToken(pipelinePtr.AuthToken)
	// Notice we use the SpecCommitID from our env, not from etcd. This is
	// because the value in etcd might get updated while the worker pod is
	// being created and we don't want to run the transform of one version of
	// the pipeline in the image of a different verison.
	pipelinePtr.SpecCommit.ID = env.PPSSpecCommitID
	return ppsutil.GetPipelineInfo(pachClient, &pipelinePtr)
}

func do(config interface{}) error {
	env := serviceenv.InitServiceEnv(serviceenv.NewConfiguration(config))
	// Expose PProf service
	go func() {
		log.Println(http.ListenAndServe(fmt.Sprintf(":%d", env.PProfPort), nil))
	}()

	// Construct a client that connects to the sidecar.
	pachClient, err := client.NewFromAddress(net.JoinHostPort("localhost", fmt.Sprintf("%d", env.PeerPort)))
	if err != nil {
		return fmt.Errorf("error constructing pachClient: %v", err)
	}

	// Get etcd client, so we can register our IP (so pachd can discover us)
	pipelineInfo, err := getPipelineInfo(env)
	if err != nil {
		return fmt.Errorf("error getting pipelineInfo: %v", err)
	}

	// Construct worker API server.
	workerRcName := ppsutil.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	apiServer, err := worker.NewAPIServer(env.GetPachClient(context.Background()), env.GetEtcdClient(), env.PPSEtcdPrefix, pipelineInfo, env.PodName, env.Namespace, env.StorageRoot)
	if err != nil {
		return err
	}

	// Start worker api server
	eg := errgroup.Group{}
	ready := make(chan error)
	eg.Go(func() error {
		return grpcutil.Serve(
			grpcutil.ServerOptions{
				MaxMsgSize: grpcutil.MaxMsgSize,
				Port:       env.PPSWorkerPort,
				RegisterFunc: func(s *grpc.Server) error {
					defer close(ready)
					worker.RegisterWorkerServer(s, apiServer)
					versionpb.RegisterAPIServer(s, version.NewAPIServer(version.Version, version.APIServerOptions{}))
					debugclient.RegisterDebugServer(s, debugserver.NewDebugServer(env.PodName, env.GetEtcdClient(), env.PPSEtcdPrefix, env.PPSWorkerPort))
					return nil
				},
			},
		)
	})

	// Wait until server is ready, then put our IP address into etcd, so pachd can
	// discover us
	<-ready
	key := path.Join(env.PPSEtcdPrefix, worker.WorkerEtcdPrefix, workerRcName, env.PPSWorkerIP)

	// Prepare to write "key" into etcd by creating lease -- if worker dies, our
	// IP will be removed from etcd
	ctx, cancel := context.WithTimeout(pachClient.Ctx(), 10*time.Second)
	defer cancel()
	resp, err := env.GetEtcdClient().Grant(ctx, 10 /* seconds */)
	if err != nil {
		return fmt.Errorf("error granting lease: %v", err)
	}

	// keepalive forever
	if _, err := env.GetEtcdClient().KeepAlive(context.Background(), resp.ID); err != nil {
		return fmt.Errorf("error with KeepAlive: %v", err)
	}

	// Actually write "key" into etcd
	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) // new ctx
	defer cancel()
	if _, err := env.GetEtcdClient().Put(ctx, key, "", etcd.WithLease(resp.ID)); err != nil {
		return fmt.Errorf("error putting IP address: %v", err)
	}

	// If server ever exits, return error
	return eg.Wait()
}
<commit_msg>Auth token needs to be in pach client for worker<commit_after>package main

import (
	"context"
	"fmt"
	"io"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path"
	"path/filepath"
	"time"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/pachyderm/pachyderm/src/client"
	debugclient "github.com/pachyderm/pachyderm/src/client/debug"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/client/version/versionpb"
	debugserver "github.com/pachyderm/pachyderm/src/server/debug/server"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	logutil "github.com/pachyderm/pachyderm/src/server/pkg/log"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/serviceenv"
	"github.com/pachyderm/pachyderm/src/server/worker"
	"google.golang.org/grpc"

	log "github.com/sirupsen/logrus"
)

func main() {
	log.SetFormatter(logutil.FormatterFunc(logutil.Pretty))

	// Copy the contents of /pach-bin/certs into /etc/ssl/certs. Don't return an
	// error (which would cause 'Walk()' to exit early) but do record if any certs
	// are known to be missing so we can inform the user
	copyErr := false
	if err := filepath.Walk("/pach-bin/certs", func(inPath string, info os.FileInfo, err error) error {
		if err != nil {
			log.Warnf("skipping \"%s\", could not stat path: %v", inPath, err)
			copyErr = true
			return nil // Don't try and fix any errors encountered by Walk() itself
		}
		if info.IsDir() {
			return nil // We'll just copy the children of any directories when we traverse them
		}

		// Open input file (src)
		in, err := os.OpenFile(inPath, os.O_RDONLY, 0)
		if err != nil {
			log.Warnf("could not read \"%s\": %v", inPath, err)
			copyErr = true
			return nil
		}
		defer in.Close()

		// Create output file (dest) and open for writing
		outRelPath, err := filepath.Rel("/pach-bin/certs", inPath)
		if err != nil {
			log.Warnf("skipping \"%s\", could not extract relative path: %v", inPath, err)
			copyErr = true
			return nil
		}
		outPath := filepath.Join("/etc/ssl/certs", outRelPath)
		outDir := filepath.Dir(outPath)
		if err := os.MkdirAll(outDir, 0755); err != nil {
			log.Warnf("skipping \"%s\", could not create directory \"%s\": %v", inPath, outDir, err)
			copyErr = true
			return nil
		}
		out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode())
		if err != nil {
			log.Warnf("skipping \"%s\", could not create output file \"%s\": %v", inPath, outPath, err)
			copyErr = true
			return nil
		}
		defer out.Close()

		// Copy src -> dest
		if _, err := io.Copy(out, in); err != nil {
			log.Warnf("could not copy \"%s\" to \"%s\": %v", inPath, outPath, err)
			copyErr = true
			return nil
		}
		return nil
	}); err != nil {
		// Should never happen
		log.Warnf("could not copy /pach-bin/certs to /etc/ssl/certs: %v", err)
	}
	if copyErr {
		log.Warnf("Errors were encountered while copying /pach-bin/certs to /etc/ssl/certs (see above--might result in subsequent SSL/TLS errors)")
	}
	cmdutil.Main(do, &serviceenv.WorkerFullConfiguration{})
}

// getPipelineInfo gets the PipelineInfo proto describing the pipeline that this
// worker is part of.
// getPipelineInfo has the side effect of adding auth to the passed pachClient
// which is necessary to get the PipelineInfo from pfs.
func getPipelineInfo(pachClient *client.APIClient, env *serviceenv.ServiceEnv) (*pps.PipelineInfo, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	resp, err := env.GetEtcdClient().Get(ctx, path.Join(env.PPSEtcdPrefix, "pipelines", env.PPSPipelineName))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 pipeline (%s), got %d: %v", env.PPSPipelineName, len(resp.Kvs), resp)
	}
	var pipelinePtr pps.EtcdPipelineInfo
	if err := pipelinePtr.Unmarshal(resp.Kvs[0].Value); err != nil {
		return nil, err
	}
	pachClient.SetAuthToken(pipelinePtr.AuthToken)
	// Notice we use the SpecCommitID from our env, not from etcd. This is
	// because the value in etcd might get updated while the worker pod is
	// being created and we don't want to run the transform of one version of
	// the pipeline in the image of a different verison.
	pipelinePtr.SpecCommit.ID = env.PPSSpecCommitID
	return ppsutil.GetPipelineInfo(pachClient, &pipelinePtr)
}

func do(config interface{}) error {
	env := serviceenv.InitServiceEnv(serviceenv.NewConfiguration(config))
	// Expose PProf service
	go func() {
		log.Println(http.ListenAndServe(fmt.Sprintf(":%d", env.PProfPort), nil))
	}()

	// Construct a client that connects to the sidecar.
	pachClient := env.GetPachClient(context.Background())

	// Get etcd client, so we can register our IP (so pachd can discover us)
	pipelineInfo, err := getPipelineInfo(pachClient, env)
	if err != nil {
		return fmt.Errorf("error getting pipelineInfo: %v", err)
	}

	// Construct worker API server.
	workerRcName := ppsutil.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	apiServer, err := worker.NewAPIServer(pachClient, env.GetEtcdClient(), env.PPSEtcdPrefix, pipelineInfo, env.PodName, env.Namespace, env.StorageRoot)
	if err != nil {
		return err
	}

	// Start worker api server
	eg := errgroup.Group{}
	ready := make(chan error)
	eg.Go(func() error {
		return grpcutil.Serve(
			grpcutil.ServerOptions{
				MaxMsgSize: grpcutil.MaxMsgSize,
				Port:       env.PPSWorkerPort,
				RegisterFunc: func(s *grpc.Server) error {
					defer close(ready)
					worker.RegisterWorkerServer(s, apiServer)
					versionpb.RegisterAPIServer(s, version.NewAPIServer(version.Version, version.APIServerOptions{}))
					debugclient.RegisterDebugServer(s, debugserver.NewDebugServer(env.PodName, env.GetEtcdClient(), env.PPSEtcdPrefix, env.PPSWorkerPort))
					return nil
				},
			},
		)
	})

	// Wait until server is ready, then put our IP address into etcd, so pachd can
	// discover us
	<-ready
	key := path.Join(env.PPSEtcdPrefix, worker.WorkerEtcdPrefix, workerRcName, env.PPSWorkerIP)

	// Prepare to write "key" into etcd by creating lease -- if worker dies, our
	// IP will be removed from etcd
	ctx, cancel := context.WithTimeout(pachClient.Ctx(), 10*time.Second)
	defer cancel()
	resp, err := env.GetEtcdClient().Grant(ctx, 10 /* seconds */)
	if err != nil {
		return fmt.Errorf("error granting lease: %v", err)
	}

	// keepalive forever
	if _, err := env.GetEtcdClient().KeepAlive(context.Background(), resp.ID); err != nil {
		return fmt.Errorf("error with KeepAlive: %v", err)
	}

	// Actually write "key" into etcd
	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) // new ctx
	defer cancel()
	if _, err := env.GetEtcdClient().Put(ctx, key, "", etcd.WithLease(resp.ID)); err != nil {
		return fmt.Errorf("error putting IP address: %v", err)
	}

	// If server ever exits, return error
	return eg.Wait()
}
<|endoftext|>


<commit_before>// Copyright (c) Liam Stanley <me@liamstanley.io>. All rights reserved. Use
// of this source code is governed by the MIT license that can be found in
// the LICENSE file.

package girc

import (
	"bytes"
	"fmt"
	"strings"
	"time"
)

const (
	eventSpace byte = 0x20 // Separator.
	maxLength       = 510  // Maximum length is 510 (2 for line endings).
)

// cutCRFunc is used to trim CR characters from prefixes/messages.
func cutCRFunc(r rune) bool {
	return r == '\r' || r == '\n'
}

// Event represents an IRC protocol message, see RFC1459 section 2.3.1
//
//    <message>  :: [':' <prefix> <SPACE>] <command> <params> <crlf>
//    <prefix>   :: <servername> | <nick> ['!' <user>] ['@' <host>]
//    <command>  :: <letter>{<letter>} | <number> <number> <number>
//    <SPACE>    :: ' '{' '}
//    <params>   :: <SPACE> [':' <trailing> | <middle> <params>]
//    <middle>   :: <Any *non-empty* sequence of octets not including SPACE or NUL
//                   or CR or LF, the first of which may not be ':'>
//    <trailing> :: <Any, possibly empty, sequence of octets not including NUL or
//                   CR or LF>
//    <crlf>     :: CR LF
type Event struct {
	Source        *Source  // The source of the event.
	Tags          Tags     // IRCv3 style message tags. Only use if network supported.
	Command       string   // the IRC command, e.g. JOIN, PRIVMSG, KILL.
	Params        []string // parameters to the command. Commonly nickname, channel, etc.
	Trailing      string   // any trailing data. e.g. with a PRIVMSG, this is the message text.
	EmptyTrailing bool     // if true, trailing prefix (:) will be added even if Event.Trailing is empty.
	Sensitive     bool     // if the message is sensitive (e.g. and should not be logged).
}

// ParseEvent takes a string and attempts to create a Event struct.
//
// Returns nil if the Event is invalid.
func ParseEvent(raw string) (e *Event) {
	// Ignore empty events.
	if raw = strings.TrimFunc(raw, cutCRFunc); len(raw) < 2 {
		return nil
	}

	i, j := 0, 0
	e = &Event{}

	if raw[0] == prefixTag {
		// Tags end with a space.
		i = strings.IndexByte(raw, eventSpace)

		if i < 2 {
			return nil
		}

		e.Tags = ParseTags(raw[1:i])
		raw = raw[i+1:]
	}

	if raw[0] == messagePrefix {
		// Prefix ends with a space.
		i = strings.IndexByte(raw, eventSpace)

		// Prefix string must not be empty if the indicator is present.
		if i < 2 {
			return nil
		}

		e.Source = ParseSource(raw[1:i])

		// Skip space at the end of the prefix.
		i++
	}

	// Find end of command.
	j = i + strings.IndexByte(raw[i:], eventSpace)

	// Extract command.
	if j < i {
		e.Command = strings.ToUpper(raw[i:])
		return e
	}

	e.Command = strings.ToUpper(raw[i:j])
	// Skip space after command.
	j++

	// Find prefix for trailer.
	i = bytes.Index([]byte(raw[j:]), []byte{eventSpace, messagePrefix})
	if i != -1 {
		i += 1
	}

	if i < 0 || raw[j+i-1] != eventSpace {
		// No trailing argument.
		e.Params = strings.Split(raw[j:], string(eventSpace))
		return e
	}

	// Compensate for index on substring.
	i = i + j

	// Check if we need to parse arguments.
	if i > j {
		e.Params = strings.Split(raw[j:i-1], string(eventSpace))
	}

	e.Trailing = raw[i+1:]

	// We need to re-encode the trailing argument even if it was empty.
	if len(e.Trailing) <= 0 {
		e.EmptyTrailing = true
	}

	return e
}

// Len calculates the length of the string representation of event.
func (e *Event) Len() (length int) {
	if e.Tags != nil {
		// Include tags and trailing space.
		length = e.Tags.Len() + 1
	}
	if e.Source != nil {
		// Include prefix and trailing space.
		length += e.Source.Len() + 2
	}

	length += len(e.Command)

	if len(e.Params) > 0 {
		length += len(e.Params)

		for i := 0; i < len(e.Params); i++ {
			length += len(e.Params[i])
		}
	}

	if len(e.Trailing) > 0 || e.EmptyTrailing {
		// Include prefix and space.
		length += len(e.Trailing) + 2
	}

	return
}

// Bytes returns a []byte representation of event. Strips all newlines and
// carriage returns.
//
// Per RFC2812 section 2.3, messages should not exceed 512 characters in
// length. This method forces that limit by discarding any characters
// exceeding the length limit.
func (e *Event) Bytes() []byte {
	buffer := new(bytes.Buffer)

	// Tags.
	if e.Tags != nil {
		e.Tags.writeTo(buffer)
	}

	// Event prefix.
	if e.Source != nil {
		buffer.WriteByte(messagePrefix)
		e.Source.writeTo(buffer)
		buffer.WriteByte(eventSpace)
	}

	// Command is required.
	buffer.WriteString(e.Command)

	// Space separated list of arguments.
	if len(e.Params) > 0 {
		buffer.WriteByte(eventSpace)
		buffer.WriteString(strings.Join(e.Params, string(eventSpace)))
	}

	if len(e.Trailing) > 0 || e.EmptyTrailing {
		buffer.WriteByte(eventSpace)
		buffer.WriteByte(messagePrefix)
		buffer.WriteString(e.Trailing)
	}

	// We need the limit the buffer length.
	if buffer.Len() > (maxLength) {
		if e.Tags != nil {
			// regular message, max tag length, and the splitting space.
			buffer.Truncate(maxLength + maxTagLength + 1)
		} else {
			buffer.Truncate(maxLength)
		}
	}

	out := buffer.Bytes()

	// Strip newlines and carriage returns.
	for i := 0; i < len(out); i++ {
		if out[i] == 0x0A || out[i] == 0x0D {
			out = append(out[:i], out[i+1:]...)
			i-- // Decrease the index so we can pick up where we left off.
		}
	}

	return out
}

// String returns a string representation of this event. Strips all newlines
// and carriage returns.
func (e *Event) String() string {
	return string(e.Bytes())
}

// Pretty returns a prettified string of the event. If the event doesn't
// support prettification, ok is false. Pretty is not just useful to make
// an event prettier, but also to filter out events that most don't visually
// see in normal IRC clients. e.g. most clients don't show WHO queries.
func (e *Event) Pretty() (out string, ok bool) {
	if e.Command == INITIALIZED {
		return fmt.Sprintf("[*] connection to %s initialized", e.Trailing), true
	}

	if e.Command == CONNECTED {
		return fmt.Sprintf("[*] successfully connected to %s", e.Trailing), true
	}

	if (e.Command == PRIVMSG || e.Command == NOTICE) && len(e.Params) > 0 {
		return fmt.Sprintf("[%s] (%s) %s", strings.Join(e.Params, ","), e.Source.Name, e.Trailing), true
	}

	if e.Command == RPL_MOTD || e.Command == RPL_MOTDSTART ||
		e.Command == RPL_WELCOME || e.Command == RPL_YOURHOST ||
		e.Command == RPL_CREATED || e.Command == RPL_LUSERCLIENT {
		return fmt.Sprintf("[*] " + e.Trailing), true
	}

	if e.Command == JOIN && len(e.Params) > 0 {
		return fmt.Sprintf("[*] %s has joined %s", e.Source.Name, e.Params[0]), true
	}

	if e.Command == PART && len(e.Params) > 0 {
		return fmt.Sprintf("[*] %s has left %s (%s)", e.Source.Name, e.Params[0], e.Trailing), true
	}

	if e.Command == ERROR {
		return fmt.Sprintf("[*] an error occurred: %s", e.Trailing), true
	}

	if e.Command == QUIT {
		return fmt.Sprintf("[*] %s has quit (%s)", e.Source.Name, e.Trailing), true
	}

	if e.Command == KICK && len(e.Params) == 2 {
		return fmt.Sprintf("[%s] *** %s has kicked %s: %s", e.Params[0], e.Source.Name, e.Params[1], e.Trailing), true
	}

	if e.Command == NICK && len(e.Params) == 1 {
		return fmt.Sprintf("[*] %s is now known as %s", e.Source.Name, e.Params[0]), true
	}

	if e.Command == TOPIC && len(e.Params) > 0 {
		return fmt.Sprintf("[%s] *** %s has set the topic to: %s", e.Params[len(e.Params)-1], e.Source.Name, e.Trailing), true
	}

	if e.Command == MODE && len(e.Params) > 2 {
		return fmt.Sprintf("[%s] %s set modes: %s", e.Params[0], e.Source.Name, strings.Join(e.Params[1:], " ")), true
	}

	if e.Command == AWAY {
		if len(e.Trailing) > 0 {
			return fmt.Sprintf("[*] %s is now away: %s", e.Source.Name, e.Trailing), true
		}

		return fmt.Sprintf("[*] %s is no longer away", e.Source.Name), true
	}

	return "", false
}

// IsAction checks to see if the event is a PRIVMSG, and is an ACTION (/me).
func (e *Event) IsAction() bool {
	if len(e.Trailing) <= 0 || e.Command != PRIVMSG {
		return false
	}

	if !strings.HasPrefix(e.Trailing, "\001ACTION") || e.Trailing[len(e.Trailing)-1] != ctcpDelim {
		return false
	}

	return true
}

// IsFromChannel checks to see if a message was from a channel (rather than
// a private message).
func (e *Event) IsFromChannel() bool {
	if len(e.Params) != 1 {
		return false
	}

	if e.Command != "PRIVMSG" || !IsValidChannel(e.Params[0]) {
		return false
	}

	return true
}

// IsFromUser checks to see if a message was from a user (rather than a
// channel).
func (e *Event) IsFromUser() bool {
	if len(e.Params) != 1 {
		return false
	}

	if e.Command != "PRIVMSG" || !IsValidNick(e.Params[0]) {
		return false
	}

	return true
}

// StripAction returns the stripped version of the action encoding from a
// PRIVMSG ACTION (/me).
func (e *Event) StripAction() string {
	if !e.IsAction() || len(e.Trailing) < 9 {
		return e.Trailing
	}

	return e.Trailing[8 : len(e.Trailing)-1]
}

// EventLimiter is a custom ticker which lets you rate limit sending events
// to a function (e.g. Client.Send()), with optional burst support. See
// NewEventLimiter() for more information.
type EventLimiter struct {
	tick     *time.Ticker
	throttle chan time.Time
	fn       func(*Event) error
}

// loop is used to read events from the internal time.Ticker.
func (el *EventLimiter) loop() {
	// This should exit itself once el.Stop() is called.
	for t := range el.tick.C {
		el.throttle <- t
	}
}

// Stop closes the ticker, and prevents re-use of the EventLimiter. Use this
// to prevent EventLimiter from keeping unnecessary pointers in memory.
func (el *EventLimiter) Stop() {
	el.tick.Stop()
	el.fn = nil
}

// Send is the subtitute function used to send the event the the previously
// specified send function.
//
// This WILL panic if Stop() was already called on the EventLimiter.
func (el *EventLimiter) Send(event *Event) error {
	// Ensure nobody is sending to it once it's closed.
	if el.fn == nil {
		panic("attempted send on closed EventLimiter")
	}

	<-el.throttle
	return el.fn(event)
}

// SendAll sends a list of events to Send(). SendAll will return the first
// error it gets when attempting to Send() to the predefined Send function.
// It will not attempt to continue processing the list of events.
func (el *EventLimiter) SendAll(events ...*Event) error {
	for i := 0; i < len(events); i++ {
		if err := el.Send(events[i]); err != nil {
			return err
		}
	}

	return nil
}

// NewEventLimiter returns a NewEventLimiter which can be used to rate limit
// events being sent to a Send function. This does support bursting a
// certain amount of messages if there are less than burstCount.
//
// Ensure that Stop() is called on the returned EventLimiter, otherwise
// the limiter may keep unwanted pointers to data in memory.
func NewEventLimiter(burstCount int, rate time.Duration, eventFunc func(event *Event) error) *EventLimiter {
	limiter := &EventLimiter{
		tick:     time.NewTicker(rate),
		throttle: make(chan time.Time, burstCount),
		fn:       eventFunc,
	}

	// Push the ticket into the background. If you want to stop this, simply
	// use EventLimiter.Stop().
	go limiter.loop()

	return limiter
}
<commit_msg>add prettification for CHGHOST events<commit_after>// Copyright (c) Liam Stanley <me@liamstanley.io>. All rights reserved. Use
// of this source code is governed by the MIT license that can be found in
// the LICENSE file.

package girc

import (
	"bytes"
	"fmt"
	"strings"
	"time"
)

const (
	eventSpace byte = 0x20 // Separator.
	maxLength       = 510  // Maximum length is 510 (2 for line endings).
)

// cutCRFunc is used to trim CR characters from prefixes/messages.
func cutCRFunc(r rune) bool {
	return r == '\r' || r == '\n'
}

// Event represents an IRC protocol message, see RFC1459 section 2.3.1
//
//    <message>  :: [':' <prefix> <SPACE>] <command> <params> <crlf>
//    <prefix>   :: <servername> | <nick> ['!' <user>] ['@' <host>]
//    <command>  :: <letter>{<letter>} | <number> <number> <number>
//    <SPACE>    :: ' '{' '}
//    <params>   :: <SPACE> [':' <trailing> | <middle> <params>]
//    <middle>   :: <Any *non-empty* sequence of octets not including SPACE or NUL
//                   or CR or LF, the first of which may not be ':'>
//    <trailing> :: <Any, possibly empty, sequence of octets not including NUL or
//                   CR or LF>
//    <crlf>     :: CR LF
type Event struct {
	Source        *Source  // The source of the event.
	Tags          Tags     // IRCv3 style message tags. Only use if network supported.
	Command       string   // the IRC command, e.g. JOIN, PRIVMSG, KILL.
	Params        []string // parameters to the command. Commonly nickname, channel, etc.
	Trailing      string   // any trailing data. e.g. with a PRIVMSG, this is the message text.
	EmptyTrailing bool     // if true, trailing prefix (:) will be added even if Event.Trailing is empty.
	Sensitive     bool     // if the message is sensitive (e.g. and should not be logged).
}

// ParseEvent takes a string and attempts to create a Event struct.
//
// Returns nil if the Event is invalid.
func ParseEvent(raw string) (e *Event) {
	// Ignore empty events.
	if raw = strings.TrimFunc(raw, cutCRFunc); len(raw) < 2 {
		return nil
	}

	i, j := 0, 0
	e = &Event{}

	if raw[0] == prefixTag {
		// Tags end with a space.
		i = strings.IndexByte(raw, eventSpace)

		if i < 2 {
			return nil
		}

		e.Tags = ParseTags(raw[1:i])
		raw = raw[i+1:]
	}

	if raw[0] == messagePrefix {
		// Prefix ends with a space.
		i = strings.IndexByte(raw, eventSpace)

		// Prefix string must not be empty if the indicator is present.
		if i < 2 {
			return nil
		}

		e.Source = ParseSource(raw[1:i])

		// Skip space at the end of the prefix.
		i++
	}

	// Find end of command.
	j = i + strings.IndexByte(raw[i:], eventSpace)

	// Extract command.
	if j < i {
		e.Command = strings.ToUpper(raw[i:])
		return e
	}

	e.Command = strings.ToUpper(raw[i:j])
	// Skip space after command.
	j++

	// Find prefix for trailer.
	i = bytes.Index([]byte(raw[j:]), []byte{eventSpace, messagePrefix})
	if i != -1 {
		i += 1
	}

	if i < 0 || raw[j+i-1] != eventSpace {
		// No trailing argument.
		e.Params = strings.Split(raw[j:], string(eventSpace))
		return e
	}

	// Compensate for index on substring.
	i = i + j

	// Check if we need to parse arguments.
	if i > j {
		e.Params = strings.Split(raw[j:i-1], string(eventSpace))
	}

	e.Trailing = raw[i+1:]

	// We need to re-encode the trailing argument even if it was empty.
	if len(e.Trailing) <= 0 {
		e.EmptyTrailing = true
	}

	return e
}

// Len calculates the length of the string representation of event.
func (e *Event) Len() (length int) {
	if e.Tags != nil {
		// Include tags and trailing space.
		length = e.Tags.Len() + 1
	}
	if e.Source != nil {
		// Include prefix and trailing space.
		length += e.Source.Len() + 2
	}

	length += len(e.Command)

	if len(e.Params) > 0 {
		length += len(e.Params)

		for i := 0; i < len(e.Params); i++ {
			length += len(e.Params[i])
		}
	}

	if len(e.Trailing) > 0 || e.EmptyTrailing {
		// Include prefix and space.
		length += len(e.Trailing) + 2
	}

	return
}

// Bytes returns a []byte representation of event. Strips all newlines and
// carriage returns.
//
// Per RFC2812 section 2.3, messages should not exceed 512 characters in
// length. This method forces that limit by discarding any characters
// exceeding the length limit.
func (e *Event) Bytes() []byte {
	buffer := new(bytes.Buffer)

	// Tags.
	if e.Tags != nil {
		e.Tags.writeTo(buffer)
	}

	// Event prefix.
	if e.Source != nil {
		buffer.WriteByte(messagePrefix)
		e.Source.writeTo(buffer)
		buffer.WriteByte(eventSpace)
	}

	// Command is required.
	buffer.WriteString(e.Command)

	// Space separated list of arguments.
	if len(e.Params) > 0 {
		buffer.WriteByte(eventSpace)
		buffer.WriteString(strings.Join(e.Params, string(eventSpace)))
	}

	if len(e.Trailing) > 0 || e.EmptyTrailing {
		buffer.WriteByte(eventSpace)
		buffer.WriteByte(messagePrefix)
		buffer.WriteString(e.Trailing)
	}

	// We need the limit the buffer length.
	if buffer.Len() > (maxLength) {
		if e.Tags != nil {
			// regular message, max tag length, and the splitting space.
			buffer.Truncate(maxLength + maxTagLength + 1)
		} else {
			buffer.Truncate(maxLength)
		}
	}

	out := buffer.Bytes()

	// Strip newlines and carriage returns.
	for i := 0; i < len(out); i++ {
		if out[i] == 0x0A || out[i] == 0x0D {
			out = append(out[:i], out[i+1:]...)
			i-- // Decrease the index so we can pick up where we left off.
		}
	}

	return out
}

// String returns a string representation of this event. Strips all newlines
// and carriage returns.
func (e *Event) String() string {
	return string(e.Bytes())
}

// Pretty returns a prettified string of the event. If the event doesn't
// support prettification, ok is false. Pretty is not just useful to make
// an event prettier, but also to filter out events that most don't visually
// see in normal IRC clients. e.g. most clients don't show WHO queries.
func (e *Event) Pretty() (out string, ok bool) {
	if e.Command == INITIALIZED {
		return fmt.Sprintf("[*] connection to %s initialized", e.Trailing), true
	}

	if e.Command == CONNECTED {
		return fmt.Sprintf("[*] successfully connected to %s", e.Trailing), true
	}

	if (e.Command == PRIVMSG || e.Command == NOTICE) && len(e.Params) > 0 {
		return fmt.Sprintf("[%s] (%s) %s", strings.Join(e.Params, ","), e.Source.Name, e.Trailing), true
	}

	if e.Command == RPL_MOTD || e.Command == RPL_MOTDSTART ||
		e.Command == RPL_WELCOME || e.Command == RPL_YOURHOST ||
		e.Command == RPL_CREATED || e.Command == RPL_LUSERCLIENT {
		return fmt.Sprintf("[*] " + e.Trailing), true
	}

	if e.Command == JOIN && len(e.Params) > 0 {
		return fmt.Sprintf("[*] %s has joined %s", e.Source.Name, e.Params[0]), true
	}

	if e.Command == PART && len(e.Params) > 0 {
		return fmt.Sprintf("[*] %s has left %s (%s)", e.Source.Name, e.Params[0], e.Trailing), true
	}

	if e.Command == ERROR {
		return fmt.Sprintf("[*] an error occurred: %s", e.Trailing), true
	}

	if e.Command == QUIT {
		return fmt.Sprintf("[*] %s has quit (%s)", e.Source.Name, e.Trailing), true
	}

	if e.Command == KICK && len(e.Params) == 2 {
		return fmt.Sprintf("[%s] *** %s has kicked %s: %s", e.Params[0], e.Source.Name, e.Params[1], e.Trailing), true
	}

	if e.Command == NICK && len(e.Params) == 1 {
		return fmt.Sprintf("[*] %s is now known as %s", e.Source.Name, e.Params[0]), true
	}

	if e.Command == TOPIC && len(e.Params) > 0 {
		return fmt.Sprintf("[%s] *** %s has set the topic to: %s", e.Params[len(e.Params)-1], e.Source.Name, e.Trailing), true
	}

	if e.Command == MODE && len(e.Params) > 2 {
		return fmt.Sprintf("[%s] %s set modes: %s", e.Params[0], e.Source.Name, strings.Join(e.Params[1:], " ")), true
	}

	if e.Command == AWAY {
		if len(e.Trailing) > 0 {
			return fmt.Sprintf("[*] %s is now away: %s", e.Source.Name, e.Trailing), true
		}

		return fmt.Sprintf("[*] %s is no longer away", e.Source.Name), true
	}

	if e.Command == CAP_CHGHOST && len(e.Params) == 2 {
		return fmt.Sprintf("[*] %s has changed their host to %s (was %s)", e.Source.Name, e.Params[1], e.Source.Host), true
	}

	return "", false
}

// IsAction checks to see if the event is a PRIVMSG, and is an ACTION (/me).
func (e *Event) IsAction() bool {
	if len(e.Trailing) <= 0 || e.Command != PRIVMSG {
		return false
	}

	if !strings.HasPrefix(e.Trailing, "\001ACTION") || e.Trailing[len(e.Trailing)-1] != ctcpDelim {
		return false
	}

	return true
}

// IsFromChannel checks to see if a message was from a channel (rather than
// a private message).
func (e *Event) IsFromChannel() bool {
	if len(e.Params) != 1 {
		return false
	}

	if e.Command != "PRIVMSG" || !IsValidChannel(e.Params[0]) {
		return false
	}

	return true
}

// IsFromUser checks to see if a message was from a user (rather than a
// channel).
func (e *Event) IsFromUser() bool {
	if len(e.Params) != 1 {
		return false
	}

	if e.Command != "PRIVMSG" || !IsValidNick(e.Params[0]) {
		return false
	}

	return true
}

// StripAction returns the stripped version of the action encoding from a
// PRIVMSG ACTION (/me).
func (e *Event) StripAction() string {
	if !e.IsAction() || len(e.Trailing) < 9 {
		return e.Trailing
	}

	return e.Trailing[8 : len(e.Trailing)-1]
}

// EventLimiter is a custom ticker which lets you rate limit sending events
// to a function (e.g. Client.Send()), with optional burst support. See
// NewEventLimiter() for more information.
type EventLimiter struct {
	tick     *time.Ticker
	throttle chan time.Time
	fn       func(*Event) error
}

// loop is used to read events from the internal time.Ticker.
func (el *EventLimiter) loop() {
	// This should exit itself once el.Stop() is called.
	for t := range el.tick.C {
		el.throttle <- t
	}
}

// Stop closes the ticker, and prevents re-use of the EventLimiter. Use this
// to prevent EventLimiter from keeping unnecessary pointers in memory.
func (el *EventLimiter) Stop() {
	el.tick.Stop()
	el.fn = nil
}

// Send is the subtitute function used to send the event the the previously
// specified send function.
//
// This WILL panic if Stop() was already called on the EventLimiter.
func (el *EventLimiter) Send(event *Event) error {
	// Ensure nobody is sending to it once it's closed.
	if el.fn == nil {
		panic("attempted send on closed EventLimiter")
	}

	<-el.throttle
	return el.fn(event)
}

// SendAll sends a list of events to Send(). SendAll will return the first
// error it gets when attempting to Send() to the predefined Send function.
// It will not attempt to continue processing the list of events.
func (el *EventLimiter) SendAll(events ...*Event) error {
	for i := 0; i < len(events); i++ {
		if err := el.Send(events[i]); err != nil {
			return err
		}
	}

	return nil
}

// NewEventLimiter returns a NewEventLimiter which can be used to rate limit
// events being sent to a Send function. This does support bursting a
// certain amount of messages if there are less than burstCount.
//
// Ensure that Stop() is called on the returned EventLimiter, otherwise
// the limiter may keep unwanted pointers to data in memory.
func NewEventLimiter(burstCount int, rate time.Duration, eventFunc func(event *Event) error) *EventLimiter {
	limiter := &EventLimiter{
		tick:     time.NewTicker(rate),
		throttle: make(chan time.Time, burstCount),
		fn:       eventFunc,
	}

	// Push the ticket into the background. If you want to stop this, simply
	// use EventLimiter.Stop().
	go limiter.loop()

	return limiter
}
<|endoftext|>


<commit_before>package slack

import (
	"encoding/json"
)

type messageWrapper struct {
	message *Message
	status  Status
}

func (bot *Bot) OnEvent(event string, handler BotAction) {
	handlers, ok := bot.Handlers[event]
	if !ok {
		handlers make([]BotAction, 0)
	}
	handlers = append(handlers, handler)
	bot.Handlers[event] = handlers
}

func (bot *Bot) OnEventWithSubtype(event, subtype string, handler BotAction) {
	subtypeMap, ok := bot.Subhandlers[event]
	if !ok {
		subtypeMap = make(map[string]([]BotAction))
		bot.Subhandlers[event] = subtypeMap
	}
	handlers, ok := bot.Subhandlers[event][subtype]
	if !ok {
		handlers = make([]BotAction, 0)
	}
	handlers = append(handlers, handler)
	bot.Subhandlers[event][subtype] = handlers
}

func unpackEvent(bytes []byte) (map[string]interface{}, error) {
	var message map[string]interface{}
	err := json.Unmarshal(bytes, &message)
	if err != nil {
		return nil, err
	}
	return message, nil
}

func (bot *Bot) handle(event map[string]interface{}) []messageWrapper {
	wrappers := make([]messageWrapper, 0)

	eventType, hasType := event["type"].(string)
	eventSubtype, hasSubtype := event["subtype"].(string)

	if hasSubtype {
		subhandlers, ok := bot.Subhandlers[eventType][eventSubtype]
		if ok {
			for _, subhandler := range subhandlers {
				message, status := subhandler(bot, event)
				wrappers = append(wrappers, messageWrapper{message, status})
			}
		}
	}
	if hasType {
		handlers, ok := bot.Handlers[eventType]
		if ok {
			for _, handler := range handlers {
				message, status := handler(bot, event)
				wrappers = append(wrappers, messageWrapper{message, status})
			}
		}
	}
	return wrappers
}
<commit_msg>fix silly syntax error<commit_after>package slack

import (
	"encoding/json"
)

type messageWrapper struct {
	message *Message
	status  Status
}

func (bot *Bot) OnEvent(event string, handler BotAction) {
	handlers, ok := bot.Handlers[event]
	if !ok {
		handlers = make([]BotAction, 0)
	}
	handlers = append(handlers, handler)
	bot.Handlers[event] = handlers
}

func (bot *Bot) OnEventWithSubtype(event, subtype string, handler BotAction) {
	subtypeMap, ok := bot.Subhandlers[event]
	if !ok {
		subtypeMap = make(map[string]([]BotAction))
		bot.Subhandlers[event] = subtypeMap
	}
	handlers, ok := bot.Subhandlers[event][subtype]
	if !ok {
		handlers = make([]BotAction, 0)
	}
	handlers = append(handlers, handler)
	bot.Subhandlers[event][subtype] = handlers
}

func unpackEvent(bytes []byte) (map[string]interface{}, error) {
	var message map[string]interface{}
	err := json.Unmarshal(bytes, &message)
	if err != nil {
		return nil, err
	}
	return message, nil
}

func (bot *Bot) handle(event map[string]interface{}) []messageWrapper {
	wrappers := make([]messageWrapper, 0)

	eventType, hasType := event["type"].(string)
	eventSubtype, hasSubtype := event["subtype"].(string)

	if hasSubtype {
		subhandlers, ok := bot.Subhandlers[eventType][eventSubtype]
		if ok {
			for _, subhandler := range subhandlers {
				message, status := subhandler(bot, event)
				wrappers = append(wrappers, messageWrapper{message, status})
			}
		}
	}
	if hasType {
		handlers, ok := bot.Handlers[eventType]
		if ok {
			for _, handler := range handlers {
				message, status := handler(bot, event)
				wrappers = append(wrappers, messageWrapper{message, status})
			}
		}
	}
	return wrappers
}
<|endoftext|>


<commit_before>package main

// Code responsible for handling an incoming event from github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
)

// This function is called whenever an event happens on github.
func handleEvent(eventType string, document []byte) (err error) {

	// log.Println("Incoming request:", string(document))

	switch eventType {
	case "push":

		var event PushEvent
		err = json.Unmarshal(document, &event)
		if err != nil {
			return
		}

		log.Printf("Received PushEvent %#+v", event)

		if event.Deleted {
			// When a branch is deleted we get a "push" event we don't care
			// about (after = "0000")
			return
		}

		err = eventPush(event)
		if err != nil {
			return
		}

	default:
		log.Println("Unhandled event:", eventType)
	}

	return
}

// HTTP handler for /hook
// It is expecting a POST with a JSON payload according to
// http://developer.github.com/v3/activity/events/
func handleHook(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		fmt.Fprintf(w, "Expected JSON POST payload.\n")
		return
	}

	request, err := ioutil.ReadAll(r.Body)
	check(err)

	var buf bytes.Buffer
	// r.Header.Write(&buf)
	// log.Println("Incoming request headers: ", string(buf.Bytes()))
	// buf.Reset()

	err = json.Indent(&buf, request, "", "  ")
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Expected valid JSON POST payload.\n")
		log.Println("Not a valid JSON payload. NOOP.")
		return
	}

	if len(r.Header["X-Github-Event"]) != 1 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Expected X-Github-Event header.\n")
		log.Println("No X-Github-Event header. NOOP")
		return
	}
	eventType := r.Header["X-Github-Event"][0]
	data := buf.Bytes()

	// Check to see if we have data from somewhere which is not github
	j, err := ParseJustNongithub(request)
	if !j.NonGithub.Wait {
		go func() {
			err := handleEvent(eventType, data)
			if err != nil {
				log.Printf("Error processing %v %v %q", eventType, string(data), err)
			}
		}()

		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, "OK. Not waiting for build.\n")
		return
	}

	// Handle the event
	err = handleEvent(eventType, data)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error handling event: %q\n", err)
		log.Printf("Error handling event: %q", err)
		return
	}

	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, "OK\n")
}

// Invoked when a respository we are watching changes
func runTang(repo, sha, repo_path, ref, logPath string) (err error) {

    // TODO(pwaller): do tee in go.
	c := `./tang.hook |& tee $TANG_LOGPATH; exit ${PIPESTATUS[0]}`
	cmd := Command(repo_path, "bash", "-c", c)

	cmd.Env = append(os.Environ(),
		"TANG_SHA="+sha, "TANG_REF="+ref, "TANG_LOGPATH="+logPath)
	err = cmd.Run()

	return
}

// Invoked when there is a push event to github.
func eventPush(event PushEvent) (err error) {
	if event.Repository.Name == "" {
		return ErrEmptyRepoName
	}

	if event.Repository.Organization == "" {
		return ErrEmptyRepoOrganization
	}

	if _, ok := allowedPushersSet[event.Pusher.Name]; !ok {
		log.Printf("Ignoring %q, not allowed", event.Pusher.Name)
		return ErrUserNotAllowed
	}

	gh_repo := path.Join(event.Repository.Organization, event.Repository.Name)

	log.Println("Push to", event.Repository.Url, event.Ref, "after", event.After)

	// The name of the subdirectory where the git
	// mirror is (or will appear, if it hasn't been
	// cloned yet).
	git_dir := path.Join(GIT_BASE_DIR, gh_repo)

	// Update our local mirror
	err = gitLocalMirror(event.Repository.Url, git_dir)
	if err != nil {
		return
	}

	// Check if we there is a tang hook
	tang_hook_present, err := gitHaveFile(git_dir, event.After, "tang.hook")
	if err != nil || !tang_hook_present || event.NonGithub.NoBuild {
		// Bail out, error, no tang.hook or instructed not to build it.
		return
	}

	// Dereference event.After, always. Not needed for github but useful for
	// `tang-event`, where we don't know the sha beforehand.
	sha, err := gitRevParse(git_dir, event.After)
	if err != nil {
		err = fmt.Errorf("gitRevParse: %q", err)
		return
	}

	// Only use 6 characters of sha for the name of the
	// directory checked out for this repository by tang.
	short_sha := sha[:6]
	checkout_dir := path.Join("checkout", short_sha)

	// Checkout the target sha
	err = gitCheckout(git_dir, checkout_dir, event.After)
	if err != nil {
		return
	}

	log.Println("Created", checkout_dir)

	pwd, err := os.Getwd()
	if err != nil {
		err = fmt.Errorf("runTang/getwd %q", err)
		return
	}

	logDir := path.Join("logs", short_sha)
	err = os.MkdirAll(logDir, 0777)
	if err != nil {
		err = fmt.Errorf("runTang/MkdirAll(%q): ", logDir, err)
		return
	}

	logPath := path.Join(logDir, "log.txt")
	fullLogPath := path.Join(pwd, logPath)

	// TODO(pwaller): One day this will have more information, e.g, QA link.
	infoURL := "http://services.scraperwiki.com/tang/" + logPath

	// Set the state of the commit to "in progress" (seen as yellow in
	// a github pull request)
	status := GithubStatus{"pending", infoURL, "Running"}
	updateStatus(gh_repo, event.After, status)

	// Run the tang script for the repository, if there is one.
	repo_workdir := path.Join(git_dir, checkout_dir)
	err = runTang(gh_repo, event.After, repo_workdir, event.Ref, fullLogPath)

	if err == nil {
		// All OK, send along a green
		s := GithubStatus{"success", infoURL, "Tests passed"}
		updateStatus(gh_repo, event.After, s)
		return
	}

	// Not OK, send along red.
	s := GithubStatus{"failure", infoURL, err.Error()}
	updateStatus(gh_repo, event.After, s)
	return
}
<commit_msg>go fmt<commit_after>package main

// Code responsible for handling an incoming event from github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
)

// This function is called whenever an event happens on github.
func handleEvent(eventType string, document []byte) (err error) {

	// log.Println("Incoming request:", string(document))

	switch eventType {
	case "push":

		var event PushEvent
		err = json.Unmarshal(document, &event)
		if err != nil {
			return
		}

		log.Printf("Received PushEvent %#+v", event)

		if event.Deleted {
			// When a branch is deleted we get a "push" event we don't care
			// about (after = "0000")
			return
		}

		err = eventPush(event)
		if err != nil {
			return
		}

	default:
		log.Println("Unhandled event:", eventType)
	}

	return
}

// HTTP handler for /hook
// It is expecting a POST with a JSON payload according to
// http://developer.github.com/v3/activity/events/
func handleHook(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		fmt.Fprintf(w, "Expected JSON POST payload.\n")
		return
	}

	request, err := ioutil.ReadAll(r.Body)
	check(err)

	var buf bytes.Buffer
	// r.Header.Write(&buf)
	// log.Println("Incoming request headers: ", string(buf.Bytes()))
	// buf.Reset()

	err = json.Indent(&buf, request, "", "  ")
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Expected valid JSON POST payload.\n")
		log.Println("Not a valid JSON payload. NOOP.")
		return
	}

	if len(r.Header["X-Github-Event"]) != 1 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Expected X-Github-Event header.\n")
		log.Println("No X-Github-Event header. NOOP")
		return
	}
	eventType := r.Header["X-Github-Event"][0]
	data := buf.Bytes()

	// Check to see if we have data from somewhere which is not github
	j, err := ParseJustNongithub(request)
	if !j.NonGithub.Wait {
		go func() {
			err := handleEvent(eventType, data)
			if err != nil {
				log.Printf("Error processing %v %v %q", eventType, string(data), err)
			}
		}()

		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, "OK. Not waiting for build.\n")
		return
	}

	// Handle the event
	err = handleEvent(eventType, data)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error handling event: %q\n", err)
		log.Printf("Error handling event: %q", err)
		return
	}

	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, "OK\n")
}

// Invoked when a respository we are watching changes
func runTang(repo, sha, repo_path, ref, logPath string) (err error) {

	// TODO(pwaller): do tee in go.
	c := `./tang.hook |& tee $TANG_LOGPATH; exit ${PIPESTATUS[0]}`
	cmd := Command(repo_path, "bash", "-c", c)

	cmd.Env = append(os.Environ(),
		"TANG_SHA="+sha, "TANG_REF="+ref, "TANG_LOGPATH="+logPath)
	err = cmd.Run()

	return
}

// Invoked when there is a push event to github.
func eventPush(event PushEvent) (err error) {
	if event.Repository.Name == "" {
		return ErrEmptyRepoName
	}

	if event.Repository.Organization == "" {
		return ErrEmptyRepoOrganization
	}

	if _, ok := allowedPushersSet[event.Pusher.Name]; !ok {
		log.Printf("Ignoring %q, not allowed", event.Pusher.Name)
		return ErrUserNotAllowed
	}

	gh_repo := path.Join(event.Repository.Organization, event.Repository.Name)

	log.Println("Push to", event.Repository.Url, event.Ref, "after", event.After)

	// The name of the subdirectory where the git
	// mirror is (or will appear, if it hasn't been
	// cloned yet).
	git_dir := path.Join(GIT_BASE_DIR, gh_repo)

	// Update our local mirror
	err = gitLocalMirror(event.Repository.Url, git_dir)
	if err != nil {
		return
	}

	// Check if we there is a tang hook
	tang_hook_present, err := gitHaveFile(git_dir, event.After, "tang.hook")
	if err != nil || !tang_hook_present || event.NonGithub.NoBuild {
		// Bail out, error, no tang.hook or instructed not to build it.
		return
	}

	// Dereference event.After, always. Not needed for github but useful for
	// `tang-event`, where we don't know the sha beforehand.
	sha, err := gitRevParse(git_dir, event.After)
	if err != nil {
		err = fmt.Errorf("gitRevParse: %q", err)
		return
	}

	// Only use 6 characters of sha for the name of the
	// directory checked out for this repository by tang.
	short_sha := sha[:6]
	checkout_dir := path.Join("checkout", short_sha)

	// Checkout the target sha
	err = gitCheckout(git_dir, checkout_dir, event.After)
	if err != nil {
		return
	}

	log.Println("Created", checkout_dir)

	pwd, err := os.Getwd()
	if err != nil {
		err = fmt.Errorf("runTang/getwd %q", err)
		return
	}

	logDir := path.Join("logs", short_sha)
	err = os.MkdirAll(logDir, 0777)
	if err != nil {
		err = fmt.Errorf("runTang/MkdirAll(%q): ", logDir, err)
		return
	}

	logPath := path.Join(logDir, "log.txt")
	fullLogPath := path.Join(pwd, logPath)

	// TODO(pwaller): One day this will have more information, e.g, QA link.
	infoURL := "http://services.scraperwiki.com/tang/" + logPath

	// Set the state of the commit to "in progress" (seen as yellow in
	// a github pull request)
	status := GithubStatus{"pending", infoURL, "Running"}
	updateStatus(gh_repo, event.After, status)

	// Run the tang script for the repository, if there is one.
	repo_workdir := path.Join(git_dir, checkout_dir)
	err = runTang(gh_repo, event.After, repo_workdir, event.Ref, fullLogPath)

	if err == nil {
		// All OK, send along a green
		s := GithubStatus{"success", infoURL, "Tests passed"}
		updateStatus(gh_repo, event.After, s)
		return
	}

	// Not OK, send along red.
	s := GithubStatus{"failure", infoURL, err.Error()}
	updateStatus(gh_repo, event.After, s)
	return
}
<|endoftext|>


<commit_before>/*
 *    Copyright (C) 2014 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// beehive's Serial module.
package serialbee

import (
	"github.com/huin/goserial"
	"github.com/muesli/beehive/modules"
	"io"
	"log"
	"bytes"
	"encoding/binary"
	"time"
)

type SerialBee struct {
	modules.Module

	conn io.ReadWriteCloser

	device   string
	baudrate     int
}

func (mod *SerialBee) Action(action modules.Action) []modules.Placeholder {
	outs := []modules.Placeholder{}
	text := ""

	switch action.Name {
	case "send":
		for _, opt := range action.Options {
			if opt.Name == "text" {
				text = opt.Value.(string)
			}
		}

		bufOut := new(bytes.Buffer)
		err := binary.Write(bufOut, binary.LittleEndian, text)
		if err != nil {
			panic(err)
		}

		_, err = mod.conn.Write(bufOut.Bytes())
		if err != nil {
			panic(err)
		}

	default:
		// unknown action
		return outs
	}

	return outs
}

func (mod *SerialBee) Run(eventChan chan modules.Event) {
	if mod.baudrate == 0 || mod.device == "" {
		return
	}

	var err error
	c := &goserial.Config{Name: mod.device, Baud: mod.baudrate}
	mod.conn, err = goserial.OpenPort(c)
	if err != nil {
		log.Fatal(err)
	}
	time.Sleep(1 * time.Second)

	for {
/*			if len(v.Text) > 0 {
				text := strings.TrimSpace(v.Text)

				ev := modules.Event{
					Bee:  mod.Name(),
					Name: "message",
					Options: []modules.Placeholder{
						modules.Placeholder{
							Name:  "user",
							Type:  "string",
							Value: v.Remote,
						},
						modules.Placeholder{
							Name:  "text",
							Type:  "string",
							Value: text,
						},
					},
				}
				eventChan <- ev
			}*/
			time.Sleep(1 * time.Second)
	}
}
<commit_msg>* Write as byte array in SerialBee.<commit_after>/*
 *    Copyright (C) 2014 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// beehive's Serial module.
package serialbee

import (
	"github.com/huin/goserial"
	"github.com/muesli/beehive/modules"
	"io"
	"log"
	"bytes"
	"encoding/binary"
	"time"
)

type SerialBee struct {
	modules.Module

	conn io.ReadWriteCloser

	device   string
	baudrate     int
}

func (mod *SerialBee) Action(action modules.Action) []modules.Placeholder {
	outs := []modules.Placeholder{}
	text := ""

	switch action.Name {
	case "send":
		for _, opt := range action.Options {
			if opt.Name == "text" {
				text = opt.Value.(string)
			}
		}

		bufOut := new(bytes.Buffer)
		err := binary.Write(bufOut, binary.LittleEndian, text)
		if err != nil {
			panic(err)
		}

		for _, v := range [][]byte{bufOut.Bytes()} {
			_, err = mod.conn.Write(v)
			if err != nil {
				panic(err)
			}
		}

	default:
		// unknown action
		return outs
	}

	return outs
}

func (mod *SerialBee) Run(eventChan chan modules.Event) {
	if mod.baudrate == 0 || mod.device == "" {
		return
	}

	var err error
	c := &goserial.Config{Name: mod.device, Baud: mod.baudrate}
	mod.conn, err = goserial.OpenPort(c)
	if err != nil {
		log.Fatal(err)
	}
	time.Sleep(1 * time.Second)

	for {
/*			if len(v.Text) > 0 {
				text := strings.TrimSpace(v.Text)

				ev := modules.Event{
					Bee:  mod.Name(),
					Name: "message",
					Options: []modules.Placeholder{
						modules.Placeholder{
							Name:  "user",
							Type:  "string",
							Value: v.Remote,
						},
						modules.Placeholder{
							Name:  "text",
							Type:  "string",
							Value: text,
						},
					},
				}
				eventChan <- ev
			}*/
			time.Sleep(1 * time.Second)
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
)

func TestAccAWSRoute53ZoneAssociation_basic(t *testing.T) {
	var zone route53.HostedZone

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckRoute53ZoneAssociationDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccRoute53ZoneAssociationConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckRoute53ZoneAssociationExists("aws_route53_zone_association.foobar", &zone),
				),
			},
		},
	})
}

func TestAccAWSRoute53ZoneAssociation_region(t *testing.T) {
	var zone route53.HostedZone

	// record the initialized providers so that we can use them to
	// check for the instances in each region
	var providers []*schema.Provider
	providerFactories := map[string]terraform.ResourceProviderFactory{
		"aws": func() (terraform.ResourceProvider, error) {
			p := Provider()
			providers = append(providers, p.(*schema.Provider))
			return p, nil
		},
	}

	resource.Test(t, resource.TestCase{
		PreCheck:          func() { testAccPreCheck(t) },
		ProviderFactories: providerFactories,
		CheckDestroy:      testAccCheckRoute53ZoneAssociationDestroyWithProviders(&providers),
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccRoute53ZoneAssociationRegionConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckRoute53ZoneAssociationExistsWithProviders("aws_route53_zone_association.foobar", &zone, &providers),
				),
			},
		},
	})
}

func testAccCheckRoute53ZoneAssociationDestroy(s *terraform.State) error {
	return testAccCheckRoute53ZoneAssociationDestroyWithProvider(s, testAccProvider)
}

func testAccCheckRoute53ZoneAssociationDestroyWithProviders(providers *[]*schema.Provider) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		for _, provider := range *providers {
			if provider.Meta() == nil {
				continue
			}
			if err := testAccCheckRoute53ZoneAssociationDestroyWithProvider(s, provider); err != nil {
				return err
			}
		}
		return nil
	}
}

func testAccCheckRoute53ZoneAssociationDestroyWithProvider(s *terraform.State, provider *schema.Provider) error {
	conn := provider.Meta().(*AWSClient).r53conn
	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_route53_zone_association" {
			continue
		}

		zone_id, vpc_id := resourceAwsRoute53ZoneAssociationParseId(rs.Primary.ID)

		resp, err := conn.GetHostedZone(&route53.GetHostedZoneInput{Id: aws.String(zone_id)})
		if err != nil {
			exists := false
			for _, vpc := range resp.VPCs {
				if vpc_id == *vpc.VPCId {
					exists = true
				}
			}
			if exists {
				return fmt.Errorf("VPC: %v is still associated to HostedZone: %v", vpc_id, zone_id)
			}
		}
	}
	return nil
}

func testAccCheckRoute53ZoneAssociationExists(n string, zone *route53.HostedZone) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		return testAccCheckRoute53ZoneAssociationExistsWithProvider(s, n, zone, testAccProvider)
	}
}

func testAccCheckRoute53ZoneAssociationExistsWithProviders(n string, zone *route53.HostedZone, providers *[]*schema.Provider) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		for _, provider := range *providers {
			if provider.Meta() == nil {
				continue
			}
			if err := testAccCheckRoute53ZoneAssociationExistsWithProvider(s, n, zone, provider); err != nil {
				return err
			}
		}
		return nil
	}
}

func testAccCheckRoute53ZoneAssociationExistsWithProvider(s *terraform.State, n string, zone *route53.HostedZone, provider *schema.Provider) error {
	rs, ok := s.RootModule().Resources[n]
	if !ok {
		return fmt.Errorf("Not found: %s", n)
	}

	if rs.Primary.ID == "" {
		return fmt.Errorf("No zone association ID is set")
	}

	zone_id, vpc_id := resourceAwsRoute53ZoneAssociationParseId(rs.Primary.ID)

	conn := provider.Meta().(*AWSClient).r53conn
	resp, err := conn.GetHostedZone(&route53.GetHostedZoneInput{Id: aws.String(zone_id)})
	if err != nil {
		return fmt.Errorf("Hosted zone err: %v", err)
	}

	exists := false
	for _, vpc := range resp.VPCs {
		if vpc_id == *vpc.VPCId {
			exists = true
		}
	}
	if !exists {
		return fmt.Errorf("Hosted zone association not found")
	}

	*zone = *resp.HostedZone
	return nil
}

const testAccRoute53ZoneAssociationConfig = `
resource "aws_vpc" "foo" {
	cidr_block = "10.6.0.0/16"
	enable_dns_hostnames = true
	enable_dns_support = true
}

resource "aws_vpc" "bar" {
	cidr_block = "10.7.0.0/16"
	enable_dns_hostnames = true
	enable_dns_support = true
}

resource "aws_route53_zone" "foo" {
	name = "foo.com"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_route53_zone_association" "foobar" {
	zone_id = "${aws_route53_zone.foo.id}"
	vpc_id  = "${aws_vpc.bar.id}"
}
`

const testAccRoute53ZoneAssociationRegionConfig = `
provider "aws" {
	alias = "west"
	region = "us-west-2"
}

provider "aws" {
	alias = "east"
	region = "us-east-1"
}

resource "aws_vpc" "foo" {
	provider = "aws.west"
	cidr_block = "10.6.0.0/16"
	enable_dns_hostnames = true
	enable_dns_support = true
}

resource "aws_vpc" "bar" {
	provider = "aws.east"
	cidr_block = "10.7.0.0/16"
	enable_dns_hostnames = true
	enable_dns_support = true
}

resource "aws_route53_zone" "foo" {
	provider = "aws.west"
	name = "foo.com"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_route53_zone_association" "foobar" {
	provider = "aws.west"
	zone_id = "${aws_route53_zone.foo.id}"
	vpc_id  = "${aws_vpc.bar.id}"
	vpc_region = "us-east-1"
}
`
<commit_msg>Update resource_aws_route53_zone_association_test.go<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
)

func TestAccAWSRoute53ZoneAssociation_basic(t *testing.T) {
	var zone route53.HostedZone

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckRoute53ZoneAssociationDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccRoute53ZoneAssociationConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckRoute53ZoneAssociationExists("aws_route53_zone_association.foobar", &zone),
				),
			},
		},
	})
}

func TestAccAWSRoute53ZoneAssociation_region(t *testing.T) {
	var zone route53.HostedZone

	// record the initialized providers so that we can use them to
	// check for the instances in each region
	var providers []*schema.Provider
	providerFactories := map[string]terraform.ResourceProviderFactory{
		"aws": func() (terraform.ResourceProvider, error) {
			p := Provider()
			providers = append(providers, p.(*schema.Provider))
			return p, nil
		},
	}

	resource.Test(t, resource.TestCase{
		PreCheck:          func() { testAccPreCheck(t) },
		ProviderFactories: providerFactories,
		CheckDestroy:      testAccCheckRoute53ZoneAssociationDestroyWithProviders(&providers),
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccRoute53ZoneAssociationRegionConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckRoute53ZoneAssociationExistsWithProviders("aws_route53_zone_association.foobar", &zone, &providers),
				),
			},
		},
	})
}

func testAccCheckRoute53ZoneAssociationDestroy(s *terraform.State) error {
	return testAccCheckRoute53ZoneAssociationDestroyWithProvider(s, testAccProvider)
}

func testAccCheckRoute53ZoneAssociationDestroyWithProviders(providers *[]*schema.Provider) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		for _, provider := range *providers {
			if provider.Meta() == nil {
				continue
			}
			if err := testAccCheckRoute53ZoneAssociationDestroyWithProvider(s, provider); err != nil {
				return err
			}
		}
		return nil
	}
}

func testAccCheckRoute53ZoneAssociationDestroyWithProvider(s *terraform.State, provider *schema.Provider) error {
	conn := provider.Meta().(*AWSClient).r53conn
	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_route53_zone_association" {
			continue
		}

		zone_id, vpc_id := resourceAwsRoute53ZoneAssociationParseId(rs.Primary.ID)

		resp, err := conn.GetHostedZone(&route53.GetHostedZoneInput{Id: aws.String(zone_id)})
		if err != nil {
			exists := false
			for _, vpc := range resp.VPCs {
				if vpc_id == *vpc.VPCId {
					exists = true
				}
			}
			if exists {
				return fmt.Errorf("VPC: %v is still associated to HostedZone: %v", vpc_id, zone_id)
			}
		}
	}
	return nil
}

func testAccCheckRoute53ZoneAssociationExists(n string, zone *route53.HostedZone) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		return testAccCheckRoute53ZoneAssociationExistsWithProvider(s, n, zone, testAccProvider)
	}
}

func testAccCheckRoute53ZoneAssociationExistsWithProviders(n string, zone *route53.HostedZone, providers *[]*schema.Provider) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		for _, provider := range *providers {
			if provider.Meta() == nil {
				continue
			}
			if err := testAccCheckRoute53ZoneAssociationExistsWithProvider(s, n, zone, provider); err != nil {
				return err
			}
		}
		return nil
	}
}

func testAccCheckRoute53ZoneAssociationExistsWithProvider(s *terraform.State, n string, zone *route53.HostedZone, provider *schema.Provider) error {
	rs, ok := s.RootModule().Resources[n]
	if !ok {
		return fmt.Errorf("Not found: %s", n)
	}

	if rs.Primary.ID == "" {
		return fmt.Errorf("No zone association ID is set")
	}

	zone_id, vpc_id := resourceAwsRoute53ZoneAssociationParseId(rs.Primary.ID)

	conn := provider.Meta().(*AWSClient).r53conn
	resp, err := conn.GetHostedZone(&route53.GetHostedZoneInput{Id: aws.String(zone_id)})
	if err != nil {
		return fmt.Errorf("Hosted zone err: %v", err)
	}

	exists := false
	for _, vpc := range resp.VPCs {
		if vpc_id == *vpc.VPCId {
			exists = true
		}
	}
	if !exists {
		return fmt.Errorf("Hosted zone association not found")
	}

	*zone = *resp.HostedZone
	return nil
}

const testAccRoute53ZoneAssociationConfig = `
resource "aws_vpc" "foo" {
	cidr_block = "10.6.0.0/16"
	enable_dns_hostnames = true
	enable_dns_support = true
}

resource "aws_vpc" "bar" {
	cidr_block = "10.7.0.0/16"
	enable_dns_hostnames = true
	enable_dns_support = true
}

resource "aws_route53_zone" "foo" {
	name = "foo.com"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_route53_zone_association" "foobar" {
	zone_id = "${aws_route53_zone.foo.id}"
	vpc_id  = "${aws_vpc.bar.id}"
}
`

const testAccRoute53ZoneAssociationRegionConfig = `
provider "aws" {
	alias = "west"
	region = "us-west-2"
}

provider "aws" {
	alias = "east"
	region = "us-east-1"
}

resource "aws_vpc" "foo" {
	provider = "aws.west"
	cidr_block = "10.6.0.0/16"
	enable_dns_hostnames = true
	enable_dns_support = true
}

resource "aws_vpc" "bar" {
	provider = "aws.east"
	cidr_block = "10.7.0.0/16"
	enable_dns_hostnames = true
	enable_dns_support = true
}

resource "aws_route53_zone" "foo" {
	provider = "aws.west"
	name = "foo.com"
	vpc_id = "${aws_vpc.foo.id}"
	vpc_region = "us-west-2"
}

resource "aws_route53_zone_association" "foobar" {
	provider = "aws.west"
	zone_id = "${aws_route53_zone.foo.id}"
	vpc_id  = "${aws_vpc.bar.id}"
	vpc_region = "us-east-1"
}
`
<|endoftext|>


<commit_before>package builder

import (
	"github.com/appc/spec/schema/types"
	"github.com/blablacar/cnt/log"
	"github.com/blablacar/cnt/spec"
	"github.com/ghodss/yaml"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

const EXEC_FILES = `
execute_files() {
  fdir=$1
  [ -d "$fdir" ] || return 0

  for file in $fdir/*; do
    [ -x "$file" ] || /cnt/bin/busybox chmod +x "$file"
    echo -e "\e[1m\e[32mRunning script -> $file\e[0m"
    $file
  done
}`

const BUILD_SCRIPT = `#!/cnt/bin/busybox sh
set -x
set -e
export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

` + EXEC_FILES + `

execute_files "$ROOTFS/cnt/runlevels/inherit-build-early"
execute_files "$TARGET/runlevels/build"
`

const BUILD_SCRIPT_LATE = `#!/cnt/bin/busybox sh
set -x
set -e
export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

` + EXEC_FILES + `

execute_files "$TARGET/runlevels/build-late"
execute_files "$ROOTFS/cnt/runlevels/inherit-build-late"
`

const PRESTART = `#!/cnt/bin/busybox sh
set -x
set -e

BASEDIR=${0%/*}
CNT_PATH=/cnt

` + EXEC_FILES + `

execute_files ${CNT_PATH}/runlevels/prestart-early

${BASEDIR}/attributes-merger -i ${CNT_PATH}/attributes -e CONFD_OVERRIDE
export CONFD_DATA=$(cat attributes.json)
${BASEDIR}/confd -onetime -config-file=${CNT_PATH}/prestart/confd.toml

execute_files ${CNT_PATH}/runlevels/prestart-late
`

const PATH_MANIFEST = "/manifest"
const PATH_IMAGE_ACI = "/image.aci"
const PATH_ROOTFS = "/rootfs"
const PATH_TARGET = "/target"
const PATH_CNT = "/cnt"
const PATH_CNT_MANIFEST = "/cnt-manifest.yml"
const PATH_RUNLEVELS = "/runlevels"
const PATH_PRESTART_EARLY = "/prestart-early"
const PATH_PRESTART_LATE = "/prestart-late"
const PATH_INHERIT_BUILD_LATE = "/inherit-build-late"
const PATH_INHERIT_BUILD_EARLY = "/inherit-build-early"
const PATH_ATTRIBUTES = "/attributes"
const PATH_FILES = "/files"
const PATH_CONFD = "/confd"
const PATH_BUILD_LATE = "/build-late"
const PATH_BUILD_SETUP = "/build-setup"
const PATH_BUILD = "/build"
const PATH_CONFDOTD = "/conf.d"
const PATH_TEMPLATES = "/templates"

type Img struct {
	path     string
	target   string
	rootfs   string
	PodName  *spec.ACFullname
	manifest spec.AciManifest
	args     BuildArgs
}

func Version(nameAndVersion string) string {
	split := strings.Split(nameAndVersion, ":")
	if len(split) == 1 {
		return ""
	}
	return split[1]
}

func ShortNameId(name types.ACIdentifier) string {
	return strings.Split(string(name), "/")[1]
}

func ShortName(nameAndVersion string) string {
	return strings.Split(Name(nameAndVersion), "/")[1]
}

func Name(nameAndVersion string) string {
	return strings.Split(nameAndVersion, ":")[0]
}

////////////////////////////////////////////

func NewAciWithManifest(path string, args BuildArgs, manifest spec.AciManifest) (*Img, error) {
	log.Get().Debug("New aci", path, args, manifest)
	cnt, err := PrepAci(path, args)
	if err != nil {
		return nil, err
	}
	cnt.manifest = manifest
	return cnt, nil
}

func NewAci(path string, args BuildArgs) (*Img, error) {
	manifest, err := readManifest(path + PATH_CNT_MANIFEST)
	if err != nil {
		log.Get().Debug(path, PATH_CNT_MANIFEST+" does not exists")
		return nil, err
	}
	return NewAciWithManifest(path, args, *manifest)
}

func PrepAci(aciPath string, args BuildArgs) (*Img, error) {
	cnt := new(Img)
	cnt.args = args

	if fullPath, err := filepath.Abs(aciPath); err != nil {
		log.Get().Panic("Cannot get fullpath of project", err)
	} else {
		cnt.path = fullPath
		cnt.target = cnt.path + PATH_TARGET
		if args.TargetPath != "" {
			currentAbsDir, err := filepath.Abs(args.TargetPath + "/" + cnt.manifest.NameAndVersion.ShortName())
			if err != nil {
				log.Get().Panic("invalid target path")
			}
			cnt.target = currentAbsDir
		}
		cnt.rootfs = cnt.target + PATH_ROOTFS
	}
	return cnt, nil
}

//////////////////////////////////////////////////////////////////

func readManifest(manifestPath string) (*spec.AciManifest, error) {
	manifest := spec.AciManifest{Aci: spec.AciDefinition{}}

	source, err := ioutil.ReadFile(manifestPath)
	if err != nil {
		return nil, err
	}
	err = yaml.Unmarshal([]byte(source), &manifest)
	if err != nil {
		log.Get().Panic(err)
	}

	return &manifest, nil
}

func (i *Img) checkBuilt() {
	if _, err := os.Stat(i.target + PATH_IMAGE_ACI); os.IsNotExist(err) {
		if err := i.Build(); err != nil {
			log.Get().Panic("Cannot Install since build failed")
		}
	}
}
<commit_msg>Chmod if file exist and is not executable<commit_after>package builder

import (
	"github.com/appc/spec/schema/types"
	"github.com/blablacar/cnt/log"
	"github.com/blablacar/cnt/spec"
	"github.com/ghodss/yaml"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

const EXEC_FILES = `
execute_files() {
  fdir=$1
  [ -d "$fdir" ] || return 0

  for file in $fdir; do
    [ -e "$file" ] && {
     [ -x "$file" ] || /cnt/bin/busybox chmod +x "$file"
    }
    echo -e "\e[1m\e[32mRunning script -> $file\e[0m"
    $file
  done
}`

const BUILD_SCRIPT = `#!/cnt/bin/busybox sh
set -x
set -e
export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

` + EXEC_FILES + `

execute_files "$ROOTFS/cnt/runlevels/inherit-build-early"
execute_files "$TARGET/runlevels/build"
`

const BUILD_SCRIPT_LATE = `#!/cnt/bin/busybox sh
set -x
set -e
export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

` + EXEC_FILES + `

execute_files "$TARGET/runlevels/build-late"
execute_files "$ROOTFS/cnt/runlevels/inherit-build-late"
`

const PRESTART = `#!/cnt/bin/busybox sh
set -x
set -e

BASEDIR=${0%/*}
CNT_PATH=/cnt

` + EXEC_FILES + `

execute_files ${CNT_PATH}/runlevels/prestart-early

${BASEDIR}/attributes-merger -i ${CNT_PATH}/attributes -e CONFD_OVERRIDE
export CONFD_DATA=$(cat attributes.json)
${BASEDIR}/confd -onetime -config-file=${CNT_PATH}/prestart/confd.toml

execute_files ${CNT_PATH}/runlevels/prestart-late
`

const PATH_MANIFEST = "/manifest"
const PATH_IMAGE_ACI = "/image.aci"
const PATH_ROOTFS = "/rootfs"
const PATH_TARGET = "/target"
const PATH_CNT = "/cnt"
const PATH_CNT_MANIFEST = "/cnt-manifest.yml"
const PATH_RUNLEVELS = "/runlevels"
const PATH_PRESTART_EARLY = "/prestart-early"
const PATH_PRESTART_LATE = "/prestart-late"
const PATH_INHERIT_BUILD_LATE = "/inherit-build-late"
const PATH_INHERIT_BUILD_EARLY = "/inherit-build-early"
const PATH_ATTRIBUTES = "/attributes"
const PATH_FILES = "/files"
const PATH_CONFD = "/confd"
const PATH_BUILD_LATE = "/build-late"
const PATH_BUILD_SETUP = "/build-setup"
const PATH_BUILD = "/build"
const PATH_CONFDOTD = "/conf.d"
const PATH_TEMPLATES = "/templates"

type Img struct {
	path     string
	target   string
	rootfs   string
	PodName  *spec.ACFullname
	manifest spec.AciManifest
	args     BuildArgs
}

func Version(nameAndVersion string) string {
	split := strings.Split(nameAndVersion, ":")
	if len(split) == 1 {
		return ""
	}
	return split[1]
}

func ShortNameId(name types.ACIdentifier) string {
	return strings.Split(string(name), "/")[1]
}

func ShortName(nameAndVersion string) string {
	return strings.Split(Name(nameAndVersion), "/")[1]
}

func Name(nameAndVersion string) string {
	return strings.Split(nameAndVersion, ":")[0]
}

////////////////////////////////////////////

func NewAciWithManifest(path string, args BuildArgs, manifest spec.AciManifest) (*Img, error) {
	log.Get().Debug("New aci", path, args, manifest)
	cnt, err := PrepAci(path, args)
	if err != nil {
		return nil, err
	}
	cnt.manifest = manifest
	return cnt, nil
}

func NewAci(path string, args BuildArgs) (*Img, error) {
	manifest, err := readManifest(path + PATH_CNT_MANIFEST)
	if err != nil {
		log.Get().Debug(path, PATH_CNT_MANIFEST+" does not exists")
		return nil, err
	}
	return NewAciWithManifest(path, args, *manifest)
}

func PrepAci(aciPath string, args BuildArgs) (*Img, error) {
	cnt := new(Img)
	cnt.args = args

	if fullPath, err := filepath.Abs(aciPath); err != nil {
		log.Get().Panic("Cannot get fullpath of project", err)
	} else {
		cnt.path = fullPath
		cnt.target = cnt.path + PATH_TARGET
		if args.TargetPath != "" {
			currentAbsDir, err := filepath.Abs(args.TargetPath + "/" + cnt.manifest.NameAndVersion.ShortName())
			if err != nil {
				log.Get().Panic("invalid target path")
			}
			cnt.target = currentAbsDir
		}
		cnt.rootfs = cnt.target + PATH_ROOTFS
	}
	return cnt, nil
}

//////////////////////////////////////////////////////////////////

func readManifest(manifestPath string) (*spec.AciManifest, error) {
	manifest := spec.AciManifest{Aci: spec.AciDefinition{}}

	source, err := ioutil.ReadFile(manifestPath)
	if err != nil {
		return nil, err
	}
	err = yaml.Unmarshal([]byte(source), &manifest)
	if err != nil {
		log.Get().Panic(err)
	}

	return &manifest, nil
}

func (i *Img) checkBuilt() {
	if _, err := os.Stat(i.target + PATH_IMAGE_ACI); os.IsNotExist(err) {
		if err := i.Build(); err != nil {
			log.Get().Panic("Cannot Install since build failed")
		}
	}
}
<|endoftext|>


<commit_before>package anderson

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/ryanuber/go-license"
)

const (
	maxParentHops = 10
)

type LicenseClassifier struct {
	Config Config
}

func (c LicenseClassifier) Classify(path string, importPath string) (LicenseStatus, string, error) {
	for hops := 0; hops < maxParentHops; hops++ {
		newPath := c.parentPath(path, hops)

		if c.pathIsAGopath(newPath) {
			break
		}

		status, licenseType, err := c.classifyPath(newPath, importPath)

		if status != LicenseTypeNoLicense {
			return status, licenseType, err
		}
	}

	return LicenseTypeNoLicense, "Unknown", nil
}

func (c LicenseClassifier) classifyPath(path string, importPath string) (LicenseStatus, string, error) {
	l, err := license.NewFromDir(path)

	if err != nil {
		switch err.Error() {
		case license.ErrNoLicenseFile:
			if contains(c.Config.Exceptions, importPath) {
				return LicenseTypeAllowed, "Unknown", nil
			}

			return LicenseTypeNoLicense, "Unknown", nil
		case license.ErrUnrecognizedLicense:
			if contains(c.Config.Exceptions, importPath) {
				return LicenseTypeAllowed, "Unknown", nil
			}

			return LicenseTypeUnknown, "Unknown", nil
		default:
			return LicenseTypeUnknown, "Error", fmt.Errorf("Could not determine license for: %s", importPath)
		}
	}

	if contains(c.Config.Blacklist, l.Type) {
		return LicenseTypeBanned, l.Type, nil
	}

	if contains(c.Config.Whitelist, l.Type) {
		return LicenseTypeAllowed, l.Type, nil
	}

	if contains(c.Config.Exceptions, importPath) {
		return LicenseTypeAllowed, l.Type, nil
	} else {
		return LicenseTypeMarginal, l.Type, nil
	}
}

func (c LicenseClassifier) pathIsAGopath(path string) bool {
	paths, _ := Gopaths()

	return contains(paths, path)
}

func (c LicenseClassifier) parentPath(path string, hops int) string {
	dots := strings.Fields(strings.Repeat(".. ", hops))
	elements := []string{}
	elements = append(elements, path)
	elements = append(elements, dots...)

	return filepath.Clean(filepath.Join(elements...))
}

func contains(haystack []string, needle string) bool {
	for _, element := range haystack {
		if element == needle {
			return true
		}
	}
	return false
}
<commit_msg>make golint happy<commit_after>package anderson

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/ryanuber/go-license"
)

const (
	maxParentHops = 10
)

type LicenseClassifier struct {
	Config Config
}

func (c LicenseClassifier) Classify(path string, importPath string) (LicenseStatus, string, error) {
	for hops := 0; hops < maxParentHops; hops++ {
		newPath := c.parentPath(path, hops)

		if c.pathIsAGopath(newPath) {
			break
		}

		status, licenseType, err := c.classifyPath(newPath, importPath)

		if status != LicenseTypeNoLicense {
			return status, licenseType, err
		}
	}

	return LicenseTypeNoLicense, "Unknown", nil
}

func (c LicenseClassifier) classifyPath(path string, importPath string) (LicenseStatus, string, error) {
	l, err := license.NewFromDir(path)

	if err != nil {
		switch err.Error() {
		case license.ErrNoLicenseFile:
			if contains(c.Config.Exceptions, importPath) {
				return LicenseTypeAllowed, "Unknown", nil
			}

			return LicenseTypeNoLicense, "Unknown", nil
		case license.ErrUnrecognizedLicense:
			if contains(c.Config.Exceptions, importPath) {
				return LicenseTypeAllowed, "Unknown", nil
			}

			return LicenseTypeUnknown, "Unknown", nil
		default:
			return LicenseTypeUnknown, "Error", fmt.Errorf("Could not determine license for: %s", importPath)
		}
	}

	if contains(c.Config.Blacklist, l.Type) {
		return LicenseTypeBanned, l.Type, nil
	}

	if contains(c.Config.Whitelist, l.Type) {
		return LicenseTypeAllowed, l.Type, nil
	}

	if contains(c.Config.Exceptions, importPath) {
		return LicenseTypeAllowed, l.Type, nil
	}

	return LicenseTypeMarginal, l.Type, nil
}

func (c LicenseClassifier) pathIsAGopath(path string) bool {
	paths, _ := Gopaths()

	return contains(paths, path)
}

func (c LicenseClassifier) parentPath(path string, hops int) string {
	dots := strings.Fields(strings.Repeat(".. ", hops))
	elements := []string{}
	elements = append(elements, path)
	elements = append(elements, dots...)

	return filepath.Clean(filepath.Join(elements...))
}

func contains(haystack []string, needle string) bool {
	for _, element := range haystack {
		if element == needle {
			return true
		}
	}
	return false
}
<|endoftext|>