<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"strings"
	"testing"
	"time"

	dataproc "cloud.google.com/go/dataproc/apiv1"
	"cloud.google.com/go/storage"
	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
	dataprocpb "google.golang.org/genproto/googleapis/cloud/dataproc/v1"
)

var (
	clusterName string
	bktName     string
	jobFilePath string
	jobFName    = "sum.py"
	code        = `import pyspark
sc = pyspark.SparkContext()
rdd = sc.parallelize((1,2,3,4,5))
sum = rdd.reduce(lambda x, y: x + y)`
	region = "us-central1"
)

func setup(t *testing.T, projectID string) {
	ctx := context.Background()
	flag.Parse()

	clusterName = "go-qs-test-" + projectID
	bktName = "go-dataproc-qs-test-" + projectID
	jobFilePath = fmt.Sprintf("gs://%s/%s", bktName, jobFName)

	sc, err := storage.NewClient(ctx)
	if err != nil {
		t.Errorf("storage.NewClient: %v", err)
	}

	testutil.CleanBucket(ctx, t, projectID, bktName)
	bkt := sc.Bucket(bktName)

	obj := bkt.Object(jobFName)

	w := obj.NewWriter(ctx)

	if _, err := fmt.Fprintf(w, code); err != nil {
		if err2 := w.Close(); err != nil {
			t.Errorf("Error writing to file and closing it: %v", err2)
		}
		t.Errorf("Error writing to file: %v", err)
	}

	if err := w.Close(); err != nil {
		t.Errorf("Error closing file: %v", err)
	}

	deleteClusters(ctx, projectID) // Ignore any errors.
}

func teardown(t *testing.T, projectID string) {
	ctx := context.Background()

	sc, err := storage.NewClient(ctx)
	if err != nil {
		t.Errorf("storage.NewClient: %v", err)
	}

	if err := sc.Bucket(bktName).Object(jobFName).Delete(ctx); err != nil {
		t.Errorf("Error deleting object: %v", err)
	}

	if err := sc.Bucket(bktName).Delete(ctx); err != nil {
		t.Errorf("Error deleting bucket: %v", err)
	}

	if err := deleteClusters(ctx, projectID); err != nil {
		t.Errorf("deleteClusters: %v", err)
	}
}

func deleteClusters(ctx context.Context, projectID string) error {
	endpoint := fmt.Sprintf("%s-dataproc.googleapis.com:443", region)
	client, err := dataproc.NewClusterControllerClient(ctx, option.WithEndpoint(endpoint))
	if err != nil {
		return fmt.Errorf("dataproc.NewClusterControllerClient: %v", err)
	}

	lReq := &dataprocpb.ListClustersRequest{ProjectId: projectID, Region: region}
	it := client.ListClusters(ctx, lReq)

	for {
		resp, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return fmt.Errorf("ListClusters.Next: %v", err)
		}
		if resp.ClusterName == clusterName {
			dReq := &dataprocpb.DeleteClusterRequest{ProjectId: projectID, Region: region, ClusterName: clusterName}
			op, err := client.DeleteCluster(ctx, dReq)
			if err != nil {
				return fmt.Errorf("DeleteCluster: %v", err)
			}

			if err := op.Wait(ctx); err != nil {
				return fmt.Errorf("DeleteCluster.Wait: %v", err)
			}
		}
	}
	return nil
}

func TestQuickstart(t *testing.T) {
	tc := testutil.SystemTest(t)
	m := testutil.BuildMain(t)
	setup(t, tc.ProjectID)
	defer teardown(t, tc.ProjectID)

	if !m.Built() {
		t.Fatalf("failed to build app")
	}

	testutil.Retry(t, 3, 30*time.Second, func(r *testutil.R) {
		stdOut, stdErr, err := m.Run(nil, 10*time.Minute,
			"--project_id", tc.ProjectID,
			"--region", region,
			"--cluster_name", clusterName,
			"--job_file_path", jobFilePath,
		)
		if err != nil {
			r.Errorf("stdout: %v", string(stdOut))
			r.Errorf("stderr: %v", string(stdErr))
			r.Errorf("execution failed: %v", err)
			return
		}

		got := string(stdOut)
		wants := []string{
			"Cluster created successfully",
			"Submitted job",
			"finished with state DONE:",
			"successfully deleted",
		}
		for _, want := range wants {
			if !strings.Contains(got, want) {
				r.Errorf("got %q, want to contain %q", got, want)
			}
		}
	})
}
<commit_msg>test(dataproc): make quickstart test delete before retry and E2E (#1686)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"strings"
	"testing"
	"time"

	dataproc "cloud.google.com/go/dataproc/apiv1"
	"cloud.google.com/go/storage"
	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
	dataprocpb "google.golang.org/genproto/googleapis/cloud/dataproc/v1"
)

var (
	clusterName string
	bktName     string
	jobFilePath string
	jobFName    = "sum.py"
	code        = `import pyspark
sc = pyspark.SparkContext()
rdd = sc.parallelize((1,2,3,4,5))
sum = rdd.reduce(lambda x, y: x + y)`
	region = "us-central1"
)

func setup(t *testing.T, projectID string) {
	ctx := context.Background()
	flag.Parse()

	clusterName = "go-qs-test-" + projectID
	bktName = "go-dataproc-qs-test-" + projectID
	jobFilePath = fmt.Sprintf("gs://%s/%s", bktName, jobFName)

	sc, err := storage.NewClient(ctx)
	if err != nil {
		t.Errorf("storage.NewClient: %v", err)
	}

	testutil.CleanBucket(ctx, t, projectID, bktName)
	bkt := sc.Bucket(bktName)

	obj := bkt.Object(jobFName)

	w := obj.NewWriter(ctx)

	if _, err := fmt.Fprintf(w, code); err != nil {
		if err2 := w.Close(); err != nil {
			t.Errorf("Error writing to file and closing it: %v", err2)
		}
		t.Errorf("Error writing to file: %v", err)
	}

	if err := w.Close(); err != nil {
		t.Errorf("Error closing file: %v", err)
	}

	deleteClusters(ctx, projectID) // Ignore any errors.
}

func teardown(t *testing.T, projectID string) {
	ctx := context.Background()

	sc, err := storage.NewClient(ctx)
	if err != nil {
		t.Errorf("storage.NewClient: %v", err)
	}

	if err := sc.Bucket(bktName).Object(jobFName).Delete(ctx); err != nil {
		t.Errorf("Error deleting object: %v", err)
	}

	if err := sc.Bucket(bktName).Delete(ctx); err != nil {
		t.Errorf("Error deleting bucket: %v", err)
	}

	if err := deleteClusters(ctx, projectID); err != nil {
		t.Errorf("deleteClusters: %v", err)
	}
}

func deleteClusters(ctx context.Context, projectID string) error {
	endpoint := fmt.Sprintf("%s-dataproc.googleapis.com:443", region)
	client, err := dataproc.NewClusterControllerClient(ctx, option.WithEndpoint(endpoint))
	if err != nil {
		return fmt.Errorf("dataproc.NewClusterControllerClient: %v", err)
	}

	lReq := &dataprocpb.ListClustersRequest{ProjectId: projectID, Region: region}
	it := client.ListClusters(ctx, lReq)

	for {
		resp, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return fmt.Errorf("ListClusters.Next: %v", err)
		}
		if resp.ClusterName == clusterName {
			dReq := &dataprocpb.DeleteClusterRequest{ProjectId: projectID, Region: region, ClusterName: clusterName}
			op, err := client.DeleteCluster(ctx, dReq)
			if err != nil {
				return fmt.Errorf("DeleteCluster: %v", err)
			}

			if err := op.Wait(ctx); err != nil {
				return fmt.Errorf("DeleteCluster.Wait: %v", err)
			}
		}
	}
	return nil
}

func TestQuickstart(t *testing.T) {
	tc := testutil.EndToEndTest(t)
	m := testutil.BuildMain(t)
	setup(t, tc.ProjectID)
	defer teardown(t, tc.ProjectID)

	if !m.Built() {
		t.Fatalf("failed to build app")
	}

	testutil.Retry(t, 3, 30*time.Second, func(r *testutil.R) {
		if err := deleteClusters(context.Background(), tc.ProjectID); err != nil {
			r.Errorf("failed to deleteClusters: %v", err)
			return
		}
		stdOut, stdErr, err := m.Run(nil, 10*time.Minute,
			"--project_id", tc.ProjectID,
			"--region", region,
			"--cluster_name", clusterName,
			"--job_file_path", jobFilePath,
		)
		if err != nil {
			r.Errorf("stdout: %v", string(stdOut))
			r.Errorf("stderr: %v", string(stdErr))
			r.Errorf("execution failed: %v", err)
			return
		}

		got := string(stdOut)
		wants := []string{
			"Cluster created successfully",
			"Submitted job",
			"finished with state DONE:",
			"successfully deleted",
		}
		for _, want := range wants {
			if !strings.Contains(got, want) {
				r.Errorf("got %q, want to contain %q", got, want)
			}
		}
	})
}
<|endoftext|>


<commit_before>package platform

import (
	"context"
	"time"
)

// UsageMetric used to track classes of usage.
type UsageMetric string

const (
	// UsageWriteRequestCount is the name of the metrics for tracking write request count.
	UsageWriteRequestCount UsageMetric = "usage_write_request_count"
	// UsageWriteRequestBytes is the name of the metrics for tracking the number of write bytes.
	UsageWriteRequestBytes UsageMetric = "usage_write_request_bytes"
	// UsageQueryRequestCount is the name of the metrics for tracking query request count.
	UsageQueryRequestCount UsageMetric = "usage_query_request_count"
	// UsageQueryRequestBytes is the name of the metrics for tracking the number of query bytes.
	UsageQueryRequestBytes UsageMetric = "usage_query_request_bytes"
)

// Usage is a metric associated with the utilization of a particular resource.
type Usage struct {
	OrganizationID *ID         `json:"organizationID,omitempty"`
	BucketID       *ID         `json:"bucketID,omitempty"`
	Type           UsageMetric `json:"type"`
	Value          float64     `json:"value"`
}

// UsageService is a service for accessing usage statistics.
type UsageService interface {
	GetUsage(ctx context.Context, filter UsageFilter) (map[UsageMetric]*Usage, error)
}

// UsageFilter is used to filter usage.
type UsageFilter struct {
	OrgID    *ID
	BucketID *ID
	Range    *Timespan
}

// Timespan represents a range of time.
type Timespan struct {
	Start time.Time `json:"start"`
	Stop  time.Time `json:"stop"`
}
<commit_msg>feat(usage): add values and series metrics to usage service<commit_after>package platform

import (
	"context"
	"time"
)

// UsageMetric used to track classes of usage.
type UsageMetric string

const (
	// UsageWriteRequestCount is the name of the metrics for tracking write request count.
	UsageWriteRequestCount UsageMetric = "usage_write_request_count"
	// UsageWriteRequestBytes is the name of the metrics for tracking the number of write bytes.
	UsageWriteRequestBytes UsageMetric = "usage_write_request_bytes"

	// UsageValues is the name of the metrics for tracking the number of values.
	UsageValues UsageMetric = "usage_values"
	// UsageSeries is the name of the metrics for tracking the number of series written.
	UsageSeries UsageMetric = "usage_series"

	// UsageQueryRequestCount is the name of the metrics for tracking query request count.
	UsageQueryRequestCount UsageMetric = "usage_query_request_count"
	// UsageQueryRequestBytes is the name of the metrics for tracking the number of query bytes.
	UsageQueryRequestBytes UsageMetric = "usage_query_request_bytes"
)

// Usage is a metric associated with the utilization of a particular resource.
type Usage struct {
	OrganizationID *ID         `json:"organizationID,omitempty"`
	BucketID       *ID         `json:"bucketID,omitempty"`
	Type           UsageMetric `json:"type"`
	Value          float64     `json:"value"`
}

// UsageService is a service for accessing usage statistics.
type UsageService interface {
	GetUsage(ctx context.Context, filter UsageFilter) (map[UsageMetric]*Usage, error)
}

// UsageFilter is used to filter usage.
type UsageFilter struct {
	OrgID    *ID
	BucketID *ID
	Range    *Timespan
}

// Timespan represents a range of time.
type Timespan struct {
	Start time.Time `json:"start"`
	Stop  time.Time `json:"stop"`
}
<|endoftext|>


<commit_before>package worker

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"regexp"
	"strings"
	"time"
)

type eventsWorker struct {
	sm                state.Manager
	qm                queue.Manager
	conf              config.Config
	log               flotillaLog.Logger
	pollInterval      time.Duration
	t                 tomb.Tomb
	queue             string
	emrJobStatusQueue string
	s3Client          *s3.S3
	kClient           kubernetes.Clientset
	emrHistoryServer  string
	emrMetricsServer  string
	emrMaxPodEvents   int
	eksEngine         engine.Engine
	emrEngine         engine.Engine
}

func (ew *eventsWorker) Initialize(conf config.Config, sm state.Manager, eksEngine engine.Engine, emrEngine engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, qm queue.Manager) error {
	ew.pollInterval = pollInterval
	ew.conf = conf
	ew.sm = sm
	ew.qm = qm
	ew.log = log
	ew.eksEngine = eksEngine
	ew.emrEngine = emrEngine
	eventsQueue, err := ew.qm.QurlFor(conf.GetString("eks.events_queue"), false)
	emrJobStatusQueue, err := ew.qm.QurlFor(conf.GetString("emr.job_status_queue"), false)
	ew.emrHistoryServer = conf.GetString("emr.history_server_uri")
	ew.emrMetricsServer = conf.GetString("emr.metrics_server_uri")
	if conf.IsSet("emr.max_attempt_count") {
		ew.emrMaxPodEvents = conf.GetInt("emr.max_pod_events")
	} else {
		ew.emrMaxPodEvents = 2000
	}

	if err != nil {
		_ = ew.log.Log("message", "Error receiving Kubernetes Event queue", "error", fmt.Sprintf("%+v", err))
		return nil
	}
	ew.queue = eventsQueue
	ew.emrJobStatusQueue = emrJobStatusQueue
	_ = ew.qm.Initialize(ew.conf, "eks")

	clusterName := conf.GetStringSlice("eks.cluster_override")[0]

	filename := fmt.Sprintf("%s/%s", conf.GetString("eks.kubeconfig_basepath"), clusterName)
	clientConf, err := clientcmd.BuildConfigFromFlags("", filename)
	if err != nil {
		_ = ew.log.Log("message", "error initializing-eksEngine-clusters", "error", fmt.Sprintf("%+v", err))
		return err
	}
	kClient, err := kubernetes.NewForConfig(clientConf)
	if err != nil {
		_ = ew.log.Log("message", fmt.Sprintf("%+v", err))
		return err
	}
	ew.kClient = *kClient
	return nil
}

func (ew *eventsWorker) GetTomb() *tomb.Tomb {
	return &ew.t
}

func (ew *eventsWorker) Run() error {
	for {
		select {
		case <-ew.t.Dying():
			_ = ew.log.Log("message", "A CloudTrail worker was terminated")
			return nil
		default:
			ew.runOnce()
			ew.runOnceEMR()
			time.Sleep(ew.pollInterval)
		}
	}
}

func (ew *eventsWorker) runOnceEMR() {
	emrEvent, err := ew.qm.ReceiveEMREvent(ew.emrJobStatusQueue)
	if err != nil {
		_ = ew.log.Log("message", "Error receiving EMR Events", "error", fmt.Sprintf("%+v", err))
		return
	}
	ew.processEventEMR(emrEvent)
}

func (ew *eventsWorker) processEventEMR(emrEvent state.EmrEvent) {
	if emrEvent.Detail == nil {
		return
	}

	emrJobId := emrEvent.Detail.ID
	run, err := ew.sm.GetRunByEMRJobId(*emrJobId)
	if err == nil {
		layout := "2020-08-31T17:27:50Z"
		timestamp, err := time.Parse(layout, *emrEvent.Time)
		if err != nil {
			timestamp = time.Now()
		}
		switch *emrEvent.Detail.State {
		case "COMPLETED":
			run.ExitCode = aws.Int64(0)
			run.Status = state.StatusStopped
			run.FinishedAt = &timestamp
			if run.StartedAt == nil {
				run.StartedAt = run.QueuedAt
			}
			run.ExitReason = emrEvent.Detail.StateDetails
		case "RUNNING":
			run.Status = state.StatusRunning
			run.StartedAt = &timestamp
		case "FAILED":
			run.ExitCode = aws.Int64(-1)
			run.Status = state.StatusStopped
			run.FinishedAt = &timestamp
			if run.StartedAt == nil {
				run.StartedAt = run.QueuedAt
			}
			run.ExitReason = emrEvent.Detail.FailureReason
		case "SUBMITTED":
			run.Status = state.StatusPending
		}

		_, err = ew.sm.UpdateRun(run.RunID, run)
		if err == nil {
			_ = emrEvent.Done()
		}
	}
}
func (ew *eventsWorker) runOnce() {
	kubernetesEvent, err := ew.qm.ReceiveKubernetesEvent(ew.queue)
	if err != nil {
		_ = ew.log.Log("message", "Error receiving Kubernetes Events", "error", fmt.Sprintf("%+v", err))
		return
	}
	ew.processEvent(kubernetesEvent)
}
func (ew *eventsWorker) processEMRPodEvents(kubernetesEvent state.KubernetesEvent) {
	if kubernetesEvent.InvolvedObject.Kind == "Pod" {
		pod, err := ew.kClient.CoreV1().Pods(kubernetesEvent.InvolvedObject.Namespace).Get(kubernetesEvent.InvolvedObject.Name, metav1.GetOptions{})
		var emrJobId *string = nil
		var sparkAppId *string = nil
		if err == nil {
			for k, v := range pod.Labels {
				if emrJobId == nil && strings.Compare(k, "emr-containers.amazonaws.com/job.id") == 0 {
					emrJobId = aws.String(v)
				}
				if sparkAppId == nil && strings.Compare(k, "spark-app-selector") == 0 {
					sparkAppId = aws.String(v)
				}
				if sparkAppId != nil && emrJobId != nil {
					break
				}
			}
		}

		if emrJobId != nil {
			run, err := ew.sm.GetRunByEMRJobId(*emrJobId)
			if err == nil {
				layout := "2020-08-31T17:27:50Z"
				timestamp, err := time.Parse(layout, kubernetesEvent.FirstTimestamp)
				if err != nil {
					timestamp = time.Now()
				}

				event := state.PodEvent{
					Timestamp:    &timestamp,
					EventType:    kubernetesEvent.Type,
					Reason:       kubernetesEvent.Reason,
					SourceObject: kubernetesEvent.InvolvedObject.Name,
					Message:      kubernetesEvent.Message,
				}

				var events state.PodEvents
				if run.PodEvents != nil {
					events = append(*run.PodEvents, event)
				} else {
					events = state.PodEvents{event}
				}
				run.PodEvents = &events

				if sparkAppId != nil {
					sparkHistoryUri := fmt.Sprintf("%s/%s/jobs", ew.emrHistoryServer, *sparkAppId)
					run.SparkExtension.SparkAppId = sparkAppId
					run.SparkExtension.HistoryUri = &sparkHistoryUri

					to := time.Now().UnixNano()
					if run.FinishedAt != nil {
						to = run.FinishedAt.UnixNano()
					}

					from := time.Now().UnixNano()
					if run.QueuedAt != nil {
						from = run.QueuedAt.UnixNano()
					}

					metricsUri :=
						fmt.Sprintf("%svar-run_id=%s&from=%d&to=%d&var-driver_id=%s",
							ew.emrMetricsServer,
							run.RunID,
							from/1000000,
							to/1000000,
							*run.SparkExtension.EMRJobId,
						)

					run.SparkExtension.MetricsUri = &metricsUri
				}

				run, err = ew.sm.UpdateRun(run.RunID, run)
				if err != nil {
					_ = ew.log.Log("message", "error saving kubernetes events", "emrJobId", emrJobId, "error", fmt.Sprintf("%+v", err))
				}

				if run.PodEvents != nil && len(*run.PodEvents) >= ew.emrMaxPodEvents {
					_ = ew.emrEngine.Terminate(run)
				}

			}
		}
		_ = kubernetesEvent.Done()
	}
}
func (ew *eventsWorker) processEvent(kubernetesEvent state.KubernetesEvent) {
	runId := kubernetesEvent.InvolvedObject.Labels.JobName
	if !strings.HasPrefix(runId, "eks") {
		ew.processEMRPodEvents(kubernetesEvent)
	}

	layout := "2020-08-31T17:27:50Z"
	timestamp, err := time.Parse(layout, kubernetesEvent.FirstTimestamp)

	if err != nil {
		timestamp = time.Now()
	}

	run, err := ew.sm.GetRun(runId)
	if err == nil {
		event := state.PodEvent{
			Timestamp:    &timestamp,
			EventType:    kubernetesEvent.Type,
			Reason:       kubernetesEvent.Reason,
			SourceObject: kubernetesEvent.InvolvedObject.Name,
			Message:      kubernetesEvent.Message,
		}

		var events state.PodEvents
		if run.PodEvents != nil {
			events = append(*run.PodEvents, event)
		} else {
			events = state.PodEvents{event}
		}
		run.PodEvents = &events
		if kubernetesEvent.Reason == "Scheduled" {
			podName, err := ew.parsePodName(kubernetesEvent)
			if err == nil {
				run.PodName = &podName
			}
		}

		if kubernetesEvent.Reason == "DeadlineExceeded" {
			run.ExitReason = &kubernetesEvent.Message
			exitCode := int64(124)
			run.ExitCode = &exitCode
			run.Status = state.StatusStopped
			run.StartedAt = run.QueuedAt
			run.FinishedAt = &timestamp
		}

		if kubernetesEvent.Reason == "Completed" {
			run.ExitReason = &kubernetesEvent.Message
			exitCode := int64(0)
			run.ExitCode = &exitCode
			run.Status = state.StatusStopped
			run.StartedAt = run.QueuedAt
			run.FinishedAt = &timestamp
		}
		run, err = ew.sm.UpdateRun(runId, run)
		if err != nil {
			_ = ew.log.Log("message", "error saving kubernetes events", "run", runId, "error", fmt.Sprintf("%+v", err))
		} else {
			_ = kubernetesEvent.Done()
		}
	}
}

func (ew *eventsWorker) parsePodName(kubernetesEvent state.KubernetesEvent) (string, error) {
	expression := regexp.MustCompile(`(eks-\w+-\w+-\w+-\w+-\w+-\w+)`)
	matches := expression.FindStringSubmatch(kubernetesEvent.Message)
	if matches != nil && len(matches) >= 1 {
		return matches[0], nil
	}
	return "", errors.Errorf("no pod name found for [%s]", kubernetesEvent.Message)
}
<commit_msg>switch to started_at time<commit_after>package worker

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"regexp"
	"strings"
	"time"
)

type eventsWorker struct {
	sm                state.Manager
	qm                queue.Manager
	conf              config.Config
	log               flotillaLog.Logger
	pollInterval      time.Duration
	t                 tomb.Tomb
	queue             string
	emrJobStatusQueue string
	s3Client          *s3.S3
	kClient           kubernetes.Clientset
	emrHistoryServer  string
	emrMetricsServer  string
	emrMaxPodEvents   int
	eksEngine         engine.Engine
	emrEngine         engine.Engine
}

func (ew *eventsWorker) Initialize(conf config.Config, sm state.Manager, eksEngine engine.Engine, emrEngine engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, qm queue.Manager) error {
	ew.pollInterval = pollInterval
	ew.conf = conf
	ew.sm = sm
	ew.qm = qm
	ew.log = log
	ew.eksEngine = eksEngine
	ew.emrEngine = emrEngine
	eventsQueue, err := ew.qm.QurlFor(conf.GetString("eks.events_queue"), false)
	emrJobStatusQueue, err := ew.qm.QurlFor(conf.GetString("emr.job_status_queue"), false)
	ew.emrHistoryServer = conf.GetString("emr.history_server_uri")
	ew.emrMetricsServer = conf.GetString("emr.metrics_server_uri")
	if conf.IsSet("emr.max_attempt_count") {
		ew.emrMaxPodEvents = conf.GetInt("emr.max_pod_events")
	} else {
		ew.emrMaxPodEvents = 2000
	}

	if err != nil {
		_ = ew.log.Log("message", "Error receiving Kubernetes Event queue", "error", fmt.Sprintf("%+v", err))
		return nil
	}
	ew.queue = eventsQueue
	ew.emrJobStatusQueue = emrJobStatusQueue
	_ = ew.qm.Initialize(ew.conf, "eks")

	clusterName := conf.GetStringSlice("eks.cluster_override")[0]

	filename := fmt.Sprintf("%s/%s", conf.GetString("eks.kubeconfig_basepath"), clusterName)
	clientConf, err := clientcmd.BuildConfigFromFlags("", filename)
	if err != nil {
		_ = ew.log.Log("message", "error initializing-eksEngine-clusters", "error", fmt.Sprintf("%+v", err))
		return err
	}
	kClient, err := kubernetes.NewForConfig(clientConf)
	if err != nil {
		_ = ew.log.Log("message", fmt.Sprintf("%+v", err))
		return err
	}
	ew.kClient = *kClient
	return nil
}

func (ew *eventsWorker) GetTomb() *tomb.Tomb {
	return &ew.t
}

func (ew *eventsWorker) Run() error {
	for {
		select {
		case <-ew.t.Dying():
			_ = ew.log.Log("message", "A CloudTrail worker was terminated")
			return nil
		default:
			ew.runOnce()
			ew.runOnceEMR()
			time.Sleep(ew.pollInterval)
		}
	}
}

func (ew *eventsWorker) runOnceEMR() {
	emrEvent, err := ew.qm.ReceiveEMREvent(ew.emrJobStatusQueue)
	if err != nil {
		_ = ew.log.Log("message", "Error receiving EMR Events", "error", fmt.Sprintf("%+v", err))
		return
	}
	ew.processEventEMR(emrEvent)
}

func (ew *eventsWorker) processEventEMR(emrEvent state.EmrEvent) {
	if emrEvent.Detail == nil {
		return
	}

	emrJobId := emrEvent.Detail.ID
	run, err := ew.sm.GetRunByEMRJobId(*emrJobId)
	if err == nil {
		layout := "2020-08-31T17:27:50Z"
		timestamp, err := time.Parse(layout, *emrEvent.Time)
		if err != nil {
			timestamp = time.Now()
		}
		switch *emrEvent.Detail.State {
		case "COMPLETED":
			run.ExitCode = aws.Int64(0)
			run.Status = state.StatusStopped
			run.FinishedAt = &timestamp
			if run.StartedAt == nil {
				run.StartedAt = run.QueuedAt
			}
			run.ExitReason = emrEvent.Detail.StateDetails
		case "RUNNING":
			run.Status = state.StatusRunning
			run.StartedAt = &timestamp
		case "FAILED":
			run.ExitCode = aws.Int64(-1)
			run.Status = state.StatusStopped
			run.FinishedAt = &timestamp
			if run.StartedAt == nil {
				run.StartedAt = run.QueuedAt
			}
			run.ExitReason = emrEvent.Detail.FailureReason
		case "SUBMITTED":
			run.Status = state.StatusPending
		}

		_, err = ew.sm.UpdateRun(run.RunID, run)
		if err == nil {
			_ = emrEvent.Done()
		}
	}
}
func (ew *eventsWorker) runOnce() {
	kubernetesEvent, err := ew.qm.ReceiveKubernetesEvent(ew.queue)
	if err != nil {
		_ = ew.log.Log("message", "Error receiving Kubernetes Events", "error", fmt.Sprintf("%+v", err))
		return
	}
	ew.processEvent(kubernetesEvent)
}
func (ew *eventsWorker) processEMRPodEvents(kubernetesEvent state.KubernetesEvent) {
	if kubernetesEvent.InvolvedObject.Kind == "Pod" {
		pod, err := ew.kClient.CoreV1().Pods(kubernetesEvent.InvolvedObject.Namespace).Get(kubernetesEvent.InvolvedObject.Name, metav1.GetOptions{})
		var emrJobId *string = nil
		var sparkAppId *string = nil
		if err == nil {
			for k, v := range pod.Labels {
				if emrJobId == nil && strings.Compare(k, "emr-containers.amazonaws.com/job.id") == 0 {
					emrJobId = aws.String(v)
				}
				if sparkAppId == nil && strings.Compare(k, "spark-app-selector") == 0 {
					sparkAppId = aws.String(v)
				}
				if sparkAppId != nil && emrJobId != nil {
					break
				}
			}
		}

		if emrJobId != nil {
			run, err := ew.sm.GetRunByEMRJobId(*emrJobId)
			if err == nil {
				layout := "2020-08-31T17:27:50Z"
				timestamp, err := time.Parse(layout, kubernetesEvent.FirstTimestamp)
				if err != nil {
					timestamp = time.Now()
				}

				event := state.PodEvent{
					Timestamp:    &timestamp,
					EventType:    kubernetesEvent.Type,
					Reason:       kubernetesEvent.Reason,
					SourceObject: kubernetesEvent.InvolvedObject.Name,
					Message:      kubernetesEvent.Message,
				}

				var events state.PodEvents
				if run.PodEvents != nil {
					events = append(*run.PodEvents, event)
				} else {
					events = state.PodEvents{event}
				}
				run.PodEvents = &events

				if sparkAppId != nil {
					sparkHistoryUri := fmt.Sprintf("%s/%s/jobs", ew.emrHistoryServer, *sparkAppId)
					run.SparkExtension.SparkAppId = sparkAppId
					run.SparkExtension.HistoryUri = &sparkHistoryUri

					to := time.Now().UnixNano()
					if run.FinishedAt != nil {
						to = run.FinishedAt.UnixNano()
					}

					from := time.Now().UnixNano()
					if run.StartedAt != nil {
						from = run.StartedAt.UnixNano()
					}

					metricsUri :=
						fmt.Sprintf("%svar-run_id=%s&from=%d&to=%d&var-driver_id=%s",
							ew.emrMetricsServer,
							run.RunID,
							from/1000000,
							to/1000000,
							*run.SparkExtension.EMRJobId,
						)

					run.SparkExtension.MetricsUri = &metricsUri
				}

				run, err = ew.sm.UpdateRun(run.RunID, run)
				if err != nil {
					_ = ew.log.Log("message", "error saving kubernetes events", "emrJobId", emrJobId, "error", fmt.Sprintf("%+v", err))
				}

				if run.PodEvents != nil && len(*run.PodEvents) >= ew.emrMaxPodEvents {
					_ = ew.emrEngine.Terminate(run)
				}

			}
		}
		_ = kubernetesEvent.Done()
	}
}
func (ew *eventsWorker) processEvent(kubernetesEvent state.KubernetesEvent) {
	runId := kubernetesEvent.InvolvedObject.Labels.JobName
	if !strings.HasPrefix(runId, "eks") {
		ew.processEMRPodEvents(kubernetesEvent)
	}

	layout := "2020-08-31T17:27:50Z"
	timestamp, err := time.Parse(layout, kubernetesEvent.FirstTimestamp)

	if err != nil {
		timestamp = time.Now()
	}

	run, err := ew.sm.GetRun(runId)
	if err == nil {
		event := state.PodEvent{
			Timestamp:    &timestamp,
			EventType:    kubernetesEvent.Type,
			Reason:       kubernetesEvent.Reason,
			SourceObject: kubernetesEvent.InvolvedObject.Name,
			Message:      kubernetesEvent.Message,
		}

		var events state.PodEvents
		if run.PodEvents != nil {
			events = append(*run.PodEvents, event)
		} else {
			events = state.PodEvents{event}
		}
		run.PodEvents = &events
		if kubernetesEvent.Reason == "Scheduled" {
			podName, err := ew.parsePodName(kubernetesEvent)
			if err == nil {
				run.PodName = &podName
			}
		}

		if kubernetesEvent.Reason == "DeadlineExceeded" {
			run.ExitReason = &kubernetesEvent.Message
			exitCode := int64(124)
			run.ExitCode = &exitCode
			run.Status = state.StatusStopped
			run.StartedAt = run.QueuedAt
			run.FinishedAt = &timestamp
		}

		if kubernetesEvent.Reason == "Completed" {
			run.ExitReason = &kubernetesEvent.Message
			exitCode := int64(0)
			run.ExitCode = &exitCode
			run.Status = state.StatusStopped
			run.StartedAt = run.QueuedAt
			run.FinishedAt = &timestamp
		}
		run, err = ew.sm.UpdateRun(runId, run)
		if err != nil {
			_ = ew.log.Log("message", "error saving kubernetes events", "run", runId, "error", fmt.Sprintf("%+v", err))
		} else {
			_ = kubernetesEvent.Done()
		}
	}
}

func (ew *eventsWorker) parsePodName(kubernetesEvent state.KubernetesEvent) (string, error) {
	expression := regexp.MustCompile(`(eks-\w+-\w+-\w+-\w+-\w+-\w+)`)
	matches := expression.FindStringSubmatch(kubernetesEvent.Message)
	if matches != nil && len(matches) >= 1 {
		return matches[0], nil
	}
	return "", errors.Errorf("no pod name found for [%s]", kubernetesEvent.Message)
}
<|endoftext|>


<commit_before>package jsonm

import "strings"

func Read(input []byte) [][]byte {
	inputString := string(input)
	inputSlice := strings.Split(inputString, "\n")
	for

}

<commit_msg>remove non-functional code<commit_after><|endoftext|>


<commit_before>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package environments

import (
	"errors"

	ppError "github.com/pufferpanel/pufferd/errors"
	"github.com/docker/docker/client"
	"context"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/strslice"
	"github.com/docker/docker/api/types/network"
	"io"
	"io/ioutil"
	"time"
	"runtime"
	"os"
	"fmt"
)

type docker struct {
	*BaseEnvironment
	ContainerId string `json:"-"`
	ImageName   string `json:"image"`
	connection  types.HijackedResponse
	cli			*client.Client
	downloadingImage bool
}

func createDocker(containerId, imageName string) *docker {
	if imageName == "" {
		imageName = "pufferpanel/generic"
	}
	d := &docker{BaseEnvironment: &BaseEnvironment{Type: "docker"}, ContainerId: containerId, ImageName: imageName}
	d.BaseEnvironment.executeAsync = d.dockerExecuteAsync
	d.BaseEnvironment.waitForMainProcess = d.WaitForMainProcess
	return d
}

func (d *docker) dockerExecuteAsync(cmd string, args []string, callback func(graceful bool)) (error) {
	running, err := d.IsRunning()
	if err != nil {
		return err

	}
	if running {
		return errors.New("container is already running")
	}

	if d.downloadingImage {
		return errors.New("container image is downloading, cannot execute")
	}

	client, err := d.getClient()
	ctx := context.Background()

	exists, err := d.doesContainerExist(client, ctx)

	if err != nil {
		return err
	}

	//container does not exist
	if !exists {
		err = d.createContainer(client, ctx, cmd, args, d.RootDirectory)
		if err != nil {
			return err
		}
	}

	config := types.ContainerAttachOptions{
		Stdin:  true,
		Stdout: true,
		Stderr: true,
		Stream: true,
	}

	d.connection, err = client.ContainerAttach(ctx, d.ContainerId, config)
	if err != nil {
		return err
	}

	d.wait.Add(1)

	go func() {
		defer d.connection.Close()
		wrapper := d.createWrapper()
		io.Copy(wrapper, d.connection.Reader)
		c, _ := d.getClient()
		c.ContainerStop(context.Background(), d.ContainerId, nil)
		time.Sleep(3 * time.Second)
		d.wait.Done()
	}()

	startOpts := types.ContainerStartOptions{
	}

	err = client.ContainerStart(ctx, d.ContainerId, startOpts)
	if err != nil {
		return err
	}
	return err
}

func (d *docker) ExecuteInMainProcess(cmd string) (err error) {
	running, err := d.IsRunning()
	if err != nil {
		return
	}
	if !running {
		err = errors.New("main process has not been started")
		return
	}

	d.connection.Conn.Write([]byte(cmd + "\n"))
	return
}

func (d *docker) Kill() (err error) {
	running, err := d.IsRunning()
	if err != nil {
		return err
	}

	if !running {
		return
	}

	client, err := d.getClient()
	ctx := context.Background()
	err = client.ContainerKill(ctx, d.ContainerId, "SIGKILL")
	return
}

func (d *docker) Create() error {
	err := os.Mkdir(d.RootDirectory, 0755)
	if err != nil {
		return err
	}

	go func() {
		cli, err := d.getClient()
		if err != nil {
			return
		}
		err = d.pullImage(cli, context.Background(), false)
	}()

	return err
}

func (d *docker) IsRunning() (bool, error) {
	client, err := d.getClient()
	if err != nil {
		logging.Error("Error checking run status", err)
		return false, err
	}

	ctx := context.Background()

	exists, err := d.doesContainerExist(client, ctx)
	if !exists {
		return false, err
	}

	stats, err := client.ContainerInspect(ctx, d.ContainerId)
	if err != nil {
		logging.Error("Error checking run status", err)
		return false, err
	}
	return stats.State.Running, nil
}

func (d *docker) GetStats() (map[string]interface{}, error) {
	running, err := d.IsRunning()
	if err != nil {
		return nil, err
	}

	if !running {
		return nil, ppError.NewServerOffline()
	}

	resultMap := make(map[string]interface{})
	resultMap["memory"] = 0
	resultMap["cpu"] = 0
	return resultMap, nil
}

func (e *docker) WaitForMainProcess() error {
	return e.WaitForMainProcessFor(0)
}

func (e *docker) WaitForMainProcessFor(timeout int) (err error) {
	running, err := e.IsRunning()
	if err != nil {
		return
	}
	if running {
		if timeout > 0 {
			var timer = time.AfterFunc(time.Duration(timeout)*time.Millisecond, func() {
				err = e.Kill()
			})
			e.wait.Wait()
			timer.Stop()
		} else {
			e.wait.Wait()
		}
	}
	return
}

func (d *docker) getClient() (*client.Client, error) {
	var err error = nil
	if d.cli == nil {
		d.cli, err = client.NewEnvClient()
	}
	return d.cli, err
}

func (d *docker) doesContainerExist(client *client.Client, ctx context.Context) (bool, error) {
	opts := types.ContainerListOptions{
		Filters: filters.NewArgs(),
	}

	opts.All = true
	opts.Filters.Add("name", d.ContainerId)

	existingContainers, err := client.ContainerList(ctx, opts)

	logging.Debugf("Does container (%s) exist?: %t", d.ContainerId, len(existingContainers) > 0)

	if len(existingContainers) == 0 {
		return false, err
	} else {
		return true, err
	}
}

func (d *docker) pullImage(client *client.Client, ctx context.Context, force bool) error {
	exists := false

	opts := types.ImageListOptions{
		All:     true,
		Filters: filters.NewArgs(),
	}
	opts.Filters.Add("reference", d.ImageName)
	images, err := client.ImageList(ctx, opts)

	if err != nil {
		return err
	}

	if len(images) >= 1 {
		exists = true
	}

	logging.Debugf("Does image %v exist? %v", d.ImageName, exists)

	if exists && !force {
		return nil
	}

	op := types.ImagePullOptions{
	}

	logging.Debugf("Downloading image %v", d.ImageName)
	d.DisplayToConsole("Downloading image for container, please wait\n")

	d.downloadingImage = true

	r, err := client.ImagePull(ctx, d.ImageName, op)
	defer r.Close()
	if err != nil {
		return err
	}
	_, err = io.Copy(ioutil.Discard, r)

	d.downloadingImage = false
	logging.Debugf("Downloaded image %v", d.ImageName)
	d.DisplayToConsole("Downloaded image for container\n")
	return err
}

func (d *docker) createContainer(client *client.Client, ctx context.Context, cmd string, args []string, root string) error {
	err := d.pullImage(client, ctx, false)

	if err != nil {
		return err
	}

	cmdSlice := strslice.StrSlice{}

	cmdSlice = append(cmdSlice, cmd)

	for _, v := range args {
		cmdSlice = append(cmdSlice, v)
	}

	newEnv := os.Environ()
	//newEnv["home"] = root
	newEnv = append(newEnv, "HOME=" + root)

	config := &container.Config{
		AttachStderr:    true,
		AttachStdin:     true,
		AttachStdout:    true,
		Tty:             true,
		OpenStdin:       true,
		NetworkDisabled: false,
		Cmd:             cmdSlice,
		Image:           d.ImageName,
		WorkingDir:		 root,
		Env:			 newEnv,
	}

	if runtime.GOOS == "linux" {
		config.User = fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid())
	}

	hostConfig := &container.HostConfig{
		AutoRemove:  true,
		NetworkMode: "host",
		Resources: container.Resources{
		},
		Binds: make([]string, 0),
	}
	hostConfig.Binds = append(hostConfig.Binds, root+":"+root)

	networkConfig := &network.NetworkingConfig{
	}

	_, err = client.ContainerCreate(ctx, config, hostConfig, networkConfig, d.ContainerId)
	return err
}
<commit_msg>Reduce wait<commit_after>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package environments

import (
	"errors"

	ppError "github.com/pufferpanel/pufferd/errors"
	"github.com/docker/docker/client"
	"context"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/strslice"
	"github.com/docker/docker/api/types/network"
	"io"
	"io/ioutil"
	"time"
	"runtime"
	"os"
	"fmt"
)

type docker struct {
	*BaseEnvironment
	ContainerId string `json:"-"`
	ImageName   string `json:"image"`
	connection  types.HijackedResponse
	cli			*client.Client
	downloadingImage bool
}

func createDocker(containerId, imageName string) *docker {
	if imageName == "" {
		imageName = "pufferpanel/generic"
	}
	d := &docker{BaseEnvironment: &BaseEnvironment{Type: "docker"}, ContainerId: containerId, ImageName: imageName}
	d.BaseEnvironment.executeAsync = d.dockerExecuteAsync
	d.BaseEnvironment.waitForMainProcess = d.WaitForMainProcess
	return d
}

func (d *docker) dockerExecuteAsync(cmd string, args []string, callback func(graceful bool)) (error) {
	running, err := d.IsRunning()
	if err != nil {
		return err

	}
	if running {
		return errors.New("container is already running")
	}

	if d.downloadingImage {
		return errors.New("container image is downloading, cannot execute")
	}

	client, err := d.getClient()
	ctx := context.Background()

	exists, err := d.doesContainerExist(client, ctx)

	if err != nil {
		return err
	}

	//container does not exist
	if !exists {
		err = d.createContainer(client, ctx, cmd, args, d.RootDirectory)
		if err != nil {
			return err
		}
	}

	config := types.ContainerAttachOptions{
		Stdin:  true,
		Stdout: true,
		Stderr: true,
		Stream: true,
	}

	d.connection, err = client.ContainerAttach(ctx, d.ContainerId, config)
	if err != nil {
		return err
	}

	d.wait.Add(1)

	go func() {
		defer d.connection.Close()
		wrapper := d.createWrapper()
		io.Copy(wrapper, d.connection.Reader)
		c, _ := d.getClient()
		c.ContainerStop(context.Background(), d.ContainerId, nil)
		time.Sleep(1 * time.Second)
		d.wait.Done()
	}()

	startOpts := types.ContainerStartOptions{
	}

	err = client.ContainerStart(ctx, d.ContainerId, startOpts)
	if err != nil {
		return err
	}
	return err
}

func (d *docker) ExecuteInMainProcess(cmd string) (err error) {
	running, err := d.IsRunning()
	if err != nil {
		return
	}
	if !running {
		err = errors.New("main process has not been started")
		return
	}

	d.connection.Conn.Write([]byte(cmd + "\n"))
	return
}

func (d *docker) Kill() (err error) {
	running, err := d.IsRunning()
	if err != nil {
		return err
	}

	if !running {
		return
	}

	client, err := d.getClient()
	ctx := context.Background()
	err = client.ContainerKill(ctx, d.ContainerId, "SIGKILL")
	return
}

func (d *docker) Create() error {
	err := os.Mkdir(d.RootDirectory, 0755)
	if err != nil {
		return err
	}

	go func() {
		cli, err := d.getClient()
		if err != nil {
			return
		}
		err = d.pullImage(cli, context.Background(), false)
	}()

	return err
}

func (d *docker) IsRunning() (bool, error) {
	client, err := d.getClient()
	if err != nil {
		logging.Error("Error checking run status", err)
		return false, err
	}

	ctx := context.Background()

	exists, err := d.doesContainerExist(client, ctx)
	if !exists {
		return false, err
	}

	stats, err := client.ContainerInspect(ctx, d.ContainerId)
	if err != nil {
		logging.Error("Error checking run status", err)
		return false, err
	}
	return stats.State.Running, nil
}

func (d *docker) GetStats() (map[string]interface{}, error) {
	running, err := d.IsRunning()
	if err != nil {
		return nil, err
	}

	if !running {
		return nil, ppError.NewServerOffline()
	}

	resultMap := make(map[string]interface{})
	resultMap["memory"] = 0
	resultMap["cpu"] = 0
	return resultMap, nil
}

func (e *docker) WaitForMainProcess() error {
	return e.WaitForMainProcessFor(0)
}

func (e *docker) WaitForMainProcessFor(timeout int) (err error) {
	running, err := e.IsRunning()
	if err != nil {
		return
	}
	if running {
		if timeout > 0 {
			var timer = time.AfterFunc(time.Duration(timeout)*time.Millisecond, func() {
				err = e.Kill()
			})
			e.wait.Wait()
			timer.Stop()
		} else {
			e.wait.Wait()
		}
	}
	return
}

func (d *docker) getClient() (*client.Client, error) {
	var err error = nil
	if d.cli == nil {
		d.cli, err = client.NewEnvClient()
	}
	return d.cli, err
}

func (d *docker) doesContainerExist(client *client.Client, ctx context.Context) (bool, error) {
	opts := types.ContainerListOptions{
		Filters: filters.NewArgs(),
	}

	opts.All = true
	opts.Filters.Add("name", d.ContainerId)

	existingContainers, err := client.ContainerList(ctx, opts)

	logging.Debugf("Does container (%s) exist?: %t", d.ContainerId, len(existingContainers) > 0)

	if len(existingContainers) == 0 {
		return false, err
	} else {
		return true, err
	}
}

func (d *docker) pullImage(client *client.Client, ctx context.Context, force bool) error {
	exists := false

	opts := types.ImageListOptions{
		All:     true,
		Filters: filters.NewArgs(),
	}
	opts.Filters.Add("reference", d.ImageName)
	images, err := client.ImageList(ctx, opts)

	if err != nil {
		return err
	}

	if len(images) >= 1 {
		exists = true
	}

	logging.Debugf("Does image %v exist? %v", d.ImageName, exists)

	if exists && !force {
		return nil
	}

	op := types.ImagePullOptions{
	}

	logging.Debugf("Downloading image %v", d.ImageName)
	d.DisplayToConsole("Downloading image for container, please wait\n")

	d.downloadingImage = true

	r, err := client.ImagePull(ctx, d.ImageName, op)
	defer r.Close()
	if err != nil {
		return err
	}
	_, err = io.Copy(ioutil.Discard, r)

	d.downloadingImage = false
	logging.Debugf("Downloaded image %v", d.ImageName)
	d.DisplayToConsole("Downloaded image for container\n")
	return err
}

func (d *docker) createContainer(client *client.Client, ctx context.Context, cmd string, args []string, root string) error {
	err := d.pullImage(client, ctx, false)

	if err != nil {
		return err
	}

	cmdSlice := strslice.StrSlice{}

	cmdSlice = append(cmdSlice, cmd)

	for _, v := range args {
		cmdSlice = append(cmdSlice, v)
	}

	newEnv := os.Environ()
	//newEnv["home"] = root
	newEnv = append(newEnv, "HOME=" + root)

	config := &container.Config{
		AttachStderr:    true,
		AttachStdin:     true,
		AttachStdout:    true,
		Tty:             true,
		OpenStdin:       true,
		NetworkDisabled: false,
		Cmd:             cmdSlice,
		Image:           d.ImageName,
		WorkingDir:		 root,
		Env:			 newEnv,
	}

	if runtime.GOOS == "linux" {
		config.User = fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid())
	}

	hostConfig := &container.HostConfig{
		AutoRemove:  true,
		NetworkMode: "host",
		Resources: container.Resources{
		},
		Binds: make([]string, 0),
	}
	hostConfig.Binds = append(hostConfig.Binds, root+":"+root)

	networkConfig := &network.NetworkingConfig{
	}

	_, err = client.ContainerCreate(ctx, config, hostConfig, networkConfig, d.ContainerId)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"golang.org/x/sys/unix"
)

var (
	fstab   = flag.String("fstab", "/etc/fstab", "Specify path to fstab(5).")
	fstype  = flag.String("type", "", "Specify filesystem type.")
	options = flag.String("options", "", "Specify mount options.")
)

type fsEntry struct {
	devpath string
	mntpt   string
	fstype  string
	options []string
}

func main() {
	log.SetPrefix("mount: ")
	log.SetFlags(0)

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: mount [options] device [mountpoint]")
		fmt.Fprintln(os.Stderr, "       mount")
		flag.PrintDefaults()
	}
	flag.Parse()

	if flag.NArg() == 0 {
		if err := printMounts(); err != nil {
			log.Fatal(err)
		}
		os.Exit(0)
	}

	if flag.NArg() > 2 {
		flag.Usage()
		os.Exit(1)
	}

	entry, err := readEntry(flag.Arg(0))
	if err != nil && !os.IsNotExist(err) {
		log.Fatal(err)
	}

	if entry.mntpt == "" && flag.Arg(1) == "" {
		log.Fatal("mountpoint not specified")
	} else if flag.Arg(1) != "" {
		entry.mntpt = flag.Arg(1)
	}

	if *fstype != "" {
		entry.fstype = *fstype
	}
	if entry.fstype == "" {
		log.Fatal("filesystem type not specified")
	}
	if *options != "" {
		entry.options = strings.Split(*options, ",")
	}

	var flags uintptr
	for _, s := range entry.options {
		switch s {
		case "ro":
			flags |= unix.MS_RDONLY
		case "rw":
			// rw is default
		case "remount":
			flags |= unix.MS_REMOUNT
		default:
			log.Printf("unrecognized option %q", s)
		}
	}
	if err := unix.Mount(entry.devpath, entry.mntpt, entry.fstype, flags, ""); err != nil {
		log.Fatal(err)
	}
}

func printMounts() error {
	f, err := os.Open("/proc/mounts")
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err := io.Copy(os.Stdout, f); err != nil {
		return err
	}
	return nil
}

func readEntry(devpath string) (*fsEntry, error) {
	// default is rw if no entry
	e := fsEntry{
		devpath: devpath,
		options: []string{"rw"},
	}

	f, err := os.Open(*fstab)
	if err != nil {
		return &e, err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "#") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) < 4 {
			continue
		}
		if fields[0] != devpath {
			continue
		}
		e.mntpt = fields[1]
		e.fstype = fields[2]
		e.options = strings.Split(fields[3], ",")
		break
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return &e, nil
}
<commit_msg>mount: flesh out options, handle mntpt-only invocation<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"syscall"
)

var (
	fstab   = flag.String("fstab", "/etc/fstab", "Specify path to fstab(5).")
	fstype  = flag.String("type", "", "Specify filesystem type.")
	options = flag.String("options", "", "Specify mount options.")
)

type fsEntry struct {
	devpath string
	mntpt   string
	fstype  string
	options []string
}

func main() {
	log.SetPrefix("mount: ")
	log.SetFlags(0)

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: mount [options] device [mountpoint]")
		fmt.Fprintln(os.Stderr, "       mount")
		flag.PrintDefaults()
	}
	flag.Parse()

	if flag.NArg() == 0 {
		if err := printMounts(); err != nil {
			log.Fatal(err)
		}
		os.Exit(0)
	}

	if flag.NArg() > 2 {
		flag.Usage()
		os.Exit(1)
	}

	entry, err := readEntry(flag.Arg(0))
	if err != nil {
		// couln't read fstab entry; assume first arg is devpath
		entry = &fsEntry{
			devpath: flag.Arg(0),
		}
	}

	if entry.mntpt == "" && flag.Arg(1) == "" {
		log.Fatal("mountpoint not specified")
	} else if flag.Arg(1) != "" {
		entry.mntpt = flag.Arg(1)
	}

	if *fstype != "" {
		entry.fstype = *fstype
	}
	if entry.fstype == "" {
		log.Fatal("filesystem type not specified")
	}
	if *options != "" {
		entry.options = strings.Split(*options, ",")
	}

	var flags int
	var data []string
	for _, s := range entry.options {
		switch s {
		case "async":
			flags |= syscall.MS_ASYNC
		case "atime":
			flags &^= syscall.MS_NOATIME
		case "noatime":
			flags |= syscall.MS_NOATIME
		case "defaults":
			// rw | suid | dev | exec | nouser | async
			flags = syscall.MS_ASYNC
		case "dev":
			flags &^= syscall.MS_NODEV
		case "nodev":
			flags |= syscall.MS_NODEV
		case "diratime":
			flags &^= syscall.MS_NODIRATIME
		case "nodiratime":
			flags |= syscall.MS_NODIRATIME
		case "dirsync":
			flags |= syscall.MS_DIRSYNC
		case "exec":
			flags &^= syscall.MS_NOEXEC
		case "iversion":
			flags |= syscall.MS_I_VERSION
		case "noiversion":
			flags &^= syscall.MS_I_VERSION
		case "mand":
			flags |= syscall.MS_MANDLOCK
		case "nomand":
			flags &^= syscall.MS_MANDLOCK
		case "relatime":
			flags |= syscall.MS_RELATIME
		case "norelatime":
			flags &^= syscall.MS_RELATIME
		case "strictatime":
			flags |= syscall.MS_STRICTATIME
		case "nostrictatime":
			flags &^= syscall.MS_STRICTATIME
		case "suid":
			flags &^= syscall.MS_NOSUID
		case "nosuid":
			flags |= syscall.MS_NOSUID
		case "remount":
			flags |= syscall.MS_REMOUNT
		case "ro":
			flags |= syscall.MS_RDONLY
		case "rw":
			flags &^= syscall.MS_RDONLY
		case "sync":
			flags |= syscall.MS_SYNC
		case "user":
			flags &^= syscall.MS_NOUSER
		case "nouser":
			flags |= syscall.MS_NOUSER
		default:
			// assume filesystem-specific option
			data = append(data, s)
		}
	}
	err = syscall.Mount(entry.devpath, entry.mntpt, entry.fstype, uintptr(flags), strings.Join(data, ","))
	if err != nil {
		log.Fatal(err)
	}
}

func printMounts() error {
	f, err := os.Open("/proc/mounts")
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err := io.Copy(os.Stdout, f); err != nil {
		return err
	}
	return nil
}

func readEntry(arg string) (*fsEntry, error) {
	f, err := os.Open(*fstab)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var e fsEntry
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "#") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) < 4 {
			continue
		}
		if arg != fields[0] && arg != fields[1] {
			continue
		}
		e.devpath = fields[0]
		e.mntpt = fields[1]
		e.fstype = fields[2]
		e.options = strings.Split(fields[3], ",")
		return &e, nil
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return nil, errors.New("no entry found")
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"
	"strings"
	"text/tabwriter"

	circleci "github.com/mtchavez/circlecigo"
	"github.com/urfave/cli"
)

// EnvCmd - get env settings for a project
var EnvCmd = cli.Command{
	Name:        "env",
	Usage:       "get env settings for a project",
	Flags:       envFlags,
	Action:      envListAction,
	Subcommands: envSubcommands,
}

var envFlags = []cli.Flag{
	userFlag,
	projectFlag,
}

var envSubcommands = []cli.Command{
	{
		Name:   "list",
		Usage:  "all environment variables",
		Action: envListAction,
		Flags:  envFlags,
	},
	{
		Name:   "get",
		Usage:  "value of an environment variable",
		Action: envGetAction,
		Flags:  append(envFlags, envVarFlag),
	},
	{
		Name:   "set",
		Usage:  "a value for an environment variable",
		Action: envSetAction,
		Flags:  append(envFlags, envVarFlag),
	},
}

func envListAction(context *cli.Context) error {
	token := context.GlobalString("token")
	client := circleci.NewClient(token)
	padding := 1
	writer := tabwriter.NewWriter(os.Stdout, 0, 8, padding, '\t', tabwriter.AlignRight)

	envVars, _ := client.EnvVars(context.String("user"), context.String("project"))
	fmt.Fprintln(writer, "Var\tValue\t")
	for _, envVar := range envVars {
		fmt.Fprintln(writer, fmt.Sprintf("%s\t%s\t", envVar.Name, envVar.Value))
	}
	writer.Flush()
	return nil
}

func envGetAction(context *cli.Context) error {
	token := context.GlobalString("token")
	client := circleci.NewClient(token)
	padding := 1
	writer := tabwriter.NewWriter(os.Stdout, 0, 8, padding, '\t', tabwriter.AlignRight)

	envVar, _ := client.GetEnvVar(context.String("user"), context.String("project"), context.String("var"))
	fmt.Fprintln(writer, "Name\tValue\t")
	fmt.Fprintln(writer, fmt.Sprintf("%s\t%s\t", envVar.Name, envVar.Value))
	writer.Flush()
	return nil
}

func envSetAction(context *cli.Context) error {
	varInput := strings.Split(context.String("var"), "=")
	envName := ""
	envValue := ""
	if len(varInput) == 2 {
		envName = varInput[0]
		envValue = varInput[1]
	}
	token := context.GlobalString("token")
	client := circleci.NewClient(token)
	padding := 1
	writer := tabwriter.NewWriter(os.Stdout, 0, 8, padding, '\t', tabwriter.AlignRight)

	envVar, _ := client.AddEnvVar(context.String("user"), context.String("project"), envName, envValue)
	fmt.Fprintln(writer, "Name\tValue\t")
	fmt.Fprintln(writer, fmt.Sprintf("%s\t%s\t", envVar.Name, envVar.Value))
	writer.Flush()
	return nil
}
<commit_msg>Env delete command<commit_after>package commands

import (
	"fmt"
	"os"
	"strings"
	"text/tabwriter"

	circleci "github.com/mtchavez/circlecigo"
	"github.com/urfave/cli"
)

// EnvCmd - get env settings for a project
var EnvCmd = cli.Command{
	Name:        "env",
	Usage:       "get env settings for a project",
	Flags:       envFlags,
	Action:      envListAction,
	Subcommands: envSubcommands,
}

var envFlags = []cli.Flag{
	userFlag,
	projectFlag,
}

var envSubcommands = []cli.Command{
	{
		Name:    "list",
		Aliases: []string{"ls"},
		Usage:   "all environment variables",
		Action:  envListAction,
		Flags:   envFlags,
	},
	{
		Name:   "get",
		Usage:  "value of an environment variable",
		Action: envGetAction,
		Flags:  append(envFlags, envVarFlag),
	},
	{
		Name:   "set",
		Usage:  "a value for an environment variable",
		Action: envSetAction,
		Flags:  append(envFlags, envVarFlag),
	},
	{
		Name:    "delete",
		Aliases: []string{"del", "rm"},
		Usage:   "an environment variable",
		Action:  envDeleteAction,
		Flags:   append(envFlags, envVarFlag),
	},
}

func envListAction(context *cli.Context) error {
	token := context.GlobalString("token")
	client := circleci.NewClient(token)
	padding := 1
	writer := tabwriter.NewWriter(os.Stdout, 0, 8, padding, '\t', tabwriter.AlignRight)

	envVars, _ := client.EnvVars(context.String("user"), context.String("project"))
	fmt.Fprintln(writer, "Var\tValue\t")
	for _, envVar := range envVars {
		fmt.Fprintln(writer, fmt.Sprintf("%s\t%s\t", envVar.Name, envVar.Value))
	}
	writer.Flush()
	return nil
}

func envGetAction(context *cli.Context) error {
	token := context.GlobalString("token")
	client := circleci.NewClient(token)
	padding := 1
	writer := tabwriter.NewWriter(os.Stdout, 0, 8, padding, '\t', tabwriter.AlignRight)

	envVar, _ := client.GetEnvVar(context.String("user"), context.String("project"), context.String("var"))
	fmt.Fprintln(writer, "Name\tValue\t")
	fmt.Fprintln(writer, fmt.Sprintf("%s\t%s\t", envVar.Name, envVar.Value))
	writer.Flush()
	return nil
}

func envSetAction(context *cli.Context) error {
	varInput := strings.Split(context.String("var"), "=")
	envName := ""
	envValue := ""
	if len(varInput) == 2 {
		envName = varInput[0]
		envValue = varInput[1]
	}
	token := context.GlobalString("token")
	client := circleci.NewClient(token)
	padding := 1
	writer := tabwriter.NewWriter(os.Stdout, 0, 8, padding, '\t', tabwriter.AlignRight)

	envVar, _ := client.AddEnvVar(context.String("user"), context.String("project"), envName, envValue)
	fmt.Fprintln(writer, "Name\tValue\t")
	fmt.Fprintln(writer, fmt.Sprintf("%s\t%s\t", envVar.Name, envVar.Value))
	writer.Flush()
	return nil
}

func envDeleteAction(context *cli.Context) error {
	token := context.GlobalString("token")
	client := circleci.NewClient(token)
	varName := context.String("var")
	_, resp := client.DeleteEnvVar(context.String("user"), context.String("project"), varName)
	if resp.Success() {
		fmt.Printf("Removed %s\n", varName)
		return nil
	} else {
		fmt.Printf("Failed removing %s\n", varName)
		fmt.Println(string(resp.Body))
		return resp.Error
	}
}
<|endoftext|>


<commit_before>// Package mp3 implements audio data decoding in MP3 format.
package mp3

import (
	"fmt"
	"io"

	"github.com/faiface/beep"
	gomp3 "github.com/hajimehoshi/go-mp3"
	"github.com/pkg/errors"
)

const (
	gomp3NumChannels   = 2
	gomp3Precision     = 2
	gomp3BytesPerFrame = gomp3NumChannels * gomp3Precision
)

// Decode takes a ReadCloser containing audio data in MP3 format and returns a StreamSeekCloser,
// which streams that audio. The Seek method will panic if rc is not io.Seeker.
//
// Do not close the supplied ReadSeekCloser, instead, use the Close method of the returned
// StreamSeekCloser when you want to release the resources.
func Decode(rc io.ReadCloser) (s beep.StreamSeekCloser, format beep.Format, err error) {
	defer func() {
		if err != nil {
			err = errors.Wrap(err, "mp3")
		}
	}()
	d, err := gomp3.NewDecoder(rc)
	if err != nil {
		return nil, beep.Format{}, err
	}
	format = beep.Format{
		SampleRate:  beep.SampleRate(d.SampleRate()),
		NumChannels: gomp3NumChannels,
		Precision:   gomp3Precision,
	}
	return &decoder{d, format, 0, nil}, format, nil
}

type decoder struct {
	d   *gomp3.Decoder
	f   beep.Format
	pos int
	err error
}

func (d *decoder) Stream(samples [][2]float64) (n int, ok bool) {
	if d.err != nil {
		return 0, false
	}
	var tmp [gomp3BytesPerFrame]byte
	for i := range samples {
		dn, err := d.d.Read(tmp[:])
		if dn == len(tmp) {
			samples[i], _ = d.f.DecodeSigned(tmp[:])
			d.pos += dn
			n++
			ok = true
		}
		if err == io.EOF {
			break
		}
		if err != nil {
			d.err = errors.Wrap(err, "mp3")
			break
		}
	}
	return n, ok
}

func (d *decoder) Err() error {
	return d.err
}

func (d *decoder) Len() int {
	return int(d.d.Length()) / gomp3BytesPerFrame
}

func (d *decoder) Position() int {
	return d.pos / gomp3BytesPerFrame
}

func (d *decoder) Seek(p int) error {
	if p < 0 || d.Len() < p {
		return fmt.Errorf("mp3: seek position %v out of range [%v, %v]", p, 0, d.Len())
	}
	_, err := d.d.Seek(int64(p)*gomp3BytesPerFrame, io.SeekStart)
	if err != nil {
		return errors.Wrap(err, "mp3")
	}
	return nil
}

func (d *decoder) Close() error {
	err := d.d.Close()
	if err != nil {
		return errors.Wrap(err, "mp3")
	}
	return nil
}
<commit_msg>added d.pos. Issue #39<commit_after>// Package mp3 implements audio data decoding in MP3 format.
package mp3

import (
	"fmt"
	"io"

	"github.com/faiface/beep"
	gomp3 "github.com/hajimehoshi/go-mp3"
	"github.com/pkg/errors"
)

const (
	gomp3NumChannels   = 2
	gomp3Precision     = 2
	gomp3BytesPerFrame = gomp3NumChannels * gomp3Precision
)

// Decode takes a ReadCloser containing audio data in MP3 format and returns a StreamSeekCloser,
// which streams that audio. The Seek method will panic if rc is not io.Seeker.
//
// Do not close the supplied ReadSeekCloser, instead, use the Close method of the returned
// StreamSeekCloser when you want to release the resources.
func Decode(rc io.ReadCloser) (s beep.StreamSeekCloser, format beep.Format, err error) {
	defer func() {
		if err != nil {
			err = errors.Wrap(err, "mp3")
		}
	}()
	d, err := gomp3.NewDecoder(rc)
	if err != nil {
		return nil, beep.Format{}, err
	}
	format = beep.Format{
		SampleRate:  beep.SampleRate(d.SampleRate()),
		NumChannels: gomp3NumChannels,
		Precision:   gomp3Precision,
	}
	return &decoder{d, format, 0, nil}, format, nil
}

type decoder struct {
	d   *gomp3.Decoder
	f   beep.Format
	pos int
	err error
}

func (d *decoder) Stream(samples [][2]float64) (n int, ok bool) {
	if d.err != nil {
		return 0, false
	}
	var tmp [gomp3BytesPerFrame]byte
	for i := range samples {
		dn, err := d.d.Read(tmp[:])
		if dn == len(tmp) {
			samples[i], _ = d.f.DecodeSigned(tmp[:])
			d.pos += dn
			n++
			ok = true
		}
		if err == io.EOF {
			break
		}
		if err != nil {
			d.err = errors.Wrap(err, "mp3")
			break
		}
	}
	return n, ok
}

func (d *decoder) Err() error {
	return d.err
}

func (d *decoder) Len() int {
	return int(d.d.Length()) / gomp3BytesPerFrame
}

func (d *decoder) Position() int {
	return d.pos / gomp3BytesPerFrame
}

func (d *decoder) Seek(p int) error {
	if p < 0 || d.Len() < p {
		return fmt.Errorf("mp3: seek position %v out of range [%v, %v]", p, 0, d.Len())
	}
	_, err := d.d.Seek(int64(p)*gomp3BytesPerFrame, io.SeekStart)
	if err != nil {
		return errors.Wrap(err, "mp3")
	}
	d.pos = int64(p) * gomp3BytesPerFrame
	return nil
}

func (d *decoder) Close() error {
	err := d.d.Close()
	if err != nil {
		return errors.Wrap(err, "mp3")
	}
	return nil
}
<|endoftext|>


<commit_before>package checkerlution

import (
	"encoding/json"
	"github.com/couchbaselabs/logg"
	ng "github.com/tleyden/neurgo"
)

type Population struct {
	name        string
	rev         string `json:"_rev"`
	generations []Generation
}

type PopulationJson struct {
	Name        string
	Rev         string `json:"_rev"`
	Generations []Generation
}

func (population Population) MarshalJSON() ([]byte, error) {

	return json.Marshal(
		struct {
			Name        string
			Generations []Generation
		}{
			Name:        population.name,
			Generations: population.generations,
		})
}

func (population *Population) UnmarshalJSON(data []byte) error {

	logg.LogTo("CHECKERLUTION", "")
	populationJson := &PopulationJson{}
	error := json.Unmarshal(data, populationJson)
	if error == nil {
		population.name = populationJson.Name
		population.rev = populationJson.Rev
		population.generations = populationJson.Generations
	}
	return error

}

func (population Population) NextGenerationNumber() int {
	return len(population.generations)
}

func (population Population) CurrentGenerationNumber() int {
	return len(population.generations) - 1
}

func (population Population) CurrentGeneration() *Generation {
	genNumber := population.CurrentGenerationNumber()
	return &(population.generations[genNumber])
}

type Agent struct {
	cortex_id string
	parent_id string
}

func (agent Agent) MarshalJSON() ([]byte, error) {
	return json.Marshal(
		struct {
			Cortex_id string
			Parent_id string
		}{
			Cortex_id: agent.cortex_id,
			Parent_id: agent.parent_id,
		})
}

func NewAgent(cortex *ng.Cortex, parent_id string) *Agent {
	return &Agent{
		cortex_id: cortex.NodeId.UUID,
		parent_id: parent_id,
	}
}

type Generation struct {
	start_time string
	number     int
	state      string
	agents     []Agent
	games      []Game
}

func (generation *Generation) AddGame(game Game) {
	generation.games = append(generation.games, game)
}

func (generation Generation) MarshalJSON() ([]byte, error) {
	return json.Marshal(
		struct {
			Number int
			Agents []Agent
			Games  []Game
		}{
			Number: generation.number,
			Agents: generation.agents,
			Games:  generation.games,
		})
}

func NewGeneration(number int, agents []Agent) *Generation {
	games := []Game{}
	return &Generation{
		number: number,
		agents: agents,
		games:  games,
	}
}

func (population *Population) AddGeneration(generation Generation) {
	population.generations = append(population.generations, generation)
}

type Game struct {
	red_player_id  string
	blue_player_id string
	winner_id      string
}

func (game Game) MarshalJSON() ([]byte, error) {
	return json.Marshal(
		struct {
			Red_player_id  string
			Blue_player_id string
			Winner_id      string
		}{
			Red_player_id:  game.red_player_id,
			Blue_player_id: game.blue_player_id,
			Winner_id:      game.winner_id,
		})
}
<commit_msg>MarshalJson re-uses struct<commit_after>package checkerlution

import (
	"encoding/json"
	"github.com/couchbaselabs/logg"
	ng "github.com/tleyden/neurgo"
)

type Population struct {
	name        string
	rev         string `json:"_rev"`
	generations []Generation
}

type PopulationJson struct {
	Name        string
	Rev         string `json:"_rev"`
	Generations []Generation
}

func (population Population) MarshalJSON() ([]byte, error) {
	return json.Marshal(
		&PopulationJson{
			Name:        population.name,
			Generations: population.generations,
		})
}

func (population *Population) UnmarshalJSON(data []byte) error {

	logg.LogTo("CHECKERLUTION", "")
	populationJson := &PopulationJson{}
	error := json.Unmarshal(data, populationJson)
	if error == nil {
		population.name = populationJson.Name
		population.rev = populationJson.Rev
		population.generations = populationJson.Generations
	}
	return error

}

func (population Population) NextGenerationNumber() int {
	return len(population.generations)
}

func (population Population) CurrentGenerationNumber() int {
	return len(population.generations) - 1
}

func (population Population) CurrentGeneration() *Generation {
	genNumber := population.CurrentGenerationNumber()
	return &(population.generations[genNumber])
}

type Agent struct {
	cortex_id string
	parent_id string
}

func (agent Agent) MarshalJSON() ([]byte, error) {
	return json.Marshal(
		struct {
			Cortex_id string
			Parent_id string
		}{
			Cortex_id: agent.cortex_id,
			Parent_id: agent.parent_id,
		})
}

func NewAgent(cortex *ng.Cortex, parent_id string) *Agent {
	return &Agent{
		cortex_id: cortex.NodeId.UUID,
		parent_id: parent_id,
	}
}

type Generation struct {
	start_time string
	number     int
	state      string
	agents     []Agent
	games      []Game
}

func (generation *Generation) AddGame(game Game) {
	generation.games = append(generation.games, game)
}

func (generation Generation) MarshalJSON() ([]byte, error) {
	return json.Marshal(
		struct {
			Number int
			Agents []Agent
			Games  []Game
		}{
			Number: generation.number,
			Agents: generation.agents,
			Games:  generation.games,
		})
}

func NewGeneration(number int, agents []Agent) *Generation {
	games := []Game{}
	return &Generation{
		number: number,
		agents: agents,
		games:  games,
	}
}

func (population *Population) AddGeneration(generation Generation) {
	population.generations = append(population.generations, generation)
}

type Game struct {
	red_player_id  string
	blue_player_id string
	winner_id      string
}

func (game Game) MarshalJSON() ([]byte, error) {
	return json.Marshal(
		struct {
			Red_player_id  string
			Blue_player_id string
			Winner_id      string
		}{
			Red_player_id:  game.red_player_id,
			Blue_player_id: game.blue_player_id,
			Winner_id:      game.winner_id,
		})
}
<|endoftext|>


<commit_before>package iso

import (
	"errors"
	"fmt"
	"log"
	"path"
	"strings"
	"time"

	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/helper/communicator"
	hconfig "github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/template/interpolate"
	xsclient "github.com/xenserver/go-xenserver-client"
	xscommon "github.com/xenserver/packer-builder-xenserver/builder/xenserver/common"
)

type config struct {
	common.PackerConfig   `mapstructure:",squash"`
	xscommon.CommonConfig `mapstructure:",squash"`

	VCPUsMax        uint              `mapstructure:"vcpus_max"`
	VCPUsAtStartup  uint              `mapstructure:"vcpus_atstartup"`
	VMMemory        uint              `mapstructure:"vm_memory"`
	DiskSize        uint              `mapstructure:"disk_size"`
	CloneTemplate   string            `mapstructure:"clone_template"`
	VMOtherConfig   map[string]string `mapstructure:"vm_other_config"`

	ISOChecksum     string   `mapstructure:"iso_checksum"`
	ISOChecksumType string   `mapstructure:"iso_checksum_type"`
	ISOUrls         []string `mapstructure:"iso_urls"`
	ISOUrl          string   `mapstructure:"iso_url"`
	ISOName         string   `mapstructure:"iso_name"`

	PlatformArgs map[string]string `mapstructure:"platform_args"`

	RawInstallTimeout string        `mapstructure:"install_timeout"`
	InstallTimeout    time.Duration ``

	ctx interpolate.Context
}

type Builder struct {
	config config
	runner multistep.Runner
}

func (self *Builder) Prepare(raws ...interface{}) (params []string, retErr error) {

	var errs *packer.MultiError

	err := hconfig.Decode(&self.config, &hconfig.DecodeOpts{
		Interpolate: true,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{
				"boot_command",
			},
		},
	}, raws...)

	if err != nil {
		packer.MultiErrorAppend(errs, err)
	}

	errs = packer.MultiErrorAppend(
		errs, self.config.CommonConfig.Prepare(&self.config.ctx, &self.config.PackerConfig)...)
	errs = packer.MultiErrorAppend(errs, self.config.SSHConfig.Prepare(&self.config.ctx)...)

	// Set default values

	if self.config.RawInstallTimeout == "" {
		self.config.RawInstallTimeout = "200m"
	}

	if self.config.DiskSize == 0 {
		self.config.DiskSize = 40000
	}

	if self.config.VCPUsMax == 0 {
		self.config.VCPUsMax = 1
	}

	if self.config.VCPUsAtStartup == 0 {
		self.config.VCPUsAtStartup = 1
	}

	if self.config.VCPUsAtStartup > self.config.VCPUsMax {
	   	self.config.VCPUsAtStartup = self.config.VCPUsMax
	}

	if self.config.VMMemory == 0 {
		self.config.VMMemory = 1024
	}

	if self.config.CloneTemplate == "" {
		self.config.CloneTemplate = "Other install media"
	}

	if len(self.config.PlatformArgs) == 0 {
		pargs := make(map[string]string)
		pargs["viridian"] = "false"
		pargs["nx"] = "true"
		pargs["pae"] = "true"
		pargs["apic"] = "true"
		pargs["timeoffset"] = "0"
		pargs["acpi"] = "1"
		pargs["cores-per-socket"] = "1"
		self.config.PlatformArgs = pargs
	}

	// Template substitution

	templates := map[string]*string{
		"clone_template":    &self.config.CloneTemplate,
		"iso_checksum":      &self.config.ISOChecksum,
		"iso_checksum_type": &self.config.ISOChecksumType,
		"iso_url":           &self.config.ISOUrl,
		"iso_name":          &self.config.ISOName,
		"install_timeout":   &self.config.RawInstallTimeout,
	}
	for i := range self.config.ISOUrls {
		templates[fmt.Sprintf("iso_urls[%d]", i)] = &self.config.ISOUrls[i]
	}

	// Validation

	self.config.InstallTimeout, err = time.ParseDuration(self.config.RawInstallTimeout)
	if err != nil {
		errs = packer.MultiErrorAppend(
			errs, fmt.Errorf("Failed to parse install_timeout: %s", err))
	}

	if self.config.ISOName == "" {

		// If ISO name is not specified, assume a URL and checksum has been provided.

		if self.config.ISOChecksumType == "" {
			errs = packer.MultiErrorAppend(
				errs, errors.New("The iso_checksum_type must be specified."))
		} else {
			self.config.ISOChecksumType = strings.ToLower(self.config.ISOChecksumType)
			if self.config.ISOChecksumType != "none" {
				if self.config.ISOChecksum == "" {
					errs = packer.MultiErrorAppend(
						errs, errors.New("Due to the file size being large, an iso_checksum is required."))
				} else {
					self.config.ISOChecksum = strings.ToLower(self.config.ISOChecksum)
				}

				if hash := common.HashForType(self.config.ISOChecksumType); hash == nil {
					errs = packer.MultiErrorAppend(
						errs, fmt.Errorf("Unsupported checksum type: %s", self.config.ISOChecksumType))
				}

			}
		}

		if len(self.config.ISOUrls) == 0 {
			if self.config.ISOUrl == "" {
				errs = packer.MultiErrorAppend(
					errs, errors.New("One of iso_url or iso_urls must be specified."))
			} else {
				self.config.ISOUrls = []string{self.config.ISOUrl}
			}
		} else if self.config.ISOUrl != "" {
			errs = packer.MultiErrorAppend(
				errs, errors.New("Only one of iso_url or iso_urls may be specified."))
		}

		for i, url := range self.config.ISOUrls {
			self.config.ISOUrls[i], err = common.DownloadableURL(url)
			if err != nil {
				errs = packer.MultiErrorAppend(
					errs, fmt.Errorf("Failed to parse iso_urls[%d]: %s", i, err))
			}
		}
	} else {

		// An ISO name has been provided. It should be attached from an available SR.

	}

	if len(errs.Errors) > 0 {
		retErr = errors.New(errs.Error())
	}

	return nil, retErr

}

func (self *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
	//Setup XAPI client
	client := xsclient.NewXenAPIClient(self.config.HostIp, self.config.Username, self.config.Password)

	err := client.Login()
	if err != nil {
		return nil, err.(error)
	}
	ui.Say("XAPI client session established")

	client.GetHosts()

	//Share state between the other steps using a statebag
	state := new(multistep.BasicStateBag)
	state.Put("cache", cache)
	state.Put("client", client)
	state.Put("config", self.config)
	state.Put("commonconfig", self.config.CommonConfig)
	state.Put("hook", hook)
	state.Put("ui", ui)

	httpReqChan := make(chan string, 1)

	//Build the steps
	download_steps := []multistep.Step{
		&common.StepDownload{
			Checksum:     self.config.ISOChecksum,
			ChecksumType: self.config.ISOChecksumType,
			Description:  "ISO",
			ResultKey:    "iso_path",
			Url:          self.config.ISOUrls,
		},
	}

	steps := []multistep.Step{
		&xscommon.StepPrepareOutputDir{
			Force: self.config.PackerForce,
			Path:  self.config.OutputDir,
		},
		&common.StepCreateFloppy{
			Files: self.config.FloppyFiles,
		},
		&xscommon.StepHTTPServer{
			Chan: httpReqChan,
		},
		&xscommon.StepUploadVdi{
			VdiNameFunc: func() string {
				return "Packer-floppy-disk"
			},
			ImagePathFunc: func() string {
				if floppyPath, ok := state.GetOk("floppy_path"); ok {
					return floppyPath.(string)
				}
				return ""
			},
			VdiUuidKey: "floppy_vdi_uuid",
		},
		&xscommon.StepUploadVdi{
			VdiNameFunc: func() string {
				if len(self.config.ISOUrls) > 0 {
					return path.Base(self.config.ISOUrls[0])
				}
				return ""
			},
			ImagePathFunc: func() string {
				if isoPath, ok := state.GetOk("iso_path"); ok {
					return isoPath.(string)
				}
				return ""
			},
			VdiUuidKey: "iso_vdi_uuid",
		},
		&xscommon.StepFindVdi{
			VdiName:    self.config.ToolsIsoName,
			VdiUuidKey: "tools_vdi_uuid",
		},
		&xscommon.StepFindVdi{
			VdiName:    self.config.ISOName,
			VdiUuidKey: "isoname_vdi_uuid",
		},
		new(stepCreateInstance),
		&xscommon.StepAttachVdi{
			VdiUuidKey: "floppy_vdi_uuid",
			VdiType:    xsclient.Floppy,
		},
		&xscommon.StepAttachVdi{
			VdiUuidKey: "iso_vdi_uuid",
			VdiType:    xsclient.CD,
		},
		&xscommon.StepAttachVdi{
			VdiUuidKey: "isoname_vdi_uuid",
			VdiType:    xsclient.CD,
		},
		&xscommon.StepAttachVdi{
			VdiUuidKey: "tools_vdi_uuid",
			VdiType:    xsclient.CD,
		},
		new(xscommon.StepStartVmPaused),
		new(xscommon.StepSetVmHostSshAddress),
		new(xscommon.StepGetVNCPort),
		&xscommon.StepForwardPortOverSSH{
			RemotePort:  xscommon.InstanceVNCPort,
			RemoteDest:  xscommon.InstanceVNCIP,
			HostPortMin: self.config.HostPortMin,
			HostPortMax: self.config.HostPortMax,
			ResultKey:   "local_vnc_port",
		},
		new(xscommon.StepBootWait),
		&xscommon.StepTypeBootCommand{
			Ctx: self.config.ctx,
		},
		&xscommon.StepWaitForIP{
			Chan:    httpReqChan,
			Timeout: self.config.InstallTimeout, // @todo change this
		},
		&xscommon.StepForwardPortOverSSH{
			RemotePort:  xscommon.InstanceSSHPort,
			RemoteDest:  xscommon.InstanceSSHIP,
			HostPortMin: self.config.HostPortMin,
			HostPortMax: self.config.HostPortMax,
			ResultKey:   "local_ssh_port",
		},
		&communicator.StepConnect{
			Config:    &self.config.SSHConfig.Comm,
			Host:      xscommon.CommHost,
			SSHConfig: xscommon.SSHConfigFunc(self.config.CommonConfig.SSHConfig),
			SSHPort:   xscommon.SSHPort,
		},
		new(common.StepProvision),
		new(xscommon.StepShutdown),
		&xscommon.StepDetachVdi{
			VdiUuidKey: "iso_vdi_uuid",
		},
		&xscommon.StepDetachVdi{
			VdiUuidKey: "isoname_vdi_uuid",
		},
		&xscommon.StepDetachVdi{
			VdiUuidKey: "tools_vdi_uuid",
		},
		&xscommon.StepDetachVdi{
			VdiUuidKey: "floppy_vdi_uuid",
		},
		new(xscommon.StepExport),
	}

	if self.config.ISOName == "" {
		steps = append(download_steps, steps...)
	}

	self.runner = &multistep.BasicRunner{Steps: steps}
	self.runner.Run(state)

	if rawErr, ok := state.GetOk("error"); ok {
		return nil, rawErr.(error)
	}

	// If we were interrupted or cancelled, then just exit.
	if _, ok := state.GetOk(multistep.StateCancelled); ok {
		return nil, errors.New("Build was cancelled.")
	}
	if _, ok := state.GetOk(multistep.StateHalted); ok {
		return nil, errors.New("Build was halted.")
	}

	artifact, _ := xscommon.NewArtifact(self.config.OutputDir)

	return artifact, nil
}

func (self *Builder) Cancel() {
	if self.runner != nil {
		log.Println("Cancelling the step runner...")
		self.runner.Cancel()
	}
	fmt.Println("Cancelling the builder")
}
<commit_msg>Fix gofmt error.<commit_after>package iso

import (
	"errors"
	"fmt"
	"log"
	"path"
	"strings"
	"time"

	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/helper/communicator"
	hconfig "github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/template/interpolate"
	xsclient "github.com/xenserver/go-xenserver-client"
	xscommon "github.com/xenserver/packer-builder-xenserver/builder/xenserver/common"
)

type config struct {
	common.PackerConfig   `mapstructure:",squash"`
	xscommon.CommonConfig `mapstructure:",squash"`

	VCPUsMax       uint              `mapstructure:"vcpus_max"`
	VCPUsAtStartup uint              `mapstructure:"vcpus_atstartup"`
	VMMemory       uint              `mapstructure:"vm_memory"`
	DiskSize       uint              `mapstructure:"disk_size"`
	CloneTemplate  string            `mapstructure:"clone_template"`
	VMOtherConfig  map[string]string `mapstructure:"vm_other_config"`

	ISOChecksum     string   `mapstructure:"iso_checksum"`
	ISOChecksumType string   `mapstructure:"iso_checksum_type"`
	ISOUrls         []string `mapstructure:"iso_urls"`
	ISOUrl          string   `mapstructure:"iso_url"`
	ISOName         string   `mapstructure:"iso_name"`

	PlatformArgs map[string]string `mapstructure:"platform_args"`

	RawInstallTimeout string        `mapstructure:"install_timeout"`
	InstallTimeout    time.Duration ``

	ctx interpolate.Context
}

type Builder struct {
	config config
	runner multistep.Runner
}

func (self *Builder) Prepare(raws ...interface{}) (params []string, retErr error) {

	var errs *packer.MultiError

	err := hconfig.Decode(&self.config, &hconfig.DecodeOpts{
		Interpolate: true,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{
				"boot_command",
			},
		},
	}, raws...)

	if err != nil {
		packer.MultiErrorAppend(errs, err)
	}

	errs = packer.MultiErrorAppend(
		errs, self.config.CommonConfig.Prepare(&self.config.ctx, &self.config.PackerConfig)...)
	errs = packer.MultiErrorAppend(errs, self.config.SSHConfig.Prepare(&self.config.ctx)...)

	// Set default values

	if self.config.RawInstallTimeout == "" {
		self.config.RawInstallTimeout = "200m"
	}

	if self.config.DiskSize == 0 {
		self.config.DiskSize = 40000
	}

	if self.config.VCPUsMax == 0 {
		self.config.VCPUsMax = 1
	}

	if self.config.VCPUsAtStartup == 0 {
		self.config.VCPUsAtStartup = 1
	}

	if self.config.VCPUsAtStartup > self.config.VCPUsMax {
		self.config.VCPUsAtStartup = self.config.VCPUsMax
	}

	if self.config.VMMemory == 0 {
		self.config.VMMemory = 1024
	}

	if self.config.CloneTemplate == "" {
		self.config.CloneTemplate = "Other install media"
	}

	if len(self.config.PlatformArgs) == 0 {
		pargs := make(map[string]string)
		pargs["viridian"] = "false"
		pargs["nx"] = "true"
		pargs["pae"] = "true"
		pargs["apic"] = "true"
		pargs["timeoffset"] = "0"
		pargs["acpi"] = "1"
		pargs["cores-per-socket"] = "1"
		self.config.PlatformArgs = pargs
	}

	// Template substitution

	templates := map[string]*string{
		"clone_template":    &self.config.CloneTemplate,
		"iso_checksum":      &self.config.ISOChecksum,
		"iso_checksum_type": &self.config.ISOChecksumType,
		"iso_url":           &self.config.ISOUrl,
		"iso_name":          &self.config.ISOName,
		"install_timeout":   &self.config.RawInstallTimeout,
	}
	for i := range self.config.ISOUrls {
		templates[fmt.Sprintf("iso_urls[%d]", i)] = &self.config.ISOUrls[i]
	}

	// Validation

	self.config.InstallTimeout, err = time.ParseDuration(self.config.RawInstallTimeout)
	if err != nil {
		errs = packer.MultiErrorAppend(
			errs, fmt.Errorf("Failed to parse install_timeout: %s", err))
	}

	if self.config.ISOName == "" {

		// If ISO name is not specified, assume a URL and checksum has been provided.

		if self.config.ISOChecksumType == "" {
			errs = packer.MultiErrorAppend(
				errs, errors.New("The iso_checksum_type must be specified."))
		} else {
			self.config.ISOChecksumType = strings.ToLower(self.config.ISOChecksumType)
			if self.config.ISOChecksumType != "none" {
				if self.config.ISOChecksum == "" {
					errs = packer.MultiErrorAppend(
						errs, errors.New("Due to the file size being large, an iso_checksum is required."))
				} else {
					self.config.ISOChecksum = strings.ToLower(self.config.ISOChecksum)
				}

				if hash := common.HashForType(self.config.ISOChecksumType); hash == nil {
					errs = packer.MultiErrorAppend(
						errs, fmt.Errorf("Unsupported checksum type: %s", self.config.ISOChecksumType))
				}

			}
		}

		if len(self.config.ISOUrls) == 0 {
			if self.config.ISOUrl == "" {
				errs = packer.MultiErrorAppend(
					errs, errors.New("One of iso_url or iso_urls must be specified."))
			} else {
				self.config.ISOUrls = []string{self.config.ISOUrl}
			}
		} else if self.config.ISOUrl != "" {
			errs = packer.MultiErrorAppend(
				errs, errors.New("Only one of iso_url or iso_urls may be specified."))
		}

		for i, url := range self.config.ISOUrls {
			self.config.ISOUrls[i], err = common.DownloadableURL(url)
			if err != nil {
				errs = packer.MultiErrorAppend(
					errs, fmt.Errorf("Failed to parse iso_urls[%d]: %s", i, err))
			}
		}
	} else {

		// An ISO name has been provided. It should be attached from an available SR.

	}

	if len(errs.Errors) > 0 {
		retErr = errors.New(errs.Error())
	}

	return nil, retErr

}

func (self *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
	//Setup XAPI client
	client := xsclient.NewXenAPIClient(self.config.HostIp, self.config.Username, self.config.Password)

	err := client.Login()
	if err != nil {
		return nil, err.(error)
	}
	ui.Say("XAPI client session established")

	client.GetHosts()

	//Share state between the other steps using a statebag
	state := new(multistep.BasicStateBag)
	state.Put("cache", cache)
	state.Put("client", client)
	state.Put("config", self.config)
	state.Put("commonconfig", self.config.CommonConfig)
	state.Put("hook", hook)
	state.Put("ui", ui)

	httpReqChan := make(chan string, 1)

	//Build the steps
	download_steps := []multistep.Step{
		&common.StepDownload{
			Checksum:     self.config.ISOChecksum,
			ChecksumType: self.config.ISOChecksumType,
			Description:  "ISO",
			ResultKey:    "iso_path",
			Url:          self.config.ISOUrls,
		},
	}

	steps := []multistep.Step{
		&xscommon.StepPrepareOutputDir{
			Force: self.config.PackerForce,
			Path:  self.config.OutputDir,
		},
		&common.StepCreateFloppy{
			Files: self.config.FloppyFiles,
		},
		&xscommon.StepHTTPServer{
			Chan: httpReqChan,
		},
		&xscommon.StepUploadVdi{
			VdiNameFunc: func() string {
				return "Packer-floppy-disk"
			},
			ImagePathFunc: func() string {
				if floppyPath, ok := state.GetOk("floppy_path"); ok {
					return floppyPath.(string)
				}
				return ""
			},
			VdiUuidKey: "floppy_vdi_uuid",
		},
		&xscommon.StepUploadVdi{
			VdiNameFunc: func() string {
				if len(self.config.ISOUrls) > 0 {
					return path.Base(self.config.ISOUrls[0])
				}
				return ""
			},
			ImagePathFunc: func() string {
				if isoPath, ok := state.GetOk("iso_path"); ok {
					return isoPath.(string)
				}
				return ""
			},
			VdiUuidKey: "iso_vdi_uuid",
		},
		&xscommon.StepFindVdi{
			VdiName:    self.config.ToolsIsoName,
			VdiUuidKey: "tools_vdi_uuid",
		},
		&xscommon.StepFindVdi{
			VdiName:    self.config.ISOName,
			VdiUuidKey: "isoname_vdi_uuid",
		},
		new(stepCreateInstance),
		&xscommon.StepAttachVdi{
			VdiUuidKey: "floppy_vdi_uuid",
			VdiType:    xsclient.Floppy,
		},
		&xscommon.StepAttachVdi{
			VdiUuidKey: "iso_vdi_uuid",
			VdiType:    xsclient.CD,
		},
		&xscommon.StepAttachVdi{
			VdiUuidKey: "isoname_vdi_uuid",
			VdiType:    xsclient.CD,
		},
		&xscommon.StepAttachVdi{
			VdiUuidKey: "tools_vdi_uuid",
			VdiType:    xsclient.CD,
		},
		new(xscommon.StepStartVmPaused),
		new(xscommon.StepSetVmHostSshAddress),
		new(xscommon.StepGetVNCPort),
		&xscommon.StepForwardPortOverSSH{
			RemotePort:  xscommon.InstanceVNCPort,
			RemoteDest:  xscommon.InstanceVNCIP,
			HostPortMin: self.config.HostPortMin,
			HostPortMax: self.config.HostPortMax,
			ResultKey:   "local_vnc_port",
		},
		new(xscommon.StepBootWait),
		&xscommon.StepTypeBootCommand{
			Ctx: self.config.ctx,
		},
		&xscommon.StepWaitForIP{
			Chan:    httpReqChan,
			Timeout: self.config.InstallTimeout, // @todo change this
		},
		&xscommon.StepForwardPortOverSSH{
			RemotePort:  xscommon.InstanceSSHPort,
			RemoteDest:  xscommon.InstanceSSHIP,
			HostPortMin: self.config.HostPortMin,
			HostPortMax: self.config.HostPortMax,
			ResultKey:   "local_ssh_port",
		},
		&communicator.StepConnect{
			Config:    &self.config.SSHConfig.Comm,
			Host:      xscommon.CommHost,
			SSHConfig: xscommon.SSHConfigFunc(self.config.CommonConfig.SSHConfig),
			SSHPort:   xscommon.SSHPort,
		},
		new(common.StepProvision),
		new(xscommon.StepShutdown),
		&xscommon.StepDetachVdi{
			VdiUuidKey: "iso_vdi_uuid",
		},
		&xscommon.StepDetachVdi{
			VdiUuidKey: "isoname_vdi_uuid",
		},
		&xscommon.StepDetachVdi{
			VdiUuidKey: "tools_vdi_uuid",
		},
		&xscommon.StepDetachVdi{
			VdiUuidKey: "floppy_vdi_uuid",
		},
		new(xscommon.StepExport),
	}

	if self.config.ISOName == "" {
		steps = append(download_steps, steps...)
	}

	self.runner = &multistep.BasicRunner{Steps: steps}
	self.runner.Run(state)

	if rawErr, ok := state.GetOk("error"); ok {
		return nil, rawErr.(error)
	}

	// If we were interrupted or cancelled, then just exit.
	if _, ok := state.GetOk(multistep.StateCancelled); ok {
		return nil, errors.New("Build was cancelled.")
	}
	if _, ok := state.GetOk(multistep.StateHalted); ok {
		return nil, errors.New("Build was halted.")
	}

	artifact, _ := xscommon.NewArtifact(self.config.OutputDir)

	return artifact, nil
}

func (self *Builder) Cancel() {
	if self.runner != nil {
		log.Println("Cancelling the step runner...")
		self.runner.Cancel()
	}
	fmt.Println("Cancelling the builder")
}
<|endoftext|>


<commit_before>// Copyright 2012 Santiago Corredoira
// Distributed under a BSD-like license.
package email

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/smtp"
	"path/filepath"
	"strings"
)

type Attachment struct {
	Filename string
	Data     []byte
	Inline   bool
}

type Message struct {
	From            string
	To              []string
	Cc              []string
	Bcc             []string
	Subject         string
	Body            string
	BodyContentType string
	Attachments     map[string]*Attachment
}

func (m *Message) attach(file string, inline bool) error {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	_, filename := filepath.Split(file)

	m.Attachments[filename] = &Attachment{
		Filename: filename,
		Data:     data,
		Inline:   inline,
	}

	return nil
}

func (m *Message) Attach(file string) error {
	return m.attach(file, false)
}

func (m *Message) Inline(file string) error {
	return m.attach(file, true)
}

func newMessage(subject string, body string, bodyContentType string) *Message {
	m := &Message{Subject: subject, Body: body, BodyContentType: bodyContentType}

	m.Attachments = make(map[string]*Attachment)

	return m
}

// NewMessage returns a new Message that can compose an email with attachments
func NewMessage(subject string, body string) *Message {
	return newMessage(subject, body, "text/plain")
}

// NewMessage returns a new Message that can compose an HTML email with attachments
func NewHTMLMessage(subject string, body string) *Message {
	return newMessage(subject, body, "text/html")
}

// ToList returns all the recipients of the email
func (m *Message) Tolist() []string {
	tolist := m.To

	for _, cc := range m.Cc {
		tolist = append(tolist, cc)
	}

	for _, bcc := range m.Bcc {
		tolist = append(tolist, bcc)
	}

	return tolist
}

// Bytes returns the mail data
func (m *Message) Bytes() []byte {
	buf := bytes.NewBuffer(nil)

	buf.WriteString("From: " + m.From + "\n")
	buf.WriteString("To: " + strings.Join(m.To, ",") + "\n")
	if len(m.Cc) > 0 {
		buf.WriteString("Cc: " + strings.Join(m.Cc, ",") + "\n")
	}

	buf.WriteString("Subject: " + m.Subject + "\n")
	buf.WriteString("MIME-Version: 1.0\n")

	boundary := "f46d043c813270fc6b04c2d223da"

	if len(m.Attachments) > 0 {
		buf.WriteString("Content-Type: multipart/mixed; boundary=" + boundary + "\n")
		buf.WriteString("--" + boundary + "\n")
	}

	buf.WriteString(fmt.Sprintf("Content-Type: %s; charset=utf-8\n\n", m.BodyContentType))
	buf.WriteString(m.Body)
	buf.WriteString("\n")

	if len(m.Attachments) > 0 {
		for _, attachment := range m.Attachments {
			buf.WriteString("\n\n--" + boundary + "\n")

			if attachment.Inline {
				buf.WriteString("Content-Type: message/rfc822\n")
				buf.WriteString("Content-Disposition: inline; filename=\"" + attachment.Filename + "\"\n\n")

				buf.Write(attachment.Data)
			} else {
				buf.WriteString("Content-Type: application/octet-stream\n")
				buf.WriteString("Content-Transfer-Encoding: base64\n")
				buf.WriteString("Content-Disposition: attachment; filename=\"" + attachment.Filename + "\"\n\n")

				b := make([]byte, base64.StdEncoding.EncodedLen(len(attachment.Data)))
				base64.StdEncoding.Encode(b, attachment.Data)
				buf.Write(b)
			}

			buf.WriteString("\n--" + boundary)
		}

		buf.WriteString("--")
	}

	return buf.Bytes()
}

func Send(addr string, auth smtp.Auth, m *Message) error {
	return smtp.SendMail(addr, auth, m.From, m.Tolist(), m.Bytes())
}
<commit_msg>Add Date field to email header<commit_after>// Copyright 2012 Santiago Corredoira
// Distributed under a BSD-like license.
package email

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/smtp"
	"path/filepath"
	"strings"
	"time"
)

type Attachment struct {
	Filename string
	Data     []byte
	Inline   bool
}

type Message struct {
	From            string
	To              []string
	Cc              []string
	Bcc             []string
	Subject         string
	Body            string
	BodyContentType string
	Attachments     map[string]*Attachment
}

func (m *Message) attach(file string, inline bool) error {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	_, filename := filepath.Split(file)

	m.Attachments[filename] = &Attachment{
		Filename: filename,
		Data:     data,
		Inline:   inline,
	}

	return nil
}

func (m *Message) Attach(file string) error {
	return m.attach(file, false)
}

func (m *Message) Inline(file string) error {
	return m.attach(file, true)
}

func newMessage(subject string, body string, bodyContentType string) *Message {
	m := &Message{Subject: subject, Body: body, BodyContentType: bodyContentType}

	m.Attachments = make(map[string]*Attachment)

	return m
}

// NewMessage returns a new Message that can compose an email with attachments
func NewMessage(subject string, body string) *Message {
	return newMessage(subject, body, "text/plain")
}

// NewMessage returns a new Message that can compose an HTML email with attachments
func NewHTMLMessage(subject string, body string) *Message {
	return newMessage(subject, body, "text/html")
}

// ToList returns all the recipients of the email
func (m *Message) Tolist() []string {
	tolist := m.To

	for _, cc := range m.Cc {
		tolist = append(tolist, cc)
	}

	for _, bcc := range m.Bcc {
		tolist = append(tolist, bcc)
	}

	return tolist
}

// Bytes returns the mail data
func (m *Message) Bytes() []byte {
	buf := bytes.NewBuffer(nil)

	buf.WriteString("From: " + m.From + "\n")

	t := time.Now()
	buf.WriteString("Date: " + t.Format(time.RFC822) + "\n")

	buf.WriteString("To: " + strings.Join(m.To, ",") + "\n")
	if len(m.Cc) > 0 {
		buf.WriteString("Cc: " + strings.Join(m.Cc, ",") + "\n")
	}

	buf.WriteString("Subject: " + m.Subject + "\n")
	buf.WriteString("MIME-Version: 1.0\n")

	boundary := "f46d043c813270fc6b04c2d223da"

	if len(m.Attachments) > 0 {
		buf.WriteString("Content-Type: multipart/mixed; boundary=" + boundary + "\n")
		buf.WriteString("--" + boundary + "\n")
	}

	buf.WriteString(fmt.Sprintf("Content-Type: %s; charset=utf-8\n\n", m.BodyContentType))
	buf.WriteString(m.Body)
	buf.WriteString("\n")

	if len(m.Attachments) > 0 {
		for _, attachment := range m.Attachments {
			buf.WriteString("\n\n--" + boundary + "\n")

			if attachment.Inline {
				buf.WriteString("Content-Type: message/rfc822\n")
				buf.WriteString("Content-Disposition: inline; filename=\"" + attachment.Filename + "\"\n\n")

				buf.Write(attachment.Data)
			} else {
				buf.WriteString("Content-Type: application/octet-stream\n")
				buf.WriteString("Content-Transfer-Encoding: base64\n")
				buf.WriteString("Content-Disposition: attachment; filename=\"" + attachment.Filename + "\"\n\n")

				b := make([]byte, base64.StdEncoding.EncodedLen(len(attachment.Data)))
				base64.StdEncoding.Encode(b, attachment.Data)
				buf.Write(b)
			}

			buf.WriteString("\n--" + boundary)
		}

		buf.WriteString("--")
	}

	return buf.Bytes()
}

func Send(addr string, auth smtp.Auth, m *Message) error {
	return smtp.SendMail(addr, auth, m.From, m.Tolist(), m.Bytes())
}
<|endoftext|>