<commit_before>package tcdp

import (
	"context"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/operatorkit/controller/context/finalizerskeptcontext"

	"github.com/giantswarm/aws-operator/service/controller/clusterapi/v29/controllercontext"
	"github.com/giantswarm/aws-operator/service/controller/clusterapi/v29/key"
)

func (r *Resource) EnsureDeleted(ctx context.Context, obj interface{}) error {
	cr, err := key.ToCluster(obj)
	if err != nil {
		return microerror.Mask(err)
	}
	cc, err := controllercontext.FromContext(ctx)
	if err != nil {
		return microerror.Mask(err)
	}

	{
		r.logger.LogCtx(ctx, "level", "debug", "message", "disabling the termination protection of the tenant cluster's data plane cloud formation stack")

		i := &cloudformation.UpdateTerminationProtectionInput{
			EnableTerminationProtection: aws.Bool(false),
			StackName:                   aws.String(key.StackNameCPI(cr)),
		}

		_, err = cc.Client.TenantCluster.AWS.CloudFormation.UpdateTerminationProtection(i)
		if IsDeleteInProgress(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "the tenant cluster's data plane cloud formation stack is being deleted")

			r.logger.LogCtx(ctx, "level", "debug", "message", "keeping finalizers")
			finalizerskeptcontext.SetKept(ctx)

			r.logger.LogCtx(ctx, "level", "debug", "message", "canceling resource")

			return nil

		} else if IsNotExists(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "the tenant cluster's data plane cloud formation stack does not exist")
			r.logger.LogCtx(ctx, "level", "debug", "message", "canceling resource")

			return nil

		} else if err != nil {
			return microerror.Mask(err)
		}

		r.logger.LogCtx(ctx, "level", "debug", "message", "disabled the termination protection of the tenant cluster's data plane cloud formation stack")
	}

	{
		r.logger.LogCtx(ctx, "level", "debug", "message", "requesting the deletion of the tenant cluster's data plane cloud formation stack")

		i := &cloudformation.DeleteStackInput{
			StackName: aws.String(key.StackNameCPI(cr)),
		}

		_, err = cc.Client.TenantCluster.AWS.CloudFormation.DeleteStack(i)
		if IsUpdateInProgress(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "the tenant cluster's data plane cloud formation stack is being updated")

			r.logger.LogCtx(ctx, "level", "debug", "message", "keeping finalizers")
			finalizerskeptcontext.SetKept(ctx)

			r.logger.LogCtx(ctx, "level", "debug", "message", "canceling resource")

			return nil

		} else if err != nil {
			return microerror.Mask(err)
		}

		r.logger.LogCtx(ctx, "level", "debug", "message", "requested the deletion of the tenant cluster's data plane cloud formation stack")

		r.logger.LogCtx(ctx, "level", "debug", "message", "keeping finalizers")
		finalizerskeptcontext.SetKept(ctx)
	}

	return nil
}
<commit_msg>fix tcdp deletion (#1724)<commit_after>package tcdp

import (
	"context"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/operatorkit/controller/context/finalizerskeptcontext"

	"github.com/giantswarm/aws-operator/service/controller/clusterapi/v29/controllercontext"
	"github.com/giantswarm/aws-operator/service/controller/clusterapi/v29/key"
)

func (r *Resource) EnsureDeleted(ctx context.Context, obj interface{}) error {
	cr, err := key.ToMachineDeployment(obj)
	if err != nil {
		return microerror.Mask(err)
	}
	cc, err := controllercontext.FromContext(ctx)
	if err != nil {
		return microerror.Mask(err)
	}

	{
		r.logger.LogCtx(ctx, "level", "debug", "message", "disabling the termination protection of the tenant cluster's data plane cloud formation stack")

		i := &cloudformation.UpdateTerminationProtectionInput{
			EnableTerminationProtection: aws.Bool(false),
			StackName:                   aws.String(key.StackNameTCDP(&cr)),
		}

		_, err = cc.Client.TenantCluster.AWS.CloudFormation.UpdateTerminationProtection(i)
		if IsDeleteInProgress(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "the tenant cluster's data plane cloud formation stack is being deleted")

			r.logger.LogCtx(ctx, "level", "debug", "message", "keeping finalizers")
			finalizerskeptcontext.SetKept(ctx)

			r.logger.LogCtx(ctx, "level", "debug", "message", "canceling resource")

			return nil

		} else if IsNotExists(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "the tenant cluster's data plane cloud formation stack does not exist")
			r.logger.LogCtx(ctx, "level", "debug", "message", "canceling resource")

			return nil

		} else if err != nil {
			return microerror.Mask(err)
		}

		r.logger.LogCtx(ctx, "level", "debug", "message", "disabled the termination protection of the tenant cluster's data plane cloud formation stack")
	}

	{
		r.logger.LogCtx(ctx, "level", "debug", "message", "requesting the deletion of the tenant cluster's data plane cloud formation stack")

		i := &cloudformation.DeleteStackInput{
			StackName: aws.String(key.StackNameTCDP(&cr)),
		}

		_, err = cc.Client.TenantCluster.AWS.CloudFormation.DeleteStack(i)
		if IsUpdateInProgress(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "the tenant cluster's data plane cloud formation stack is being updated")

			r.logger.LogCtx(ctx, "level", "debug", "message", "keeping finalizers")
			finalizerskeptcontext.SetKept(ctx)

			r.logger.LogCtx(ctx, "level", "debug", "message", "canceling resource")

			return nil

		} else if err != nil {
			return microerror.Mask(err)
		}

		r.logger.LogCtx(ctx, "level", "debug", "message", "requested the deletion of the tenant cluster's data plane cloud formation stack")

		r.logger.LogCtx(ctx, "level", "debug", "message", "keeping finalizers")
		finalizerskeptcontext.SetKept(ctx)
	}

	return nil
}
<|endoftext|>


<commit_before>// +build !windows,!plan9

// TODO: Figure out what to do for Plan 9. It doesn't support FcntlFlock at all,
// but we might be able to ~emulate it with os.O_EXCL, but that wouldn't allow
// us to automatically release locks if a process dies.

package locking

import (
	"os"

	"golang.org/x/sys/unix"

	"github.com/pkg/errors"
)

// fcntlFlockRetryingOnEINTR is a wrapper around the fcntl system call that
// retries on EINTR errors and returns on the first successful call or non-EINTR
// error.
func fcntlFlockRetryingOnEINTR(file uintptr, command int, specification *unix.Flock_t) error {
	for {
		err := unix.FcntlFlock(file, command, specification)
		if err == unix.EINTR {
			continue
		}
		return err
	}
}

// Lock attempts to acquire the file lock.
func (l *Locker) Lock(block bool) error {
	// Verify that we don't already hold the lock.
	if l.held {
		return errors.New("lock already held")
	}

	// Set up the lock specification.
	lockSpec := unix.Flock_t{
		Type:   unix.F_WRLCK,
		Whence: int16(os.SEEK_SET),
		Start:  0,
		Len:    0,
	}

	// Set up the blocking specification.
	operation := unix.F_SETLK
	if block {
		operation = unix.F_SETLKW
	}

	// Attempt to perform locking, retrying if EINTR is encountered. According
	// to the POSIX standard, EINTR should only be expected in blocking cases
	// (i.e. when using F_SETLKW), but Linux allows it to be received when using
	// F_SETLK if it occurs before the lock is checked or acquired. Given that
	// Go's runtime preemption can also cause spurious interrupts, it's best to
	// handle EINTR in all cases.
	if err := fcntlFlockRetryingOnEINTR(l.file.Fd(), operation, &lockSpec); err != nil {
		return err
	}

	// Mark the lock as held.
	l.held = true

	// Success.
	return nil
}

// Unlock releases the file lock.
func (l *Locker) Unlock() error {
	// Verify that we hold the lock.
	if !l.held {
		return errors.New("lock not held")
	}

	// Set up the unlock specification.
	unlockSpec := unix.Flock_t{
		Type:   unix.F_UNLCK,
		Whence: int16(os.SEEK_SET),
		Start:  0,
		Len:    0,
	}

	// Attempt to perform unlocking, retrying if EINTR is encountered. According
	// to the POSIX standard, EINTR should only be expected in blocking cases
	// (i.e. when using F_SETLKW), but Linux allows it to be received in certain
	// cases when using F_SETLK. Given that Go's runtime preemption can also
	// cause spurious interrupts, it's best to handle EINTR in all cases.
	//
	// One thing worth considering here is the fact that neither POSIX nor Linux
	// makes any guarantees about the state of the lock if EINTR is received. In
	// theory, the lock could be unlocked, in which case continuing to retry
	// unlocking could lead to a race condition with other code that's locking
	// the same file (because fcntl locks are based on the underlying file and
	// not a particular file descriptor within a process). In fact, the Go
	// standard library and runtime do a similar thing with calls to close,
	// intentionally not handling EINTR from close due to a lack of information
	// about the state of the file descriptor and the desire to avoid a race
	// condition if that file descriptor is reused. In reality though, EINTR
	// here is exceedingly unlikely, and the usage of this code within Mutagen
	// is quite limited, with file unlocking only occurring right before a
	// process exits. So, for now, we'll just keep this consistent with the
	// locking case, but this EINTR behavior might be worth revisiting if it
	// becomes a problem.
	if err := fcntlFlockRetryingOnEINTR(l.file.Fd(), unix.F_SETLK, &unlockSpec); err != nil {
		return err
	}

	// Mark the lock as no longer being held.
	l.held = false

	// Success.
	return nil
}
<commit_msg>Avoid retry on EINTR in fcntl unlocking.<commit_after>// +build !windows,!plan9

// TODO: Figure out what to do for Plan 9. It doesn't support FcntlFlock at all,
// but we might be able to ~emulate it with os.O_EXCL, but that wouldn't allow
// us to automatically release locks if a process dies.

package locking

import (
	"os"

	"golang.org/x/sys/unix"

	"github.com/pkg/errors"
)

// fcntlFlockRetryingOnEINTR is a wrapper around the fcntl system call that
// retries on EINTR errors and returns on the first successful call or non-EINTR
// error.
func fcntlFlockRetryingOnEINTR(file uintptr, command int, specification *unix.Flock_t) error {
	for {
		err := unix.FcntlFlock(file, command, specification)
		if err == unix.EINTR {
			continue
		}
		return err
	}
}

// Lock attempts to acquire the file lock.
func (l *Locker) Lock(block bool) error {
	// Verify that we don't already hold the lock.
	if l.held {
		return errors.New("lock already held")
	}

	// Set up the lock specification.
	lockSpec := unix.Flock_t{
		Type:   unix.F_WRLCK,
		Whence: int16(os.SEEK_SET),
		Start:  0,
		Len:    0,
	}

	// Set up the blocking specification.
	operation := unix.F_SETLK
	if block {
		operation = unix.F_SETLKW
	}

	// Attempt to perform locking, retrying if EINTR is encountered. According
	// to the POSIX standard, EINTR should only be expected in blocking cases
	// (i.e. when using F_SETLKW), but Linux allows it to be received when using
	// F_SETLK if it occurs before the lock is checked or acquired. Given that
	// Go's runtime preemption can also cause spurious interrupts, it's best to
	// handle EINTR in all cases.
	if err := fcntlFlockRetryingOnEINTR(l.file.Fd(), operation, &lockSpec); err != nil {
		return err
	}

	// Mark the lock as held.
	l.held = true

	// Success.
	return nil
}

// Unlock releases the file lock.
func (l *Locker) Unlock() error {
	// Verify that we hold the lock.
	if !l.held {
		return errors.New("lock not held")
	}

	// Set up the unlock specification.
	unlockSpec := unix.Flock_t{
		Type:   unix.F_UNLCK,
		Whence: int16(os.SEEK_SET),
		Start:  0,
		Len:    0,
	}

	// Attempt to perform unlocking. Unlike the locking case, we don't retry if
	// EINTR is encountered because we don't have any information about the
	// state of the lock in that case (POSIX doesn't even allow for EINTR when
	// using F_SETLK and the Linux documentation only covers the locking case).
	// If the lock was successfully unlocked and we retry due to EINTR, then we
	// might end up in a race condition with other code trying to acquire the
	// lock. This is the same issue as with calls to close returning EINTR, in
	// which case the Go standard library and runtime (and Mutagen) don't retry
	// the operation because it's safer to err on the side of failure.
	//
	// In any case, this isn't ever going to be an issue for Mutagen in practice
	// because (a) EINTR is exceedingly unlikely here, (b) this code is only
	// used in Mutagen right before a process exits (usually in a defer that
	// ignores errors), and (c) Mutagen already has to be careful to avoid
	// multiple code paths managing locks on the same file (because POSIX
	// releases all fcntl locks for a file if any file descriptor for that lock
	// in the process is closed).
	if err := unix.FcntlFlock(l.file.Fd(), unix.F_SETLK, &unlockSpec); err != nil {
		return err
	}

	// Mark the lock as no longer being held.
	l.held = false

	// Success.
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package machine

import (
	"crypto"
	"fmt"
	"os"
	"path"
	"runtime"

	"github.com/golang/glog"
	"github.com/jimmidyson/go-download"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/bootstrapper"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/out"
)

// CacheBinariesForBootstrapper will cache binaries for a bootstrapper
func CacheBinariesForBootstrapper(version string, clusterBootstrapper string) error {
	binaries := bootstrapper.GetCachedBinaryList(clusterBootstrapper)

	var g errgroup.Group
	for _, bin := range binaries {
		bin := bin // https://golang.org/doc/faq#closures_and_goroutines
		g.Go(func() error {
			if _, err := CacheBinary(bin, version, "linux", runtime.GOARCH); err != nil {
				return errors.Wrapf(err, "caching binary %s", bin)
			}
			return nil
		})
	}
	return g.Wait()
}

// KubernetesReleaseURL gets the location of a kubernetes client
func KubernetesReleaseURL(binaryName, version, osName, archName string) string {
	return fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/%s/%s/%s", version, osName, archName, binaryName)
}

// KubernetesReleaseURLSHA1 gets the location of a kubernetes client checksum
func KubernetesReleaseURLSHA1(binaryName, version, osName, archName string) string {
	return fmt.Sprintf("%s.sha1", KubernetesReleaseURL(binaryName, version, osName, archName))
}

// CacheBinary will cache a binary on the host
func CacheBinary(binary, version, osName, archName string) (string, error) {
	targetDir := localpath.MakeMiniPath("cache", version)
	targetFilepath := path.Join(targetDir, binary)

	url := KubernetesReleaseURL(binary, version, osName, archName)

	_, err := os.Stat(targetFilepath)
	// If it exists, do no verification and continue
	if err == nil {
		glog.Infof("Not caching binary, using %s", url)
		return targetFilepath, nil
	}
	if !os.IsNotExist(err) {
		return "", errors.Wrapf(err, "stat %s version %s at %s", binary, version, targetDir)
	}

	if err = os.MkdirAll(targetDir, 0777); err != nil {
		return "", errors.Wrapf(err, "mkdir %s", targetDir)
	}

	options := download.FileOptions{
		Mkdirs: download.MkdirAll,
	}

	options.Checksum = KubernetesReleaseURLSHA1(binary, version, osName, archName)
	options.ChecksumHash = crypto.SHA1

	out.T(out.FileDownload, "Downloading {{.name}} {{.version}}", out.V{"name": binary, "version": version})
	if err := download.ToFile(url, targetFilepath, options); err != nil {
		return "", errors.Wrapf(err, "Error downloading %s %s", binary, version)
	}
	if osName == runtime.GOOS && archName == runtime.GOARCH {
		if err = os.Chmod(targetFilepath, 0755); err != nil {
			return "", errors.Wrapf(err, "chmod +x %s", targetFilepath)
		}
	}
	return targetFilepath, nil
}

// CopyBinary copies a locally cached binary to the guest VM
func CopyBinary(cr command.Runner, src string, dest string) error {
	f, err := assets.NewFileAsset(src, path.Dir(dest), path.Base(dest), "0755")
	if err != nil {
		return errors.Wrap(err, "new file asset")
	}
	if err := cr.Copy(f); err != nil {
		return errors.Wrapf(err, "copy")
	}
	return nil
}
<commit_msg>Use SHA256 for Kubernetes release checksums<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package machine

import (
	"crypto"
	"fmt"
	"os"
	"path"
	"runtime"

	"github.com/golang/glog"
	"github.com/jimmidyson/go-download"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/bootstrapper"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/out"
)

// CacheBinariesForBootstrapper will cache binaries for a bootstrapper
func CacheBinariesForBootstrapper(version string, clusterBootstrapper string) error {
	binaries := bootstrapper.GetCachedBinaryList(clusterBootstrapper)

	var g errgroup.Group
	for _, bin := range binaries {
		bin := bin // https://golang.org/doc/faq#closures_and_goroutines
		g.Go(func() error {
			if _, err := CacheBinary(bin, version, "linux", runtime.GOARCH); err != nil {
				return errors.Wrapf(err, "caching binary %s", bin)
			}
			return nil
		})
	}
	return g.Wait()
}

// releaseURL gets the location of a Kubernetes binary
func releaseURL(binaryName, version, osName, archName string) string {
	return fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/%s/%s/%s", version, osName, archName, binaryName)
}

// checksumURL gets the location of a kubernetes client checksum
func checksumURL(binaryName, version, osName, archName string) string {
	return fmt.Sprintf("%s.sha256", releaseURL(binaryName, version, osName, archName))
}

// CacheBinary will cache a binary on the host
func CacheBinary(binary, version, osName, archName string) (string, error) {
	targetDir := localpath.MakeMiniPath("cache", version)
	targetFilepath := path.Join(targetDir, binary)

	url := releaseURL(binary, version, osName, archName)

	_, err := os.Stat(targetFilepath)
	// If it exists, do no verification and continue
	if err == nil {
		glog.Infof("Not caching binary, using %s", url)
		return targetFilepath, nil
	}
	if !os.IsNotExist(err) {
		return "", errors.Wrapf(err, "stat %s version %s at %s", binary, version, targetDir)
	}

	if err = os.MkdirAll(targetDir, 0777); err != nil {
		return "", errors.Wrapf(err, "mkdir %s", targetDir)
	}

	options := download.FileOptions{
		Mkdirs: download.MkdirAll,
	}

	options.Checksum = checksumURL(binary, version, osName, archName)
	options.ChecksumHash = crypto.SHA256

	glog.Infof("Downloading %s: options: %+v", url, options)

	out.T(out.FileDownload, "Downloading {{.name}} {{.version}}", out.V{"name": binary, "version": version})
	if err := download.ToFile(url, targetFilepath, options); err != nil {
		return "", errors.Wrapf(err, url)
	}
	if osName == runtime.GOOS && archName == runtime.GOARCH {
		if err = os.Chmod(targetFilepath, 0755); err != nil {
			return "", errors.Wrapf(err, "chmod +x %s", targetFilepath)
		}
	}
	return targetFilepath, nil
}

// CopyBinary copies a locally cached binary to the guest VM
func CopyBinary(cr command.Runner, src string, dest string) error {
	f, err := assets.NewFileAsset(src, path.Dir(dest), path.Base(dest), "0755")
	if err != nil {
		return errors.Wrap(err, "new file asset")
	}
	if err := cr.Copy(f); err != nil {
		return errors.Wrapf(err, "copy")
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awsmodel

import (
	"fmt"
	"sort"
	"time"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/dns"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
	"k8s.io/kops/upup/pkg/fi/fitasks"
)

const LoadBalancerDefaultIdleTimeout = 5 * time.Minute

// APILoadBalancerBuilder builds a LoadBalancer for accessing the API
type APILoadBalancerBuilder struct {
	*AWSModelContext
	Lifecycle         *fi.Lifecycle
	SecurityLifecycle *fi.Lifecycle
}

var _ fi.ModelBuilder = &APILoadBalancerBuilder{}

func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
	// Configuration where an ELB fronts the API
	if !b.UseLoadBalancerForAPI() {
		return nil
	}

	lbSpec := b.Cluster.Spec.API.LoadBalancer
	if lbSpec == nil {
		// Skipping API ELB creation; not requested in Spec
		return nil
	}

	switch lbSpec.Type {
	case kops.LoadBalancerTypeInternal, kops.LoadBalancerTypePublic:
	// OK

	default:
		return fmt.Errorf("unhandled LoadBalancer type %q", lbSpec.Type)
	}

	// Compute the subnets - only one per zone, and then break ties based on chooseBestSubnetForELB
	var elbSubnets []*awstasks.Subnet
	{
		subnetsByZone := make(map[string][]*kops.ClusterSubnetSpec)
		for i := range b.Cluster.Spec.Subnets {
			subnet := &b.Cluster.Spec.Subnets[i]

			switch subnet.Type {
			case kops.SubnetTypePublic, kops.SubnetTypeUtility:
				if lbSpec.Type != kops.LoadBalancerTypePublic {
					continue
				}

			case kops.SubnetTypePrivate:
				if lbSpec.Type != kops.LoadBalancerTypeInternal {
					continue
				}

			default:
				return fmt.Errorf("subnet %q had unknown type %q", subnet.Name, subnet.Type)
			}

			subnetsByZone[subnet.Zone] = append(subnetsByZone[subnet.Zone], subnet)
		}

		for zone, subnets := range subnetsByZone {
			subnet := b.chooseBestSubnetForELB(zone, subnets)

			elbSubnets = append(elbSubnets, b.LinkToSubnet(subnet))
		}
	}

	var elb *awstasks.LoadBalancer
	{
		loadBalancerName := b.GetELBName32("api")

		idleTimeout := LoadBalancerDefaultIdleTimeout
		if lbSpec.IdleTimeoutSeconds != nil {
			idleTimeout = time.Second * time.Duration(*lbSpec.IdleTimeoutSeconds)
		}

		elb = &awstasks.LoadBalancer{
			Name:      s("api." + b.ClusterName()),
			Lifecycle: b.Lifecycle,

			LoadBalancerName: s(loadBalancerName),
			SecurityGroups: []*awstasks.SecurityGroup{
				b.LinkToELBSecurityGroup("api"),
			},
			Subnets: elbSubnets,
			Listeners: map[string]*awstasks.LoadBalancerListener{
				"443": {InstancePort: 443},
			},

			// Configure fast-recovery health-checks
			HealthCheck: &awstasks.LoadBalancerHealthCheck{
				Target:             s("SSL:443"),
				Timeout:            i64(5),
				Interval:           i64(10),
				HealthyThreshold:   i64(2),
				UnhealthyThreshold: i64(2),
			},

			ConnectionSettings: &awstasks.LoadBalancerConnectionSettings{
				IdleTimeout: i64(int64(idleTimeout.Seconds())),
			},
		}

		switch lbSpec.Type {
		case kops.LoadBalancerTypeInternal:
			elb.Scheme = s("internal")
		case kops.LoadBalancerTypePublic:
			elb.Scheme = nil
		default:
			return fmt.Errorf("unknown elb Type: %q", lbSpec.Type)
		}

		c.AddTask(elb)
	}

	// Create security group for API ELB
	{
		t := &awstasks.SecurityGroup{
			Name:      s(b.ELBSecurityGroupName("api")),
			Lifecycle: b.SecurityLifecycle,

			VPC:              b.LinkToVPC(),
			Description:      s("Security group for api ELB"),
			RemoveExtraRules: []string{"port=443"},
		}
		c.AddTask(t)
	}

	// Allow traffic from ELB to egress freely
	{
		t := &awstasks.SecurityGroupRule{
			Name:      s("api-elb-egress"),
			Lifecycle: b.SecurityLifecycle,

			SecurityGroup: b.LinkToELBSecurityGroup("api"),
			Egress:        fi.Bool(true),
			CIDR:          s("0.0.0.0/0"),
		}
		c.AddTask(t)
	}

	// Allow traffic into the ELB from KubernetesAPIAccess CIDRs
	{
		for _, cidr := range b.Cluster.Spec.KubernetesAPIAccess {
			t := &awstasks.SecurityGroupRule{
				Name:      s("https-api-elb-" + cidr),
				Lifecycle: b.SecurityLifecycle,

				SecurityGroup: b.LinkToELBSecurityGroup("api"),
				CIDR:          s(cidr),
				FromPort:      i64(443),
				ToPort:        i64(443),
				Protocol:      s("tcp"),
			}
			c.AddTask(t)
		}
	}

	// Add precreated additional security groups to the ELB
	{
		for _, id := range b.Cluster.Spec.API.LoadBalancer.AdditionalSecurityGroups {
			t := &awstasks.SecurityGroup{
				Name:   fi.String(id),
				ID:     fi.String(id),
				Shared: fi.Bool(true),
			}
			if err := c.EnsureTask(t); err != nil {
				return err
			}
			elb.SecurityGroups = append(elb.SecurityGroups, t)
		}
	}

	// Allow HTTPS to the master instances from the ELB
	{
		t := &awstasks.SecurityGroupRule{
			Name:      s("https-elb-to-master"),
			Lifecycle: b.SecurityLifecycle,

			SecurityGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleMaster),
			SourceGroup:   b.LinkToELBSecurityGroup("api"),
			FromPort:      i64(443),
			ToPort:        i64(443),
			Protocol:      s("tcp"),
		}
		c.AddTask(t)
	}

	if dns.IsGossipHostname(b.Cluster.Name) || b.UsePrivateDNS() {
		// Ensure the ELB hostname is included in the TLS certificate,
		// if we're not going to use an alias for it
		// TODO: I don't love this technique for finding the task by name & modifying it
		masterKeypairTask, found := c.Tasks["Keypair/master"]
		if !found {
			return fmt.Errorf("keypair/master task not found")
		}
		masterKeypair := masterKeypairTask.(*fitasks.Keypair)
		masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, elb)
	}

	for _, ig := range b.MasterInstanceGroups() {
		t := &awstasks.LoadBalancerAttachment{
			Name:      s("api-" + ig.ObjectMeta.Name),
			Lifecycle: b.Lifecycle,

			LoadBalancer:     b.LinkToELB("api"),
			AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
		}

		c.AddTask(t)
	}

	return nil

}

type scoredSubnet struct {
	score  int
	subnet *kops.ClusterSubnetSpec
}

type ByScoreDescending []*scoredSubnet

func (a ByScoreDescending) Len() int      { return len(a) }
func (a ByScoreDescending) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByScoreDescending) Less(i, j int) bool {
	if a[i].score != a[j].score {
		// ! to sort highest score first
		return !(a[i].score < a[j].score)
	}
	// Use name to break ties consistently
	return a[i].subnet.Name < a[j].subnet.Name
}

// Choose between subnets in a zone.
// We have already applied the rules to match internal subnets to internal ELBs and vice-versa for public-facing ELBs.
// For internal ELBs: we prefer the master subnets
// For public facing ELBs: we prefer the utility subnets
func (b *APILoadBalancerBuilder) chooseBestSubnetForELB(zone string, subnets []*kops.ClusterSubnetSpec) *kops.ClusterSubnetSpec {
	if len(subnets) == 0 {
		return nil
	}
	if len(subnets) == 1 {
		return subnets[0]
	}

	migSubnets := sets.NewString()
	for _, ig := range b.MasterInstanceGroups() {
		for _, subnet := range ig.Spec.Subnets {
			migSubnets.Insert(subnet)
		}
	}

	var scoredSubnets []*scoredSubnet
	for _, subnet := range subnets {
		score := 0

		if migSubnets.Has(subnet.Name) {
			score += 1
		}

		if subnet.Type == kops.SubnetTypeUtility {
			score += 1
		}

		scoredSubnets = append(scoredSubnets, &scoredSubnet{
			score:  score,
			subnet: subnet,
		})
	}

	sort.Sort(ByScoreDescending(scoredSubnets))

	if scoredSubnets[0].score == scoredSubnets[1].score {
		glog.V(2).Infof("Making arbitrary choice between subnets in zone %q to attach to ELB (%q vs %q)", zone, scoredSubnets[0].subnet.Name, scoredSubnets[1].subnet.Name)
	}

	return scoredSubnets[0].subnet
}
<commit_msg>adding missed lifecycles in elb code<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awsmodel

import (
	"fmt"
	"sort"
	"time"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/dns"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
	"k8s.io/kops/upup/pkg/fi/fitasks"
)

const LoadBalancerDefaultIdleTimeout = 5 * time.Minute

// APILoadBalancerBuilder builds a LoadBalancer for accessing the API
type APILoadBalancerBuilder struct {
	*AWSModelContext
	Lifecycle         *fi.Lifecycle
	SecurityLifecycle *fi.Lifecycle
}

var _ fi.ModelBuilder = &APILoadBalancerBuilder{}

func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
	// Configuration where an ELB fronts the API
	if !b.UseLoadBalancerForAPI() {
		return nil
	}

	lbSpec := b.Cluster.Spec.API.LoadBalancer
	if lbSpec == nil {
		// Skipping API ELB creation; not requested in Spec
		return nil
	}

	switch lbSpec.Type {
	case kops.LoadBalancerTypeInternal, kops.LoadBalancerTypePublic:
	// OK

	default:
		return fmt.Errorf("unhandled LoadBalancer type %q", lbSpec.Type)
	}

	// Compute the subnets - only one per zone, and then break ties based on chooseBestSubnetForELB
	var elbSubnets []*awstasks.Subnet
	{
		subnetsByZone := make(map[string][]*kops.ClusterSubnetSpec)
		for i := range b.Cluster.Spec.Subnets {
			subnet := &b.Cluster.Spec.Subnets[i]

			switch subnet.Type {
			case kops.SubnetTypePublic, kops.SubnetTypeUtility:
				if lbSpec.Type != kops.LoadBalancerTypePublic {
					continue
				}

			case kops.SubnetTypePrivate:
				if lbSpec.Type != kops.LoadBalancerTypeInternal {
					continue
				}

			default:
				return fmt.Errorf("subnet %q had unknown type %q", subnet.Name, subnet.Type)
			}

			subnetsByZone[subnet.Zone] = append(subnetsByZone[subnet.Zone], subnet)
		}

		for zone, subnets := range subnetsByZone {
			subnet := b.chooseBestSubnetForELB(zone, subnets)

			elbSubnets = append(elbSubnets, b.LinkToSubnet(subnet))
		}
	}

	var elb *awstasks.LoadBalancer
	{
		loadBalancerName := b.GetELBName32("api")

		idleTimeout := LoadBalancerDefaultIdleTimeout
		if lbSpec.IdleTimeoutSeconds != nil {
			idleTimeout = time.Second * time.Duration(*lbSpec.IdleTimeoutSeconds)
		}

		elb = &awstasks.LoadBalancer{
			Name:      s("api." + b.ClusterName()),
			Lifecycle: b.Lifecycle,

			LoadBalancerName: s(loadBalancerName),
			SecurityGroups: []*awstasks.SecurityGroup{
				b.LinkToELBSecurityGroup("api"),
			},
			Subnets: elbSubnets,
			Listeners: map[string]*awstasks.LoadBalancerListener{
				"443": {InstancePort: 443},
			},

			// Configure fast-recovery health-checks
			HealthCheck: &awstasks.LoadBalancerHealthCheck{
				Target:             s("SSL:443"),
				Timeout:            i64(5),
				Interval:           i64(10),
				HealthyThreshold:   i64(2),
				UnhealthyThreshold: i64(2),
			},

			ConnectionSettings: &awstasks.LoadBalancerConnectionSettings{
				IdleTimeout: i64(int64(idleTimeout.Seconds())),
			},
		}

		switch lbSpec.Type {
		case kops.LoadBalancerTypeInternal:
			elb.Scheme = s("internal")
		case kops.LoadBalancerTypePublic:
			elb.Scheme = nil
		default:
			return fmt.Errorf("unknown elb Type: %q", lbSpec.Type)
		}

		c.AddTask(elb)
	}

	// Create security group for API ELB
	{
		t := &awstasks.SecurityGroup{
			Name:      s(b.ELBSecurityGroupName("api")),
			Lifecycle: b.SecurityLifecycle,

			VPC:              b.LinkToVPC(),
			Description:      s("Security group for api ELB"),
			RemoveExtraRules: []string{"port=443"},
		}
		c.AddTask(t)
	}

	// Allow traffic from ELB to egress freely
	{
		t := &awstasks.SecurityGroupRule{
			Name:      s("api-elb-egress"),
			Lifecycle: b.SecurityLifecycle,

			SecurityGroup: b.LinkToELBSecurityGroup("api"),
			Egress:        fi.Bool(true),
			CIDR:          s("0.0.0.0/0"),
		}
		c.AddTask(t)
	}

	// Allow traffic into the ELB from KubernetesAPIAccess CIDRs
	{
		for _, cidr := range b.Cluster.Spec.KubernetesAPIAccess {
			t := &awstasks.SecurityGroupRule{
				Name:      s("https-api-elb-" + cidr),
				Lifecycle: b.SecurityLifecycle,

				SecurityGroup: b.LinkToELBSecurityGroup("api"),
				CIDR:          s(cidr),
				FromPort:      i64(443),
				ToPort:        i64(443),
				Protocol:      s("tcp"),
			}
			c.AddTask(t)
		}
	}

	// Add precreated additional security groups to the ELB
	{
		for _, id := range b.Cluster.Spec.API.LoadBalancer.AdditionalSecurityGroups {
			t := &awstasks.SecurityGroup{
				Name:   fi.String(id),
				ID:     fi.String(id),
				Shared: fi.Bool(true),

				Lifecycle: b.SecurityLifecycle,
			}
			if err := c.EnsureTask(t); err != nil {
				return err
			}
			elb.SecurityGroups = append(elb.SecurityGroups, t)
		}
	}

	// Allow HTTPS to the master instances from the ELB
	{
		t := &awstasks.SecurityGroupRule{
			Name:      s("https-elb-to-master"),
			Lifecycle: b.SecurityLifecycle,

			SecurityGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleMaster),
			SourceGroup:   b.LinkToELBSecurityGroup("api"),
			FromPort:      i64(443),
			ToPort:        i64(443),
			Protocol:      s("tcp"),
		}
		c.AddTask(t)
	}

	if dns.IsGossipHostname(b.Cluster.Name) || b.UsePrivateDNS() {
		// Ensure the ELB hostname is included in the TLS certificate,
		// if we're not going to use an alias for it
		// TODO: I don't love this technique for finding the task by name & modifying it
		masterKeypairTask, found := c.Tasks["Keypair/master"]
		if !found {
			return fmt.Errorf("keypair/master task not found")
		}
		masterKeypair := masterKeypairTask.(*fitasks.Keypair)
		masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, elb)
	}

	for _, ig := range b.MasterInstanceGroups() {
		t := &awstasks.LoadBalancerAttachment{
			Name:      s("api-" + ig.ObjectMeta.Name),
			Lifecycle: b.Lifecycle,

			LoadBalancer:     b.LinkToELB("api"),
			AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
		}

		c.AddTask(t)
	}

	return nil

}

type scoredSubnet struct {
	score  int
	subnet *kops.ClusterSubnetSpec
}

type ByScoreDescending []*scoredSubnet

func (a ByScoreDescending) Len() int      { return len(a) }
func (a ByScoreDescending) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByScoreDescending) Less(i, j int) bool {
	if a[i].score != a[j].score {
		// ! to sort highest score first
		return !(a[i].score < a[j].score)
	}
	// Use name to break ties consistently
	return a[i].subnet.Name < a[j].subnet.Name
}

// Choose between subnets in a zone.
// We have already applied the rules to match internal subnets to internal ELBs and vice-versa for public-facing ELBs.
// For internal ELBs: we prefer the master subnets
// For public facing ELBs: we prefer the utility subnets
func (b *APILoadBalancerBuilder) chooseBestSubnetForELB(zone string, subnets []*kops.ClusterSubnetSpec) *kops.ClusterSubnetSpec {
	if len(subnets) == 0 {
		return nil
	}
	if len(subnets) == 1 {
		return subnets[0]
	}

	migSubnets := sets.NewString()
	for _, ig := range b.MasterInstanceGroups() {
		for _, subnet := range ig.Spec.Subnets {
			migSubnets.Insert(subnet)
		}
	}

	var scoredSubnets []*scoredSubnet
	for _, subnet := range subnets {
		score := 0

		if migSubnets.Has(subnet.Name) {
			score += 1
		}

		if subnet.Type == kops.SubnetTypeUtility {
			score += 1
		}

		scoredSubnets = append(scoredSubnets, &scoredSubnet{
			score:  score,
			subnet: subnet,
		})
	}

	sort.Sort(ByScoreDescending(scoredSubnets))

	if scoredSubnets[0].score == scoredSubnets[1].score {
		glog.V(2).Infof("Making arbitrary choice between subnets in zone %q to attach to ELB (%q vs %q)", zone, scoredSubnets[0].subnet.Name, scoredSubnets[1].subnet.Name)
	}

	return scoredSubnets[0].subnet
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"fmt"

	"github.com/GoogleContainerTools/skaffold/proto"
)

type Status struct {
	ae       *proto.ActionableErr
	changed  bool
	reported bool
}

func (rs Status) Error() error {
	return fmt.Errorf(rs.ae.Message)
}

func (rs Status) ActionableError() *proto.ActionableErr {
	return rs.ae
}

func (rs Status) String() string {
	return rs.ae.Message
}

func (rs Status) Equal(other Status) bool {
	if rs.ae != nil && other.ae != nil {
		return rs.ae.Message == other.ae.Message && rs.ae.ErrCode == other.ae.ErrCode
	}
	return rs.ae == other.ae
}

func newStatus(ae *proto.ActionableErr) Status {
	return Status{
		ae:      ae,
		changed: true,
	}
}
<commit_msg>fix error status detection<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"fmt"

	"github.com/GoogleContainerTools/skaffold/proto"
)

type Status struct {
	ae       *proto.ActionableErr
	changed  bool
	reported bool
}

func (rs Status) Error() error {
	if rs.ae.ErrCode == proto.StatusCode_STATUSCHECK_SUCCESS {
		return nil
	}
	return fmt.Errorf(rs.ae.Message)
}

func (rs Status) ActionableError() *proto.ActionableErr {
	return rs.ae
}

func (rs Status) String() string {
	return rs.ae.Message
}

func (rs Status) Equal(other Status) bool {
	if rs.ae != nil && other.ae != nil {
		return rs.ae.Message == other.ae.Message && rs.ae.ErrCode == other.ae.ErrCode
	}
	return rs.ae == other.ae
}

func newStatus(ae *proto.ActionableErr) Status {
	return Status{
		ae:      ae,
		changed: true,
	}
}
<|endoftext|>


<commit_before>package users

/** Package users provides an API for managing user information.

Copyright 2015 - Joseph Lewis <joseph@josephlewis.net>
                 Daniel Kumor <rdkumor@gmail.com>

All Rights Reserved
**/
<commit_msg>Removed the unused postgres file in users<commit_after><|endoftext|>


<commit_before>package udp

import (
	"net"

	"github.com/CotaPreco/Horus/receiver"
	"github.com/CotaPreco/Horus/util"
)

type UdpReceiver struct {
	util.Observable
	host            string
	port            int
	receiveStrategy receiver.ReceiveStrategy
}

func NewUdpReceiver(
	host string,
	port int,
	receiveStrategy receiver.ReceiveStrategy,
) *UdpReceiver {
	return &UdpReceiver{
		host:            host,
		port:            port,
		receiveStrategy: receiveStrategy,
	}
}

func (r *UdpReceiver) Receive() {
	conn, err := net.ListenUDP("udp4", &net.UDPAddr{
		IP:   net.ParseIP(r.host),
		Port: r.port,
	})

	util.Invariant(
		err == nil,
		"...unexpected error: `%s` (ListenUDP)",
		err,
	)

	for {
		message := make([]byte, 1024)

		_, _, err := conn.ReadFromUDP(message)

		// TODO: ...use aggregate, attempt to receive using `O(n)` strategies
		if err == nil && r.receiveStrategy.CanReceive(message) {
			r.NotifyAll(r.receiveStrategy.Receive(message))
		}
	}
}
<commit_msg>Removing ~TODO:~ (...make use of ReceiveStrategyChain)<commit_after>package udp

import (
	"net"

	"github.com/CotaPreco/Horus/receiver"
	"github.com/CotaPreco/Horus/util"
)

type UdpReceiver struct {
	util.Observable
	host            string
	port            int
	receiveStrategy receiver.ReceiveStrategy
}

func NewUdpReceiver(
	host string,
	port int,
	receiveStrategy receiver.ReceiveStrategy,
) *UdpReceiver {
	return &UdpReceiver{
		host:            host,
		port:            port,
		receiveStrategy: receiveStrategy,
	}
}

func (r *UdpReceiver) Receive() {
	conn, err := net.ListenUDP("udp4", &net.UDPAddr{
		IP:   net.ParseIP(r.host),
		Port: r.port,
	})

	util.Invariant(
		err == nil,
		"...unexpected error: `%s` (ListenUDP)",
		err,
	)

	for {
		message := make([]byte, 1024)

		_, _, err := conn.ReadFromUDP(message)

		if err == nil && r.receiveStrategy.CanReceive(message) {
			r.NotifyAll(r.receiveStrategy.Receive(message))
		}
	}
}
<|endoftext|>


<commit_before>package consul

import (
	"errors"
	"fmt"
	"net"
	"os"
	"strconv"
	"time"

	"github.com/eBay/fabio/_third_party/github.com/hashicorp/consul/api"
	"github.com/eBay/fabio/config"
)

func serviceRegistration(addr, name string, interval, timeout time.Duration) (*api.AgentServiceRegistration, error) {
	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}
	_, portstr, err := net.SplitHostPort(addr)
	if err != nil {
		return nil, err
	}
	port, err := strconv.Atoi(portstr)
	if err != nil {
		return nil, err
	}

	ip, err := config.LocalIP()
	if err != nil {
		return nil, err
	}
	if ip == nil {
		return nil, errors.New("no local ip")
	}

	serviceID := fmt.Sprintf("%s-%s-%d", name, hostname, port)

	checkURL := fmt.Sprintf("http://%s:%d/health", ip, port)
	if ip.To16() != nil {
		checkURL = fmt.Sprintf("http://[%s]:%d/health", ip, port)
	}

	service := &api.AgentServiceRegistration{
		ID:      serviceID,
		Name:    name,
		Address: ip.String(),
		Port:    port,
		Check: &api.AgentServiceCheck{
			HTTP:     checkURL,
			Interval: interval.String(),
			Timeout:  timeout.String(),
		},
	}

	return service, nil
}
<commit_msg>[registry] fallback to given local IP address<commit_after>package consul

import (
	"errors"
	"fmt"
	"net"
	"os"
	"strconv"
	"time"

	"github.com/eBay/fabio/_third_party/github.com/hashicorp/consul/api"
	"github.com/eBay/fabio/config"
)

func serviceRegistration(addr, name string, interval, timeout time.Duration) (*api.AgentServiceRegistration, error) {
	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}
	ipstr, portstr, err := net.SplitHostPort(addr)
	if err != nil {
		return nil, err
	}
	port, err := strconv.Atoi(portstr)
	if err != nil {
		return nil, err
	}

	ip, err := config.LocalIP()
	if err != nil {
		return nil, err
	}
	if ip == nil {
		givenip := net.ParseIP(ipstr)
                if givenip == nil {
                        return nil, errors.New("no local ip")
                }
                ip = givenip
	}

	serviceID := fmt.Sprintf("%s-%s-%d", name, hostname, port)

	checkURL := fmt.Sprintf("http://%s:%d/health", ip, port)
	if ip.To16() != nil {
		checkURL = fmt.Sprintf("http://[%s]:%d/health", ip, port)
	}

	service := &api.AgentServiceRegistration{
		ID:      serviceID,
		Name:    name,
		Address: ip.String(),
		Port:    port,
		Check: &api.AgentServiceCheck{
			HTTP:     checkURL,
			Interval: interval.String(),
			Timeout:  timeout.String(),
		},
	}

	return service, nil
}
<|endoftext|>


<commit_before>// GoEuler - An implementation of Project Euler with Go.
// (c) 2015, Lindsay Bradford

package main

import (
	"goeuler/cmdline"
	"goeuler/config"
	. "goeuler/problem"
	"log"
	"os"
	"sync"
)

var (
	args     = new(cmdline.Arguments)
	builder  = new(ProblemBuilder)
	basicLog = log.New(os.Stdout, "", 0)
	timedLog = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lmicroseconds)
)

func main() {
	args.Process()
	reportVersionAndProblemsImplemented()

	minProblemId, maxProblemId := resolveProblemRange()

	if args.Concurrent {
		answerConcurrently(minProblemId, maxProblemId)
	} else {
		answerSequentially(minProblemId, maxProblemId)
	}
}

func answerConcurrently(minProblemId uint, maxProblemId uint) {
	var wg sync.WaitGroup

	for problemId := minProblemId; problemId <= maxProblemId; problemId++ {
		problem := builder.
			OfProblem(problemId).
			WithLogger(timedLog).
			Build()

		wg.Add(1)
		go func() {
			defer wg.Done()
			answer(problem)
		}()
	}

	wg.Wait()
}

func answerSequentially(minProblemId uint, maxProblemId uint) {
	for problemId := minProblemId; problemId <= maxProblemId; problemId++ {
		problem := builder.
			OfProblem(problemId).
			WithLogger(timedLog).
			Build()

		answer(problem)
	}
}

func reportVersionAndProblemsImplemented() {
	basicLog.Print(cmdline.GetVersionString())
	basicLog.Print("Project Euler via Go, (c) 2015, Lindsay Bradford.")
	basicLog.Printf(
		"Problems Implemented: (%d/%d)",
		config.LastImplementedEulerProblemID,
		config.MaxEulerProblemID,
	)
	basicLog.Println("")
}

func resolveProblemRange() (nimProblemId uint, maxProblemId uint) {
	if args.AllProblems {
		return 1, config.LastImplementedEulerProblemID
	}
	return args.Problem, args.Problem
}

func answer(p Problem) {
	timedLog.Printf("Problem %d: %s", p.GetID(), p.GetTitle())

	p.CalculateAnswer()

	timedLog.Printf("Problem %d answer: %s", p.GetID(), p.GetAnswer())

	if p.IsAnswerVerified() {
		timedLog.Printf("Problem %d as verified as correct by Project Euler.", p.GetID())
	}
}
<commit_msg>Slight refacoring around problem building & immutable problem  id  on gorroutine spawn<commit_after>// GoEuler - An implementation of Project Euler with Go.
// (c) 2015, Lindsay Bradford

package main

import (
	"goeuler/cmdline"
	"goeuler/config"
	. "goeuler/problem"
	"log"
	"os"
	"sync"
)

var (
	args     = new(cmdline.Arguments)
	builder  = new(ProblemBuilder)
	basicLog = log.New(os.Stdout, "", 0)
	timedLog = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lmicroseconds)
)

func main() {
	reportVersionAndProblemsImplemented()

	args.Process()
	minProblemId, maxProblemId := resolveProblemRange()

	if args.Concurrent {
		answerConcurrently(minProblemId, maxProblemId)
	} else {
		answerSequentially(minProblemId, maxProblemId)
	}
}

func answerConcurrently(minProblemId uint, maxProblemId uint) {
	var wg sync.WaitGroup

	for problemId := minProblemId; problemId <= maxProblemId; problemId++ {
		wg.Add(1)

		go func(problem uint) {
			defer wg.Done()
			answer(buildProblem(problem))
		}(problemId)
	}

	wg.Wait()
}

func answerSequentially(minProblemId uint, maxProblemId uint) {
	for problemId := minProblemId; problemId <= maxProblemId; problemId++ {
		answer(buildProblem(problemId))
	}
}

func buildProblem(problem uint) Problem {
	return builder.
		OfProblem(problem).
		WithLogger(timedLog).
		Build()
}

func reportVersionAndProblemsImplemented() {
	basicLog.Print(cmdline.GetVersionString())
	basicLog.Print("Project Euler via Go, (c) 2015, Lindsay Bradford.")
	basicLog.Printf(
		"Problems Implemented: (%d/%d)",
		config.LastImplementedEulerProblemID,
		config.MaxEulerProblemID,
	)
	basicLog.Println("")
}

func resolveProblemRange() (nimProblemId uint, maxProblemId uint) {
	if args.AllProblems {
		return 1, config.LastImplementedEulerProblemID
	}
	return args.Problem, args.Problem
}

func answer(p Problem) {
	timedLog.Printf("Problem %d: %s", p.GetID(), p.GetTitle())

	p.CalculateAnswer()

	timedLog.Printf("Problem %d answer: %s", p.GetID(), p.GetAnswer())

	if p.IsAnswerVerified() {
		timedLog.Printf("Problem %d as verified as correct by Project Euler.", p.GetID())
	}
}
<|endoftext|>


<commit_before>package diego

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
)

var _ = Describe("Application Lifecycle", func() {
	var appName string

	apps := func() *Session {
		return cf.Cf("apps").Wait()
	}

	BeforeEach(func() {
		appName = generator.RandomName()
	})

	AfterEach(func() {
		Eventually(cf.Cf("logs", appName, "--recent")).Should(Exit())
		Eventually(cf.Cf("delete", appName, "-f")).Should(Exit(0))
	})

	Describe("An app staged on Diego and running on Diego", func() {
		It("exercises the app through its lifecycle", func() {
			By("pushing it", func() {
				Eventually(cf.Cf("push", appName, "-p", "../assets/nora/NoraPublished", "--no-start", "-b", "java_buildpack", "-s", "windows2012R2"), CF_PUSH_TIMEOUT).Should(Exit(0))
			})

			By("staging and running it on Diego", func() {
				enableDiego(appName)
				session := cf.Cf("start", appName)
				Eventually(session, CF_PUSH_TIMEOUT).Should(Exit(0))
			})

			// FIXME: Something about stdout logging

			By("verifying it's up", func() {
				Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("hello i am nora"))
			})

			By("stopping it", func() {
				Eventually(cf.Cf("stop", appName)).Should(Exit(0))
				Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("404"))
			})

			// By("setting an environment variable", func() {
			// 	Eventually(cf.Cf("set-env", appName, "FOO", "bar")).Should(Exit(0))
			// })

			By("starting it", func() {
				Eventually(cf.Cf("start", appName), CF_PUSH_TIMEOUT).Should(Exit(0))
				Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("hello i am nora"))
			})

			// By("checking custom env variables are available", func() {
			// 	Ω(helpers.CurlApp(appName, "/env/FOO")).Should(ContainSubstring("bar"))
			// })

			By("scaling it", func() {
				Eventually(cf.Cf("scale", appName, "-i", "2")).Should(Exit(0))
				Eventually(apps).Should(Say("2/2"))
			})

			// By("restarting an instance", func() {
			// 	idsBefore := reportedIDs(2)
			// 	Eventually(cf.Cf("restart-app-instance", appName, "1")).Should(Exit(0))
			// 	Eventually(func() []string {
			// 		return differentIDsFrom(idsBefore)
			// 	}).Should(HaveLen(1))
			// })
		})
	})
})
<commit_msg>Add tests for custom environment variables [#91642896]<commit_after>package diego

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
)

var _ = Describe("Application Lifecycle", func() {
	var appName string

	apps := func() *Session {
		return cf.Cf("apps").Wait()
	}

	BeforeEach(func() {
		appName = generator.RandomName()
	})

	AfterEach(func() {
		Eventually(cf.Cf("logs", appName, "--recent")).Should(Exit())
		Eventually(cf.Cf("delete", appName, "-f")).Should(Exit(0))
	})

	Describe("An app staged on Diego and running on Diego", func() {
		It("exercises the app through its lifecycle", func() {
			By("pushing it", func() {
				Eventually(cf.Cf("push", appName, "-p", "../assets/nora/NoraPublished", "--no-start", "-b", "java_buildpack", "-s", "windows2012R2"), CF_PUSH_TIMEOUT).Should(Exit(0))
			})

			By("staging and running it on Diego", func() {
				enableDiego(appName)
				session := cf.Cf("start", appName)
				Eventually(session, CF_PUSH_TIMEOUT).Should(Exit(0))
			})

			// FIXME: Something about stdout logging

			By("verifying it's up", func() {
				Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("hello i am nora"))
			})

			By("stopping it", func() {
				Eventually(cf.Cf("stop", appName)).Should(Exit(0))
				Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("404"))
			})

			By("setting an environment variable", func() {
				Eventually(cf.Cf("set-env", appName, "FOO", "bar")).Should(Exit(0))
			})

			By("starting it", func() {
				Eventually(cf.Cf("start", appName), CF_PUSH_TIMEOUT).Should(Exit(0))
				Eventually(helpers.CurlingAppRoot(appName)).Should(ContainSubstring("hello i am nora"))
			})

			By("checking custom env variables are available", func() {
				Ω(helpers.CurlApp(appName, "/env/FOO")).Should(ContainSubstring("bar"))
			})

			By("scaling it", func() {
				Eventually(cf.Cf("scale", appName, "-i", "2")).Should(Exit(0))
				Eventually(apps).Should(Say("2/2"))
			})

			// By("restarting an instance", func() {
			// 	idsBefore := reportedIDs(2)
			// 	Eventually(cf.Cf("restart-app-instance", appName, "1")).Should(Exit(0))
			// 	Eventually(func() []string {
			// 		return differentIDsFrom(idsBefore)
			// 	}).Should(HaveLen(1))
			// })
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gob

import (
	"bytes"
	"io"
	"os"
	"reflect"
	"sync"
)

// An Encoder manages the transmission of type and data information to the
// other side of a connection.
type Encoder struct {
	mutex      sync.Mutex              // each item must be sent atomically
	w          []io.Writer             // where to send the data
	sent       map[reflect.Type]typeId // which types we've already sent
	countState *encoderState           // stage for writing counts
	freeList   *encoderState           // list of free encoderStates; avoids reallocation
	buf        []byte                  // for collecting the output.
	err        os.Error
}

// NewEncoder returns a new encoder that will transmit on the io.Writer.
func NewEncoder(w io.Writer) *Encoder {
	enc := new(Encoder)
	enc.w = []io.Writer{w}
	enc.sent = make(map[reflect.Type]typeId)
	enc.countState = enc.newEncoderState(new(bytes.Buffer))
	return enc
}

// writer() returns the innermost writer the encoder is using
func (enc *Encoder) writer() io.Writer {
	return enc.w[len(enc.w)-1]
}

// pushWriter adds a writer to the encoder.
func (enc *Encoder) pushWriter(w io.Writer) {
	enc.w = append(enc.w, w)
}

// popWriter pops the innermost writer.
func (enc *Encoder) popWriter() {
	enc.w = enc.w[0 : len(enc.w)-1]
}

func (enc *Encoder) badType(rt reflect.Type) {
	enc.setError(os.ErrorString("gob: can't encode type " + rt.String()))
}

func (enc *Encoder) setError(err os.Error) {
	if enc.err == nil { // remember the first.
		enc.err = err
	}
}

// writeMessage sends the data item preceded by a unsigned count of its length.
func (enc *Encoder) writeMessage(w io.Writer, b *bytes.Buffer) {
	enc.countState.encodeUint(uint64(b.Len()))
	// Build the buffer.
	countLen := enc.countState.b.Len()
	total := countLen + b.Len()
	if total > len(enc.buf) {
		enc.buf = make([]byte, total+1000) // extra for growth
	}
	// Place the length before the data.
	// TODO(r): avoid the extra copy here.
	enc.countState.b.Read(enc.buf[0:countLen])
	// Now the data.
	b.Read(enc.buf[countLen:total])
	// Write the data.
	_, err := w.Write(enc.buf[0:total])
	if err != nil {
		enc.setError(err)
	}
}

// sendActualType sends the requested type, without further investigation, unless
// it's been sent before.
func (enc *Encoder) sendActualType(w io.Writer, state *encoderState, ut *userTypeInfo, actual reflect.Type) (sent bool) {
	if _, alreadySent := enc.sent[actual]; alreadySent {
		return false
	}
	typeLock.Lock()
	info, err := getTypeInfo(ut)
	typeLock.Unlock()
	if err != nil {
		enc.setError(err)
		return
	}
	// Send the pair (-id, type)
	// Id:
	state.encodeInt(-int64(info.id))
	// Type:
	enc.encode(state.b, reflect.NewValue(info.wire), wireTypeUserInfo)
	enc.writeMessage(w, state.b)
	if enc.err != nil {
		return
	}

	// Remember we've sent this type, both what the user gave us and the base type.
	enc.sent[ut.base] = info.id
	if ut.user != ut.base {
		enc.sent[ut.user] = info.id
	}
	// Now send the inner types
	switch st := actual.(type) {
	case *reflect.StructType:
		for i := 0; i < st.NumField(); i++ {
			enc.sendType(w, state, st.Field(i).Type)
		}
	case reflect.ArrayOrSliceType:
		enc.sendType(w, state, st.Elem())
	}
	return true
}

// sendType sends the type info to the other side, if necessary.
func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Type) (sent bool) {
	ut := userType(origt)
	if ut.isGobEncoder {
		// The rules are different: regardless of the underlying type's representation,
		// we need to tell the other side that this exact type is a GobEncoder.
		return enc.sendActualType(w, state, ut, ut.user)
	}

	// It's a concrete value, so drill down to the base type.
	switch rt := ut.base.(type) {
	default:
		// Basic types and interfaces do not need to be described.
		return
	case *reflect.SliceType:
		// If it's []uint8, don't send; it's considered basic.
		if rt.Elem().Kind() == reflect.Uint8 {
			return
		}
		// Otherwise we do send.
		break
	case *reflect.ArrayType:
		// arrays must be sent so we know their lengths and element types.
		break
	case *reflect.MapType:
		// maps must be sent so we know their lengths and key/value types.
		break
	case *reflect.StructType:
		// structs must be sent so we know their fields.
		break
	case *reflect.ChanType, *reflect.FuncType:
		// Probably a bad field in a struct.
		enc.badType(rt)
		return
	}

	return enc.sendActualType(w, state, ut, ut.base)
}

// Encode transmits the data item represented by the empty interface value,
// guaranteeing that all necessary type information has been transmitted first.
func (enc *Encoder) Encode(e interface{}) os.Error {
	return enc.EncodeValue(reflect.NewValue(e))
}

// sendTypeDescriptor makes sure the remote side knows about this type.
// It will send a descriptor if this is the first time the type has been
// sent.
func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *userTypeInfo) {
	// Make sure the type is known to the other side.
	// First, have we already sent this type?
	rt := ut.base
	if ut.isGobEncoder {
		rt = ut.user
	}
	if _, alreadySent := enc.sent[rt]; !alreadySent {
		// No, so send it.
		sent := enc.sendType(w, state, rt)
		if enc.err != nil {
			return
		}
		// If the type info has still not been transmitted, it means we have
		// a singleton basic type (int, []byte etc.) at top level.  We don't
		// need to send the type info but we do need to update enc.sent.
		if !sent {
			typeLock.Lock()
			info, err := getTypeInfo(ut)
			typeLock.Unlock()
			if err != nil {
				enc.setError(err)
				return
			}
			enc.sent[rt] = info.id
		}
	}
}

// sendTypeId sends the id, which must have already been defined.
func (enc *Encoder) sendTypeId(state *encoderState, ut *userTypeInfo) {
	// Identify the type of this top-level value.
	state.encodeInt(int64(enc.sent[ut.base]))
}

// EncodeValue transmits the data item represented by the reflection value,
// guaranteeing that all necessary type information has been transmitted first.
func (enc *Encoder) EncodeValue(value reflect.Value) os.Error {
	// Make sure we're single-threaded through here, so multiple
	// goroutines can share an encoder.
	enc.mutex.Lock()
	defer enc.mutex.Unlock()

	// Remove any nested writers remaining due to previous errors.
	enc.w = enc.w[0:1]

	ut, err := validUserType(value.Type())
	if err != nil {
		return err
	}

	enc.err = nil
	state := enc.newEncoderState(new(bytes.Buffer))

	enc.sendTypeDescriptor(enc.writer(), state, ut)
	enc.sendTypeId(state, ut)
	if enc.err != nil {
		return enc.err
	}

	// Encode the object.
	enc.encode(state.b, value, ut)
	if enc.err == nil {
		enc.writeMessage(enc.writer(), state.b)
	}

	enc.freeEncoderState(state)
	return enc.err
}
<commit_msg>gob: remove another allocation. The top level bytes.Buffer is always there and can be re-used. Rpc goes from 83 to 79 mallocs per round trip.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gob

import (
	"bytes"
	"io"
	"os"
	"reflect"
	"sync"
)

// An Encoder manages the transmission of type and data information to the
// other side of a connection.
type Encoder struct {
	mutex      sync.Mutex              // each item must be sent atomically
	w          []io.Writer             // where to send the data
	sent       map[reflect.Type]typeId // which types we've already sent
	countState *encoderState           // stage for writing counts
	freeList   *encoderState           // list of free encoderStates; avoids reallocation
	buf        []byte                  // for collecting the output.
	byteBuf    bytes.Buffer            // buffer for top-level encoderState
	err        os.Error
}

// NewEncoder returns a new encoder that will transmit on the io.Writer.
func NewEncoder(w io.Writer) *Encoder {
	enc := new(Encoder)
	enc.w = []io.Writer{w}
	enc.sent = make(map[reflect.Type]typeId)
	enc.countState = enc.newEncoderState(new(bytes.Buffer))
	return enc
}

// writer() returns the innermost writer the encoder is using
func (enc *Encoder) writer() io.Writer {
	return enc.w[len(enc.w)-1]
}

// pushWriter adds a writer to the encoder.
func (enc *Encoder) pushWriter(w io.Writer) {
	enc.w = append(enc.w, w)
}

// popWriter pops the innermost writer.
func (enc *Encoder) popWriter() {
	enc.w = enc.w[0 : len(enc.w)-1]
}

func (enc *Encoder) badType(rt reflect.Type) {
	enc.setError(os.ErrorString("gob: can't encode type " + rt.String()))
}

func (enc *Encoder) setError(err os.Error) {
	if enc.err == nil { // remember the first.
		enc.err = err
	}
}

// writeMessage sends the data item preceded by a unsigned count of its length.
func (enc *Encoder) writeMessage(w io.Writer, b *bytes.Buffer) {
	enc.countState.encodeUint(uint64(b.Len()))
	// Build the buffer.
	countLen := enc.countState.b.Len()
	total := countLen + b.Len()
	if total > len(enc.buf) {
		enc.buf = make([]byte, total+1000) // extra for growth
	}
	// Place the length before the data.
	// TODO(r): avoid the extra copy here.
	enc.countState.b.Read(enc.buf[0:countLen])
	// Now the data.
	b.Read(enc.buf[countLen:total])
	// Write the data.
	_, err := w.Write(enc.buf[0:total])
	if err != nil {
		enc.setError(err)
	}
}

// sendActualType sends the requested type, without further investigation, unless
// it's been sent before.
func (enc *Encoder) sendActualType(w io.Writer, state *encoderState, ut *userTypeInfo, actual reflect.Type) (sent bool) {
	if _, alreadySent := enc.sent[actual]; alreadySent {
		return false
	}
	typeLock.Lock()
	info, err := getTypeInfo(ut)
	typeLock.Unlock()
	if err != nil {
		enc.setError(err)
		return
	}
	// Send the pair (-id, type)
	// Id:
	state.encodeInt(-int64(info.id))
	// Type:
	enc.encode(state.b, reflect.NewValue(info.wire), wireTypeUserInfo)
	enc.writeMessage(w, state.b)
	if enc.err != nil {
		return
	}

	// Remember we've sent this type, both what the user gave us and the base type.
	enc.sent[ut.base] = info.id
	if ut.user != ut.base {
		enc.sent[ut.user] = info.id
	}
	// Now send the inner types
	switch st := actual.(type) {
	case *reflect.StructType:
		for i := 0; i < st.NumField(); i++ {
			enc.sendType(w, state, st.Field(i).Type)
		}
	case reflect.ArrayOrSliceType:
		enc.sendType(w, state, st.Elem())
	}
	return true
}

// sendType sends the type info to the other side, if necessary.
func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Type) (sent bool) {
	ut := userType(origt)
	if ut.isGobEncoder {
		// The rules are different: regardless of the underlying type's representation,
		// we need to tell the other side that this exact type is a GobEncoder.
		return enc.sendActualType(w, state, ut, ut.user)
	}

	// It's a concrete value, so drill down to the base type.
	switch rt := ut.base.(type) {
	default:
		// Basic types and interfaces do not need to be described.
		return
	case *reflect.SliceType:
		// If it's []uint8, don't send; it's considered basic.
		if rt.Elem().Kind() == reflect.Uint8 {
			return
		}
		// Otherwise we do send.
		break
	case *reflect.ArrayType:
		// arrays must be sent so we know their lengths and element types.
		break
	case *reflect.MapType:
		// maps must be sent so we know their lengths and key/value types.
		break
	case *reflect.StructType:
		// structs must be sent so we know their fields.
		break
	case *reflect.ChanType, *reflect.FuncType:
		// Probably a bad field in a struct.
		enc.badType(rt)
		return
	}

	return enc.sendActualType(w, state, ut, ut.base)
}

// Encode transmits the data item represented by the empty interface value,
// guaranteeing that all necessary type information has been transmitted first.
func (enc *Encoder) Encode(e interface{}) os.Error {
	return enc.EncodeValue(reflect.NewValue(e))
}

// sendTypeDescriptor makes sure the remote side knows about this type.
// It will send a descriptor if this is the first time the type has been
// sent.
func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *userTypeInfo) {
	// Make sure the type is known to the other side.
	// First, have we already sent this type?
	rt := ut.base
	if ut.isGobEncoder {
		rt = ut.user
	}
	if _, alreadySent := enc.sent[rt]; !alreadySent {
		// No, so send it.
		sent := enc.sendType(w, state, rt)
		if enc.err != nil {
			return
		}
		// If the type info has still not been transmitted, it means we have
		// a singleton basic type (int, []byte etc.) at top level.  We don't
		// need to send the type info but we do need to update enc.sent.
		if !sent {
			typeLock.Lock()
			info, err := getTypeInfo(ut)
			typeLock.Unlock()
			if err != nil {
				enc.setError(err)
				return
			}
			enc.sent[rt] = info.id
		}
	}
}

// sendTypeId sends the id, which must have already been defined.
func (enc *Encoder) sendTypeId(state *encoderState, ut *userTypeInfo) {
	// Identify the type of this top-level value.
	state.encodeInt(int64(enc.sent[ut.base]))
}

// EncodeValue transmits the data item represented by the reflection value,
// guaranteeing that all necessary type information has been transmitted first.
func (enc *Encoder) EncodeValue(value reflect.Value) os.Error {
	// Make sure we're single-threaded through here, so multiple
	// goroutines can share an encoder.
	enc.mutex.Lock()
	defer enc.mutex.Unlock()

	// Remove any nested writers remaining due to previous errors.
	enc.w = enc.w[0:1]

	ut, err := validUserType(value.Type())
	if err != nil {
		return err
	}

	enc.err = nil
	enc.byteBuf.Reset()
	state := enc.newEncoderState(&enc.byteBuf)

	enc.sendTypeDescriptor(enc.writer(), state, ut)
	enc.sendTypeId(state, ut)
	if enc.err != nil {
		return enc.err
	}

	// Encode the object.
	enc.encode(state.b, value, ut)
	if enc.err == nil {
		enc.writeMessage(enc.writer(), state.b)
	}

	enc.freeEncoderState(state)
	return enc.err
}
<|endoftext|>


<commit_before>package ecsbackend

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"strings"
	"text/template"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/quintilesims/layer0/api/backend"
	"github.com/quintilesims/layer0/api/backend/ecs/id"
	"github.com/quintilesims/layer0/common/aws/autoscaling"
	"github.com/quintilesims/layer0/common/aws/ec2"
	"github.com/quintilesims/layer0/common/aws/ecs"
	"github.com/quintilesims/layer0/common/config"
	"github.com/quintilesims/layer0/common/errors"
	"github.com/quintilesims/layer0/common/models"
	"github.com/quintilesims/layer0/common/waitutils"
)

type ECSEnvironmentManager struct {
	ECS         ecs.Provider
	EC2         ec2.Provider
	AutoScaling autoscaling.Provider
	Backend     backend.Backend
	Clock       waitutils.Clock
}

func NewECSEnvironmentManager(
	ecsprovider ecs.Provider,
	ec2 ec2.Provider,
	asg autoscaling.Provider,
	backend backend.Backend) *ECSEnvironmentManager {

	return &ECSEnvironmentManager{
		ECS:         ecsprovider,
		EC2:         ec2,
		AutoScaling: asg,
		Backend:     backend,
		Clock:       waitutils.RealClock{},
	}
}

func (this *ECSEnvironmentManager) ListEnvironments() ([]*models.Environment, error) {
	clusters, err := this.ECS.Helper_DescribeClusters()
	if err != nil {
		return nil, err
	}

	environments := make([]*models.Environment, len(clusters))
	for i, cluster := range clusters {
		if strings.HasPrefix(*cluster.ClusterName, id.PREFIX) {
			ecsEnvironmentID := id.ECSEnvironmentID(*cluster.ClusterName)
			environments[i] = &models.Environment{
				EnvironmentID: ecsEnvironmentID.L0EnvironmentID(),
			}
		}
	}

	return environments, nil
}

func (this *ECSEnvironmentManager) GetEnvironment(environmentID string) (*models.Environment, error) {
	ecsEnvironmentID := id.L0EnvironmentID(environmentID).ECSEnvironmentID()
	cluster, err := this.ECS.DescribeCluster(ecsEnvironmentID.String())
	if err != nil {
		if ContainsErrCode(err, "ClusterNotFoundException") {
			return nil, errors.Newf(errors.InvalidEnvironmentID, "Environment with id '%s' was not found", environmentID)
		}

		return nil, err
	}

	return this.populateModel(cluster)
}

func (this *ECSEnvironmentManager) populateModel(cluster *ecs.Cluster) (*models.Environment, error) {
	// assuming id.ECSEnvironmentID == ECSEnvironmentID.ClusterName()
	ecsEnvironmentID := id.ECSEnvironmentID(*cluster.ClusterName)

	var clusterCount int
	var instanceSize string

	asg, err := this.describeAutoscalingGroup(ecsEnvironmentID)
	if err != nil {
		if ContainsErrMsg(err, "not found") {
			log.Errorf("Autoscaling Group for environment '%s' not found", ecsEnvironmentID)
		} else {
			return nil, err
		}
	}

	if asg != nil {
		clusterCount = len(asg.Instances)

		if asg.LaunchConfigurationName != nil {
			launchConfig, err := this.AutoScaling.DescribeLaunchConfiguration(*asg.LaunchConfigurationName)
			if err != nil {
				if ContainsErrMsg(err, "not found") {
					log.Errorf("Launch Config for environment '%s' not found", ecsEnvironmentID)
				} else {
					return nil, err
				}
			}

			if launchConfig != nil {
				instanceSize = *launchConfig.InstanceType
			}
		}
	}

	var securityGroupID string
	securityGroup, err := this.EC2.DescribeSecurityGroup(ecsEnvironmentID.SecurityGroupName())
	if err != nil {
		return nil, err
	}

	if securityGroup != nil {
		securityGroupID = *securityGroup.SecurityGroup.GroupId
	}
	model := &models.Environment{
		EnvironmentID:   ecsEnvironmentID.L0EnvironmentID(),
		ClusterCount:    clusterCount,
		InstanceSize:    instanceSize,
		SecurityGroupID: securityGroupID,
	}

	return model, nil
}

func (this *ECSEnvironmentManager) describeAutoscalingGroup(ecsEnvironmentID id.ECSEnvironmentID) (*autoscaling.Group, error) {
	autoScalingGroupName := ecsEnvironmentID.AutoScalingGroupName()
	asg, err := this.AutoScaling.DescribeAutoScalingGroup(autoScalingGroupName)
	if err != nil {
		return nil, err
	}

	return asg, nil
}

func (this *ECSEnvironmentManager) CreateEnvironment(
	environmentName string,
	instanceSize string,
	minClusterCount int,
	userDataTemplate []byte,
) (*models.Environment, error) {
	environmentID := id.GenerateHashedEntityID(environmentName)
	ecsEnvironmentID := id.L0EnvironmentID(environmentID).ECSEnvironmentID()

	if len(userDataTemplate) == 0 {
		userDataTemplate = defaultUserDataTemplate
	}

	userData, err := renderUserData(ecsEnvironmentID, userDataTemplate)
	if err != nil {
		return nil, err
	}

	cluster, err := this.ECS.CreateCluster(ecsEnvironmentID.String())
	if err != nil {
		return nil, err
	}

	description := "Auto-generated Layer0 Environment Security Group"
	vpcID := config.AWSVPCID()

	groupID, err := this.EC2.CreateSecurityGroup(ecsEnvironmentID.SecurityGroupName(), description, vpcID)
	if err != nil {
		return nil, err
	}

	// wait for security group to propagate
	this.Clock.Sleep(time.Second * 2)
	if err := this.EC2.AuthorizeSecurityGroupIngressFromGroup(groupID, groupID); err != nil {
		return nil, err
	}

	agentGroupID := config.AWSAgentGroupID()
	securityGroups := []*string{groupID, &agentGroupID}
	serviceAMI := config.AWSServiceAMI()
	ecsRole := config.AWSECSInstanceProfile()
	keyPair := config.AWSKeyPair()
	launchConfigurationName := ecsEnvironmentID.LaunchConfigurationName()

	if err := this.AutoScaling.CreateLaunchConfiguration(
		&launchConfigurationName,
		&serviceAMI,
		&ecsRole,
		&instanceSize,
		&keyPair,
		&userData,
		securityGroups,
	); err != nil {
		return nil, err
	}

	maxClusterCount := 0
	if minClusterCount > 0 {
		maxClusterCount = minClusterCount
	}

	if err := this.AutoScaling.CreateAutoScalingGroup(
		ecsEnvironmentID.AutoScalingGroupName(),
		launchConfigurationName,
		config.AWSPrivateSubnets(),
		minClusterCount,
		maxClusterCount,
	); err != nil {
		return nil, err
	}

	return this.populateModel(cluster)
}

func (this *ECSEnvironmentManager) UpdateEnvironment(environmentID string, minClusterCount int) (*models.Environment, error) {
	model, err := this.GetEnvironment(environmentID)
	if err != nil {
		return nil, err
	}

	if err := this.updateEnvironmentMinCount(model, minClusterCount); err != nil {
		return nil, err
	}

	return model, nil
}

func (this *ECSEnvironmentManager) updateEnvironmentMinCount(model *models.Environment, minClusterCount int) error {
	ecsEnvironmentID := id.L0EnvironmentID(model.EnvironmentID).ECSEnvironmentID()
	autoScalingGroupName := ecsEnvironmentID.AutoScalingGroupName()

	asg, err := this.describeAutoscalingGroup(ecsEnvironmentID)
	if err != nil {
		return err
	}

	if int(*asg.MaxSize) < minClusterCount {
		if err := this.AutoScaling.UpdateAutoScalingGroupMaxSize(autoScalingGroupName, minClusterCount); err != nil {
			return err
		}
	}

	if err := this.AutoScaling.UpdateAutoScalingGroupMinSize(autoScalingGroupName, minClusterCount); err != nil {
		return err
	}

	return nil
}

func (this *ECSEnvironmentManager) DeleteEnvironment(environmentID string) error {
	ecsEnvironmentID := id.L0EnvironmentID(environmentID).ECSEnvironmentID()

	autoScalingGroupName := ecsEnvironmentID.AutoScalingGroupName()
	if err := this.AutoScaling.UpdateAutoScalingGroupMinSize(autoScalingGroupName, 0); err != nil {
		if !ContainsErrMsg(err, "name not found") && !ContainsErrMsg(err, "is pending delete") {
			return err
		}
	}

	if err := this.AutoScaling.UpdateAutoScalingGroupMaxSize(autoScalingGroupName, 0); err != nil {
		if !ContainsErrMsg(err, "name not found") && !ContainsErrMsg(err, "is pending delete") {
			return err
		}
	}

	if err := this.AutoScaling.DeleteAutoScalingGroup(&autoScalingGroupName); err != nil {
		if !ContainsErrMsg(err, "name not found") {
			return err
		}
	}

	launchConfigurationName := ecsEnvironmentID.LaunchConfigurationName()
	if err := this.AutoScaling.DeleteLaunchConfiguration(&launchConfigurationName); err != nil {
		if !ContainsErrMsg(err, "name not found") {
			return err
		}
	}

	if err := this.waitForAutoScalingGroupInactive(ecsEnvironmentID); err != nil {
		return err
	}

	securityGroup, err := this.EC2.DescribeSecurityGroup(ecsEnvironmentID.SecurityGroupName())
	if err != nil {
		return err
	}

	if securityGroup != nil {
		if err := this.waitForSecurityGroupDeleted(securityGroup); err != nil {
			return err
		}
	}

	if err := this.ECS.DeleteCluster(ecsEnvironmentID.String()); err != nil {
		if !ContainsErrCode(err, "ClusterNotFoundException") {
			return err
		}
	}

	return nil
}

func (this *ECSEnvironmentManager) waitForAutoScalingGroupInactive(ecsEnvironmentID id.ECSEnvironmentID) error {
	autoScalingGroupName := ecsEnvironmentID.AutoScalingGroupName()

	check := func() (bool, error) {
		group, err := this.AutoScaling.DescribeAutoScalingGroup(autoScalingGroupName)
		if err != nil {
			if ContainsErrMsg(err, "not found") {
				return true, nil
			}

			return false, err
		}

		log.Debugf("Waiting for ASG %s to delete (status: '%s')", autoScalingGroupName, pstring(group.Status))
		return false, nil
	}

	waiter := waitutils.Waiter{
		Name:    fmt.Sprintf("Stop Autoscaling %s", autoScalingGroupName),
		Retries: 30,
		Delay:   time.Second * 10,
		Clock:   this.Clock,
		Check:   check,
	}

	return waiter.Wait()
}

func (this *ECSEnvironmentManager) waitForSecurityGroupDeleted(securityGroup *ec2.SecurityGroup) error {
	check := func() (bool, error) {
		if err := this.EC2.DeleteSecurityGroup(securityGroup); err == nil {
			return true, nil
		}

		return false, nil
	}

	waiter := waitutils.Waiter{
		Name:    fmt.Sprintf("SecurityGroup delete for '%v'", securityGroup),
		Retries: 30,
		Delay:   time.Second * 10,
		Clock:   this.Clock,
		Check:   check,
	}

	return waiter.Wait()
}

func renderUserData(ecsEnvironmentID id.ECSEnvironmentID, userData []byte) (string, error) {
	tmpl, err := template.New("").Parse(string(userData))
	if err != nil {
		return "", fmt.Errorf("Failed to parse user data: %v", err)
	}

	context := struct {
		ECSEnvironmentID string
		S3Bucket         string
	}{
		ECSEnvironmentID: ecsEnvironmentID.String(),
		S3Bucket:         config.AWSS3Bucket(),
	}

	var rendered bytes.Buffer
	if err := tmpl.Execute(&rendered, context); err != nil {
		return "", fmt.Errorf("Failed to render user data: %v", err)
	}

	return base64.StdEncoding.EncodeToString(rendered.Bytes()), nil
}

var defaultUserDataTemplate = []byte(
	`#!/bin/bash
    echo ECS_CLUSTER={{ .ECSEnvironmentID }} >> /etc/ecs/ecs.config
    echo ECS_ENGINE_AUTH_TYPE=dockercfg >> /etc/ecs/ecs.config
    yum install -y aws-cli awslogs jq
    aws s3 cp s3://{{ .S3Bucket }}/bootstrap/dockercfg dockercfg
    cfg=$(cat dockercfg)
    echo ECS_ENGINE_AUTH_DATA=$cfg >> /etc/ecs/ecs.config
    docker pull amazon/amazon-ecs-agent:latest
    start ecs
`)
<commit_msg>don't frontload environment slice<commit_after>package ecsbackend

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"strings"
	"text/template"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/quintilesims/layer0/api/backend"
	"github.com/quintilesims/layer0/api/backend/ecs/id"
	"github.com/quintilesims/layer0/common/aws/autoscaling"
	"github.com/quintilesims/layer0/common/aws/ec2"
	"github.com/quintilesims/layer0/common/aws/ecs"
	"github.com/quintilesims/layer0/common/config"
	"github.com/quintilesims/layer0/common/errors"
	"github.com/quintilesims/layer0/common/models"
	"github.com/quintilesims/layer0/common/waitutils"
)

type ECSEnvironmentManager struct {
	ECS         ecs.Provider
	EC2         ec2.Provider
	AutoScaling autoscaling.Provider
	Backend     backend.Backend
	Clock       waitutils.Clock
}

func NewECSEnvironmentManager(
	ecsprovider ecs.Provider,
	ec2 ec2.Provider,
	asg autoscaling.Provider,
	backend backend.Backend) *ECSEnvironmentManager {

	return &ECSEnvironmentManager{
		ECS:         ecsprovider,
		EC2:         ec2,
		AutoScaling: asg,
		Backend:     backend,
		Clock:       waitutils.RealClock{},
	}
}

func (this *ECSEnvironmentManager) ListEnvironments() ([]*models.Environment, error) {
	clusters, err := this.ECS.Helper_DescribeClusters()
	if err != nil {
		return nil, err
	}

	environments := []*models.Environment{}
	for _, cluster := range clusters {
		if strings.HasPrefix(*cluster.ClusterName, id.PREFIX) {
			ecsEnvironmentID := id.ECSEnvironmentID(*cluster.ClusterName)
			environment := &models.Environment{
				EnvironmentID: ecsEnvironmentID.L0EnvironmentID(),
			}

			environments = append(environments, environment)
		}
	}

	return environments, nil
}

func (this *ECSEnvironmentManager) GetEnvironment(environmentID string) (*models.Environment, error) {
	ecsEnvironmentID := id.L0EnvironmentID(environmentID).ECSEnvironmentID()
	cluster, err := this.ECS.DescribeCluster(ecsEnvironmentID.String())
	if err != nil {
		if ContainsErrCode(err, "ClusterNotFoundException") {
			return nil, errors.Newf(errors.InvalidEnvironmentID, "Environment with id '%s' was not found", environmentID)
		}

		return nil, err
	}

	return this.populateModel(cluster)
}

func (this *ECSEnvironmentManager) populateModel(cluster *ecs.Cluster) (*models.Environment, error) {
	// assuming id.ECSEnvironmentID == ECSEnvironmentID.ClusterName()
	ecsEnvironmentID := id.ECSEnvironmentID(*cluster.ClusterName)

	var clusterCount int
	var instanceSize string

	asg, err := this.describeAutoscalingGroup(ecsEnvironmentID)
	if err != nil {
		if ContainsErrMsg(err, "not found") {
			log.Errorf("Autoscaling Group for environment '%s' not found", ecsEnvironmentID)
		} else {
			return nil, err
		}
	}

	if asg != nil {
		clusterCount = len(asg.Instances)

		if asg.LaunchConfigurationName != nil {
			launchConfig, err := this.AutoScaling.DescribeLaunchConfiguration(*asg.LaunchConfigurationName)
			if err != nil {
				if ContainsErrMsg(err, "not found") {
					log.Errorf("Launch Config for environment '%s' not found", ecsEnvironmentID)
				} else {
					return nil, err
				}
			}

			if launchConfig != nil {
				instanceSize = *launchConfig.InstanceType
			}
		}
	}

	var securityGroupID string
	securityGroup, err := this.EC2.DescribeSecurityGroup(ecsEnvironmentID.SecurityGroupName())
	if err != nil {
		return nil, err
	}

	if securityGroup != nil {
		securityGroupID = *securityGroup.SecurityGroup.GroupId
	}
	model := &models.Environment{
		EnvironmentID:   ecsEnvironmentID.L0EnvironmentID(),
		ClusterCount:    clusterCount,
		InstanceSize:    instanceSize,
		SecurityGroupID: securityGroupID,
	}

	return model, nil
}

func (this *ECSEnvironmentManager) describeAutoscalingGroup(ecsEnvironmentID id.ECSEnvironmentID) (*autoscaling.Group, error) {
	autoScalingGroupName := ecsEnvironmentID.AutoScalingGroupName()
	asg, err := this.AutoScaling.DescribeAutoScalingGroup(autoScalingGroupName)
	if err != nil {
		return nil, err
	}

	return asg, nil
}

func (this *ECSEnvironmentManager) CreateEnvironment(
	environmentName string,
	instanceSize string,
	minClusterCount int,
	userDataTemplate []byte,
) (*models.Environment, error) {
	environmentID := id.GenerateHashedEntityID(environmentName)
	ecsEnvironmentID := id.L0EnvironmentID(environmentID).ECSEnvironmentID()

	if len(userDataTemplate) == 0 {
		userDataTemplate = defaultUserDataTemplate
	}

	userData, err := renderUserData(ecsEnvironmentID, userDataTemplate)
	if err != nil {
		return nil, err
	}

	cluster, err := this.ECS.CreateCluster(ecsEnvironmentID.String())
	if err != nil {
		return nil, err
	}

	description := "Auto-generated Layer0 Environment Security Group"
	vpcID := config.AWSVPCID()

	groupID, err := this.EC2.CreateSecurityGroup(ecsEnvironmentID.SecurityGroupName(), description, vpcID)
	if err != nil {
		return nil, err
	}

	// wait for security group to propagate
	this.Clock.Sleep(time.Second * 2)
	if err := this.EC2.AuthorizeSecurityGroupIngressFromGroup(groupID, groupID); err != nil {
		return nil, err
	}

	agentGroupID := config.AWSAgentGroupID()
	securityGroups := []*string{groupID, &agentGroupID}
	serviceAMI := config.AWSServiceAMI()
	ecsRole := config.AWSECSInstanceProfile()
	keyPair := config.AWSKeyPair()
	launchConfigurationName := ecsEnvironmentID.LaunchConfigurationName()

	if err := this.AutoScaling.CreateLaunchConfiguration(
		&launchConfigurationName,
		&serviceAMI,
		&ecsRole,
		&instanceSize,
		&keyPair,
		&userData,
		securityGroups,
	); err != nil {
		return nil, err
	}

	maxClusterCount := 0
	if minClusterCount > 0 {
		maxClusterCount = minClusterCount
	}

	if err := this.AutoScaling.CreateAutoScalingGroup(
		ecsEnvironmentID.AutoScalingGroupName(),
		launchConfigurationName,
		config.AWSPrivateSubnets(),
		minClusterCount,
		maxClusterCount,
	); err != nil {
		return nil, err
	}

	return this.populateModel(cluster)
}

func (this *ECSEnvironmentManager) UpdateEnvironment(environmentID string, minClusterCount int) (*models.Environment, error) {
	model, err := this.GetEnvironment(environmentID)
	if err != nil {
		return nil, err
	}

	if err := this.updateEnvironmentMinCount(model, minClusterCount); err != nil {
		return nil, err
	}

	return model, nil
}

func (this *ECSEnvironmentManager) updateEnvironmentMinCount(model *models.Environment, minClusterCount int) error {
	ecsEnvironmentID := id.L0EnvironmentID(model.EnvironmentID).ECSEnvironmentID()
	autoScalingGroupName := ecsEnvironmentID.AutoScalingGroupName()

	asg, err := this.describeAutoscalingGroup(ecsEnvironmentID)
	if err != nil {
		return err
	}

	if int(*asg.MaxSize) < minClusterCount {
		if err := this.AutoScaling.UpdateAutoScalingGroupMaxSize(autoScalingGroupName, minClusterCount); err != nil {
			return err
		}
	}

	if err := this.AutoScaling.UpdateAutoScalingGroupMinSize(autoScalingGroupName, minClusterCount); err != nil {
		return err
	}

	return nil
}

func (this *ECSEnvironmentManager) DeleteEnvironment(environmentID string) error {
	ecsEnvironmentID := id.L0EnvironmentID(environmentID).ECSEnvironmentID()

	autoScalingGroupName := ecsEnvironmentID.AutoScalingGroupName()
	if err := this.AutoScaling.UpdateAutoScalingGroupMinSize(autoScalingGroupName, 0); err != nil {
		if !ContainsErrMsg(err, "name not found") && !ContainsErrMsg(err, "is pending delete") {
			return err
		}
	}

	if err := this.AutoScaling.UpdateAutoScalingGroupMaxSize(autoScalingGroupName, 0); err != nil {
		if !ContainsErrMsg(err, "name not found") && !ContainsErrMsg(err, "is pending delete") {
			return err
		}
	}

	if err := this.AutoScaling.DeleteAutoScalingGroup(&autoScalingGroupName); err != nil {
		if !ContainsErrMsg(err, "name not found") {
			return err
		}
	}

	launchConfigurationName := ecsEnvironmentID.LaunchConfigurationName()
	if err := this.AutoScaling.DeleteLaunchConfiguration(&launchConfigurationName); err != nil {
		if !ContainsErrMsg(err, "name not found") {
			return err
		}
	}

	if err := this.waitForAutoScalingGroupInactive(ecsEnvironmentID); err != nil {
		return err
	}

	securityGroup, err := this.EC2.DescribeSecurityGroup(ecsEnvironmentID.SecurityGroupName())
	if err != nil {
		return err
	}

	if securityGroup != nil {
		if err := this.waitForSecurityGroupDeleted(securityGroup); err != nil {
			return err
		}
	}

	if err := this.ECS.DeleteCluster(ecsEnvironmentID.String()); err != nil {
		if !ContainsErrCode(err, "ClusterNotFoundException") {
			return err
		}
	}

	return nil
}

func (this *ECSEnvironmentManager) waitForAutoScalingGroupInactive(ecsEnvironmentID id.ECSEnvironmentID) error {
	autoScalingGroupName := ecsEnvironmentID.AutoScalingGroupName()

	check := func() (bool, error) {
		group, err := this.AutoScaling.DescribeAutoScalingGroup(autoScalingGroupName)
		if err != nil {
			if ContainsErrMsg(err, "not found") {
				return true, nil
			}

			return false, err
		}

		log.Debugf("Waiting for ASG %s to delete (status: '%s')", autoScalingGroupName, pstring(group.Status))
		return false, nil
	}

	waiter := waitutils.Waiter{
		Name:    fmt.Sprintf("Stop Autoscaling %s", autoScalingGroupName),
		Retries: 30,
		Delay:   time.Second * 10,
		Clock:   this.Clock,
		Check:   check,
	}

	return waiter.Wait()
}

func (this *ECSEnvironmentManager) waitForSecurityGroupDeleted(securityGroup *ec2.SecurityGroup) error {
	check := func() (bool, error) {
		if err := this.EC2.DeleteSecurityGroup(securityGroup); err == nil {
			return true, nil
		}

		return false, nil
	}

	waiter := waitutils.Waiter{
		Name:    fmt.Sprintf("SecurityGroup delete for '%v'", securityGroup),
		Retries: 30,
		Delay:   time.Second * 10,
		Clock:   this.Clock,
		Check:   check,
	}

	return waiter.Wait()
}

func renderUserData(ecsEnvironmentID id.ECSEnvironmentID, userData []byte) (string, error) {
	tmpl, err := template.New("").Parse(string(userData))
	if err != nil {
		return "", fmt.Errorf("Failed to parse user data: %v", err)
	}

	context := struct {
		ECSEnvironmentID string
		S3Bucket         string
	}{
		ECSEnvironmentID: ecsEnvironmentID.String(),
		S3Bucket:         config.AWSS3Bucket(),
	}

	var rendered bytes.Buffer
	if err := tmpl.Execute(&rendered, context); err != nil {
		return "", fmt.Errorf("Failed to render user data: %v", err)
	}

	return base64.StdEncoding.EncodeToString(rendered.Bytes()), nil
}

var defaultUserDataTemplate = []byte(
	`#!/bin/bash
    echo ECS_CLUSTER={{ .ECSEnvironmentID }} >> /etc/ecs/ecs.config
    echo ECS_ENGINE_AUTH_TYPE=dockercfg >> /etc/ecs/ecs.config
    yum install -y aws-cli awslogs jq
    aws s3 cp s3://{{ .S3Bucket }}/bootstrap/dockercfg dockercfg
    cfg=$(cat dockercfg)
    echo ECS_ENGINE_AUTH_DATA=$cfg >> /etc/ecs/ecs.config
    docker pull amazon/amazon-ecs-agent:latest
    start ecs
`)
<|endoftext|>


<commit_before>// The experiments package holds various experiments with NEAT.
package experiments

import (
	"github.com/yaricom/goNEAT/neat/genetics"
	"github.com/yaricom/goNEAT/neat"
	"fmt"
	"time"
)

// The interface describing evaluator for one epoch of evolution.
type EpochEvaluator interface {
	EpochEvaluate(pop *genetics.Population, epoch *Epoch, context *neat.NeatContext) (err error)
}

// The Experiment execution entry point
func (ex *Experiment) Execute(context *neat.NeatContext, start_genome *genetics.Genome, epoch_executor EpochEvaluator) (err error) {
	if ex.Trials == nil {
		ex.Trials = make(Trials, context.NumRuns)
	}

	var pop *genetics.Population
	for run := 0; run < context.NumRuns; run++ {
		neat.InfoLog("\n>>>>> Spawning new population ")
		pop, err = genetics.NewPopulation(start_genome, context)
		if err != nil {
			neat.InfoLog("Failed to spawn new population from start genome")
			return err
		} else {
			neat.InfoLog("OK <<<<<")
		}
		neat.InfoLog(">>>>> Verifying spawned population ")
		_, err = pop.Verify()
		if err != nil {
			neat.ErrorLog("\n!!!!! Population verification failed !!!!!")
			return err
		} else {
			neat.InfoLog("OK <<<<<")
		}

		// start new trial
		trial := Trial {
			Id:run,
		}

		for gen := 0; gen < context.NumGenerations; gen++ {
			neat.InfoLog(fmt.Sprintf(">>>>> Epoch: %d\tRun: %d\n", gen, run))
			epoch := Epoch{
				Id:gen,
			}
			err = epoch_executor.EpochEvaluate(pop, &epoch, context)
			if err != nil {
				neat.InfoLog(fmt.Sprintf("!!!!! Epoch %d evaluation failed !!!!!\n", gen))
				return err
			}
			epoch.Executed = time.Now()
			trial.Epochs = append(trial.Epochs, epoch)
			if epoch.Solved {
				neat.InfoLog(fmt.Sprintf(">>>>> The winner organism found in epoch %d! <<<<<\n", gen))
				break
			}
		}
		// store trial into experiment
		ex.Trials[run] = trial
	}

	return nil
}
<commit_msg>Fixed output format<commit_after>// The experiments package holds various experiments with NEAT.
package experiments

import (
	"github.com/yaricom/goNEAT/neat/genetics"
	"github.com/yaricom/goNEAT/neat"
	"fmt"
	"time"
)

// The interface describing evaluator for one epoch of evolution.
type EpochEvaluator interface {
	EpochEvaluate(pop *genetics.Population, epoch *Epoch, context *neat.NeatContext) (err error)
}

// The Experiment execution entry point
func (ex *Experiment) Execute(context *neat.NeatContext, start_genome *genetics.Genome, epoch_executor EpochEvaluator) (err error) {
	if ex.Trials == nil {
		ex.Trials = make(Trials, context.NumRuns)
	}

	var pop *genetics.Population
	for run := 0; run < context.NumRuns; run++ {
		neat.InfoLog("\n>>>>> Spawning new population ")
		pop, err = genetics.NewPopulation(start_genome, context)
		if err != nil {
			neat.InfoLog("Failed to spawn new population from start genome")
			return err
		} else {
			neat.InfoLog("OK <<<<<")
		}
		neat.InfoLog(">>>>> Verifying spawned population ")
		_, err = pop.Verify()
		if err != nil {
			neat.ErrorLog("\n!!!!! Population verification failed !!!!!")
			return err
		} else {
			neat.InfoLog("OK <<<<<")
		}

		// start new trial
		trial := Trial {
			Id:run,
		}

		for gen := 0; gen < context.NumGenerations; gen++ {
			neat.InfoLog(fmt.Sprintf(">>>>> Epoch:%3d\tRun: %d\n", gen, run))
			epoch := Epoch{
				Id:gen,
			}
			err = epoch_executor.EpochEvaluate(pop, &epoch, context)
			if err != nil {
				neat.InfoLog(fmt.Sprintf("!!!!! Epoch %d evaluation failed !!!!!\n", gen))
				return err
			}
			epoch.Executed = time.Now()
			trial.Epochs = append(trial.Epochs, epoch)
			if epoch.Solved {
				neat.InfoLog(fmt.Sprintf(">>>>> The winner organism found in epoch %d! <<<<<\n", gen))
				break
			}
		}
		// store trial into experiment
		ex.Trials[run] = trial
	}

	return nil
}
<|endoftext|>


<commit_before><commit_msg>HTTP handlers can now returns errors<commit_after><|endoftext|>