<commit_before>// Package deploy builds & deploys function zips.
package deploy

import (
	"errors"
	"fmt"
	"strings"

	"github.com/spf13/cobra"

	"github.com/apex/apex/cmd/apex/root"
	"github.com/apex/log"
)

// env supplied.
var env []string

// concurrency of deploys.
var concurrency int

// alias.
var alias string

// example output.
const example = `  Deploy all functions
  $ apex deploy

  Deploy specific functions
  $ apex deploy foo bar

  Deploy canary alias
  $ apex deploy foo --alias canary

  Deploy functions in a different project
  $ apex deploy -C ~/dev/myapp`

// Command config.
var Command = &cobra.Command{
	Use:     "deploy [<name>...]",
	Short:   "Deploy functions and config",
	Example: example,
	RunE:    run,
}

// Initialize.
func init() {
	root.Register(Command)

	f := Command.Flags()
	f.StringSliceVarP(&env, "set", "s", nil, "Set environment variable")
	f.StringVarP(&alias, "alias", "a", "current", "Function alias")
	f.IntVarP(&concurrency, "concurrency", "c", 5, "Concurrent deploys")
}

// Run command.
func run(c *cobra.Command, args []string) error {
	root.Project.Concurrency = concurrency
	root.Project.Alias = alias

	c.Root().PersistentFlags().Lookup("name string")

	if err := root.Project.LoadFunctions(args...); err != nil {
		return err
	}

	for _, s := range env {
		parts := strings.SplitN(s, "=", 2)
		if len(parts) == 2 {
			root.Project.Setenv(parts[0], parts[1])
		} else {
			err := fmt.Errorf("environment variable %s needs a value", parts[0])
			return errors.New(err)
		}
	}

	return root.Project.DeployAndClean()
}
<commit_msg>fix typo in previous commit<commit_after>// Package deploy builds & deploys function zips.
package deploy

import (
	"fmt"
	"strings"

	"github.com/spf13/cobra"

	"github.com/apex/apex/cmd/apex/root"
)

// env supplied.
var env []string

// concurrency of deploys.
var concurrency int

// alias.
var alias string

// example output.
const example = `  Deploy all functions
  $ apex deploy

  Deploy specific functions
  $ apex deploy foo bar

  Deploy canary alias
  $ apex deploy foo --alias canary

  Deploy functions in a different project
  $ apex deploy -C ~/dev/myapp`

// Command config.
var Command = &cobra.Command{
	Use:     "deploy [<name>...]",
	Short:   "Deploy functions and config",
	Example: example,
	RunE:    run,
}

// Initialize.
func init() {
	root.Register(Command)

	f := Command.Flags()
	f.StringSliceVarP(&env, "set", "s", nil, "Set environment variable")
	f.StringVarP(&alias, "alias", "a", "current", "Function alias")
	f.IntVarP(&concurrency, "concurrency", "c", 5, "Concurrent deploys")
}

// Run command.
func run(c *cobra.Command, args []string) error {
	root.Project.Concurrency = concurrency
	root.Project.Alias = alias

	c.Root().PersistentFlags().Lookup("name string")

	if err := root.Project.LoadFunctions(args...); err != nil {
		return err
	}

	for _, s := range env {
		parts := strings.SplitN(s, "=", 2)
		if len(parts) == 2 {
			root.Project.Setenv(parts[0], parts[1])
		} else {
			return fmt.Errorf("environment variable %s needs a value", parts[0])
		}
	}

	return root.Project.DeployAndClean()
}
<|endoftext|>


<commit_before>package depapi

import (
	"../../shared"
	"github.com/spf13/cobra"
	"net/url"
	"path"
)

var Cmd = &cobra.Command{
	Use:   "deploy",
	Short: "Deploys a revision of an existing API proxy",
	Long:  "Deploys a revision of an existing API proxy to an environment in an organization",
	Run: func(cmd *cobra.Command, args []string) {
		u, _ := url.Parse(shared.BaseURL)
		if overrides {
			q := u.Query()
			q.Set("override", "true")
			u.RawQuery = q.Encode()
		}
		u.Path = path.Join(u.Path, shared.RootArgs.Org, "environments", shared.RootArgs.Org, "apis", name, "revisions", revision,"deployments")
		shared.HttpClient(u.String())
	},
}

var name, revision string
var overrides bool

func init() {

	Cmd.Flags().StringVarP(&name, "name", "n",
		"", "API proxy name")
	Cmd.Flags().StringVarP(&shared.RootArgs.Env, "env", "e",
		"", "Apigee environment name")
	Cmd.Flags().StringVarP(&revision, "rev", "r",
		"", "API Proxy revision")
	Cmd.Flags().BoolVarP(&overrides, "ovr", "v",
		false, "Forces deployment of the new revision")

	Cmd.MarkFlagRequired("env")
	Cmd.MarkFlagRequired("name")

}
<commit_msg>change revision to v<commit_after>package depapi

import (
	"../../shared"
	"github.com/spf13/cobra"
	"net/url"
	"path"
)

var Cmd = &cobra.Command{
	Use:   "deploy",
	Short: "Deploys a revision of an existing API proxy",
	Long:  "Deploys a revision of an existing API proxy to an environment in an organization",
	Run: func(cmd *cobra.Command, args []string) {
		u, _ := url.Parse(shared.BaseURL)
		if overrides {
			q := u.Query()
			q.Set("override", "true")
			u.RawQuery = q.Encode()
		}
		u.Path = path.Join(u.Path, shared.RootArgs.Org, "environments", shared.RootArgs.Org, "apis", name, "revisions", revision,"deployments")
		shared.HttpClient(u.String())
	},
}

var name, revision string
var overrides bool

func init() {

	Cmd.Flags().StringVarP(&name, "name", "n",
		"", "API proxy name")
	Cmd.Flags().StringVarP(&shared.RootArgs.Env, "env", "e",
		"", "Apigee environment name")
	Cmd.Flags().StringVarP(&revision, "rev", "v",
		"", "API Proxy revision")
	Cmd.Flags().BoolVarP(&overrides, "ovr", "r",
		false, "Forces deployment of the new revision")

	Cmd.MarkFlagRequired("env")
	Cmd.MarkFlagRequired("name")

}
<|endoftext|>


<commit_before>package main

func initializeRoutes() {
	logger.Println("Initializing routes")

	// Group peer related routes together
	peerRoutes := router.Group("/voters")
	{
		peerRoutes.GET("/rewards/", GetVoters)
		peerRoutes.GET("/blocked/", GetBlocked)
	}
	deleRoutes := router.Group("/delegate")
	{
		deleRoutes.GET("/", GetDelegate)
		deleRoutes.GET("/config", GetDelegateSharingConfig)
	}
}
<commit_msg>Enabling CORS<commit_after>package main

import (
	"gopkg.in/gin-gonic/gin.v1"
)

//CORSMiddleware function enabling CORS requests
func CORSMiddleware() gin.HandlerFunc {
    return func(c *gin.Context) {
        c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
        c.Writer.Header().Set("Access-Control-Max-Age", "86400")
        c.Writer.Header().Set("Access-Control-Allow-Methods", "GET")
        c.Writer.Header().Set("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
        c.Writer.Header().Set("Access-Control-Expose-Headers", "Content-Length")
        c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")

        if c.Request.Method == "OPTIONS" {
            c.AbortWithStatus(200)
        } else {
            c.Next()
        }
    }
}

func initializeRoutes() {
	logger.Println("Initializing routes")

	router.Use(CORSMiddleware())
	// Group peer related routes together
	peerRoutes := router.Group("/voters")
	{
		peerRoutes.GET("/rewards", GetVoters)
		peerRoutes.GET("/blocked", GetBlocked)
	}
	deleRoutes := router.Group("/delegate")
	{
		deleRoutes.GET("", GetDelegate)
		deleRoutes.GET("/config", GetDelegateSharingConfig)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"log"

	"github.com/google/go-containerregistry/pkg/crane"
	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/daemon"
	"github.com/google/go-containerregistry/pkg/v1/tarball"
	"github.com/google/go-containerregistry/pkg/v1/validate"
	"github.com/spf13/cobra"
)

func init() { Root.AddCommand(NewCmdValidate()) }

// NewCmdValidate creates a new cobra.Command for the validate subcommand.
func NewCmdValidate() *cobra.Command {
	var tarballPath, remoteRef, daemonRef string
	validateCmd := &cobra.Command{
		Use:   "validate",
		Short: "Validate that an image is well-formed",
		Args:  cobra.ExactArgs(0),
		Run: func(_ *cobra.Command, args []string) {
			for flag, maker := range map[string]func(string) (v1.Image, error){
				tarballPath: makeTarball,
				remoteRef:   crane.Pull,
				daemonRef:   makeDaemon,
			} {
				img, err := maker(flag)
				if err != nil {
					log.Fatalf("failed to read image %s: %v", flag, err)
				}

				if err := validate.Image(img); err != nil {
					fmt.Printf("FAIL: %s: %v\n", flag, err)
				} else {
					fmt.Printf("PASS: %s\n", flag)
				}
			}
		},
	}
	validateCmd.Flags().StringVar(&tarballPath, "tarball", "", "Path to tarball to validate")
	validateCmd.Flags().StringVar(&remoteRef, "remote", "", "Name of remote image to validate")
	validateCmd.Flags().StringVar(&daemonRef, "daemon", "", "Name of image in daemon to validate")

	return validateCmd
}

func makeTarball(path string) (v1.Image, error) {
	return tarball.ImageFromPath(path, nil)
}

func makeDaemon(daemonRef string) (v1.Image, error) {
	ref, err := name.ParseReference(daemonRef)
	if err != nil {
		return nil, fmt.Errorf("parsing daemon ref %q: %v", daemonRef, err)
	}

	return daemon.Image(ref)
}
<commit_msg>Drop daemon dependency from crane (#587)<commit_after>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"log"

	"github.com/google/go-containerregistry/pkg/crane"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/tarball"
	"github.com/google/go-containerregistry/pkg/v1/validate"
	"github.com/spf13/cobra"
)

func init() { Root.AddCommand(NewCmdValidate()) }

// NewCmdValidate creates a new cobra.Command for the validate subcommand.
func NewCmdValidate() *cobra.Command {
	var tarballPath, remoteRef string
	validateCmd := &cobra.Command{
		Use:   "validate",
		Short: "Validate that an image is well-formed",
		Args:  cobra.ExactArgs(0),
		Run: func(_ *cobra.Command, args []string) {
			for flag, maker := range map[string]func(string) (v1.Image, error){
				tarballPath: makeTarball,
				remoteRef:   crane.Pull,
			} {
				if flag == "" {
					continue
				}
				img, err := maker(flag)
				if err != nil {
					log.Fatalf("failed to read image %s: %v", flag, err)
				}

				if err := validate.Image(img); err != nil {
					fmt.Printf("FAIL: %s: %v\n", flag, err)
				} else {
					fmt.Printf("PASS: %s\n", flag)
				}
			}
		},
	}
	validateCmd.Flags().StringVar(&tarballPath, "tarball", "", "Path to tarball to validate")
	validateCmd.Flags().StringVar(&remoteRef, "remote", "", "Name of remote image to validate")

	return validateCmd
}

func makeTarball(path string) (v1.Image, error) {
	return tarball.ImageFromPath(path, nil)
}
<|endoftext|>


<commit_before>package entrypoint

import (
	"context"
	"encoding/json"
	"sync/atomic"

	"github.com/datawire/ambassador/pkg/acp"
	"github.com/datawire/ambassador/pkg/debug"
	"github.com/datawire/ambassador/pkg/kates"
	"github.com/datawire/ambassador/pkg/snapshot/v1"
	"github.com/datawire/ambassador/pkg/watt"
	"github.com/datawire/dlib/dlog"
)

func watcher(ctx context.Context, ambwatch *acp.AmbassadorWatcher, encoded *atomic.Value, clusterID string, version string) {
	client, err := kates.NewClient(kates.ClientConfig{})
	if err != nil {
		panic(err)
	}

	serverTypeList, err := client.ServerPreferredResources()
	if err != nil {
		// It's possible that an error prevented listing some apigroups, but not all; so
		// process the output even if there is an error.
		dlog.Infof(ctx, "Warning, unable to list api-resources: %v", err)
	}

	interestingTypes := GetInterestingTypes(ctx, serverTypeList)
	queries := GetQueries(ctx, interestingTypes)

	ambassadorMeta := getAmbassadorMeta(GetAmbassadorId(), clusterID, version, client)

	// **** SETUP DONE for the Kubernetes Watcher

	notify := func(ctx context.Context, disposition SnapshotDisposition, snap *snapshot.Snapshot) {
		if disposition == SnapshotReady {
			notifyReconfigWebhooks(ctx, ambwatch)
		}
	}

	k8sSrc := newK8sSource(client)
	consulSrc := &consulWatcher{}
	istioCertSrc := newIstioCertSource()

	watcherLoop(ctx, encoded, k8sSrc, queries, consulSrc, istioCertSrc, notify, ambassadorMeta)
}

func getAmbassadorMeta(ambassadorID string, clusterID string, version string, client *kates.Client) *snapshot.AmbassadorMetaInfo {
	ambMeta := &snapshot.AmbassadorMetaInfo{
		ClusterID:         clusterID,
		AmbassadorID:      ambassadorID,
		AmbassadorVersion: version,
	}
	kubeServerVer, err := client.ServerVersion()
	if err == nil {
		ambMeta.KubeVersion = kubeServerVer.GitVersion
	}
	return ambMeta
}

type SnapshotProcessor func(context.Context, SnapshotDisposition, *snapshot.Snapshot)
type SnapshotDisposition int

const (
	// Indicates the watcher is still in the booting process and the snapshot has dangling pointers.
	SnapshotIncomplete SnapshotDisposition = iota
	// Indicates that the watcher is deferring processing of the snapshot because it is considered
	// to be a product of churn.
	SnapshotDefer
	// Indicates that the watcher is dropping the snapshot because it has determined that it is
	// logically a noop.
	SnapshotDrop
	// Indicates that the snapshot is ready to be processed.
	SnapshotReady
)

// watcher is _the_ thing that watches all the different kinds of Ambassador configuration
// events that we care about. This right here is pretty much the root of everything flowing
// into Ambassador from the outside world, so:
//
// ******** READ THE FOLLOWING COMMENT CAREFULLY! ********
//
// Since this is where _all_ the different kinds of these events (K8s, Consul, filesystem,
// whatever) are brought together and examined, and where we pass judgement on whether or
// not a given event is worth reconfiguring Ambassador or not, the interactions between
// this function and other pieces of the system can be quite a bit more complex than you
// might expect. There are two really huge things you should be bearing in mind if you
// need to work on this:
//
// 1. The set of things we're watching is not static, but it must converge.
//
//    An example: you can set up a Kubernetes watch that finds a KubernetesConsulResolver
//    resource, which will then prompt a new Consul watch to happen. At present, nothing
//    that that Consul watch could find is capable of prompting a new Kubernetes watch to
//    be created. This is important: it would be fairly easy to change things such that
//    there is a feedback loop where the set of things we watch does not converge on a
//    stable set. If such a loop exists, fixing it will probably require grokking this
//    watcher function, kates.Accumulator, and maybe the reconcilers in consul.go and
//    endpoints.go as well.
//
// 2. No one source of input events can be allowed to alter the event stream for another
//    source.
//
//    An example: at one point, a bug in the watcher function resulted in the Kubernetes
//    watcher being able to decide to short-circuit a watcher iteration -- which had the
//    effect of allowing the K8s watcher to cause _Consul_ events to be ignored. That's
//    not OK. To guard against this:
//
//    A. Refrain from adding state to the watcher loop.
//    B. Try very very hard to keep logic that applies to a single source within that
//       source's specific case in the watcher's select statement.
//    C. Don't add any more select statements, so that B. above is unambiguous.
//
// 3. If you add a new channel to watch, you MUST make sure it has a way to let the loop
//    know whether it saw real changes, so that the short-circuit logic works correctly.
//    That said, recognize that the way it works now, with the state for the individual
//    watchers in the watcher() function itself is a crock, and the path forward is to
//    refactor them into classes that can separate things more cleanly.
//
// 4. If you don't fully understand everything above, _do not touch this function without
//    guidance_.
func watcherLoop(ctx context.Context, encoded *atomic.Value, k8sSrc K8sSource, queries []kates.Query,
	consulWatcher Watcher, istioCertSrc IstioCertSource, notify SnapshotProcessor, ambassadorMeta *snapshot.AmbassadorMetaInfo) {
	// These timers keep track of various parts of the processing of the watcher loop. They don't
	// directly impact the logic at all.
	dbg := debug.FromContext(ctx)

	katesUpdateTimer := dbg.Timer("katesUpdate")
	consulUpdateTimer := dbg.Timer("consulUpdate")
	istioCertUpdateTimer := dbg.Timer("istioCertUpdate")
	notifyWebhooksTimer := dbg.Timer("notifyWebhooks")
	parseAnnotationsTimer := dbg.Timer("parseAnnotations")
	reconcileSecretsTimer := dbg.Timer("reconcileSecrets")
	reconcileConsulTimer := dbg.Timer("reconcileConsul")

	// Synthesize the thing that knows how to watch kubernetes resources. This is always either a
	// *kates.Accumulator or the mocks supplied by the Fake harness we use for testing.
	k8sWatcher := k8sSrc.Watch(ctx, queries...)
	// Synthesize the thing that knows how to validate kubernetes resources. This is always calling
	// into the kates validator even when we are being driven by the Fake harness.
	validator := newResourceValidator()

	// The consul object manages our consul watches. The consulWatcher is either the real thing or a
	// mock from the Fake harness.
	consul := newConsul(ctx, consulWatcher) // Consul Watcher: state manager

	// Likewise for the Istio cert watcher and manager.
	istioCertWatcher := istioCertSrc.Watch(ctx)
	istio := newIstioCertWatchManager(ctx, istioCertWatcher)

	// All the variables above this line are read-only with respect to the logic in the loop. The
	// four variables defined below are updated as the loop iterates.

	// These two variables represent the view of the kubernetes world and the view of the consul
	// world. This view is constructed from the raw data given to us from each respective source,
	// plus additional fields that are computed based on the raw data. These are cumulative values,
	// they always represent the entire state of their respective worlds and are fully replaced each
	// time there is an update from their respective sources.
	k8sSnapshot := NewKubernetesSnapshot()
	consulSnapshot := &watt.ConsulSnapshot{}
	// XXX: you would expect there to be an analogous snapshot for istio secrets, however the istio
	// source works by directly munging the k8sSnapshot.

	// The unsentDeltas field tracks the stream of deltas that have occured in between each
	// kubernetes snapshot. This is a passthrough of the full stream of deltas reported by kates
	// which is in turn a facade fo the deltas reported by client-go.
	var unsentDeltas []*kates.Delta

	// Is this the very first reconfigure we've done?
	firstReconfig := true

	for {
		dlog.Debugf(ctx, "WATCHER: --------")

		// XXX Hack: the istioCertWatchManager needs to reset at the start of the
		// loop, for now. A better way, I think, will be to instead track deltas in
		// ReconcileSecrets -- that way we can ditch this crap and Istio-cert changes
		// that somehow don't generate an actual change will still not trigger a
		// reconfigure.
		istio.StartLoop(ctx)

		select {
		case <-k8sWatcher.Changed():
			// Kubernetes has some changes, so we need to handle them.
			stop := katesUpdateTimer.Start()

			// We could probably get a win in some scenarios by using this filtered update thing to
			// pre-exclude based on ambassador-id.
			var deltas []*kates.Delta
			changed := k8sWatcher.FilteredUpdate(k8sSnapshot, &deltas, func(un *kates.Unstructured) bool {
				return validator.isValid(ctx, un)
			})

			stop()

			if !changed {
				continue
			}

			unsentDeltas = append(unsentDeltas, deltas...)

			parseAnnotationsTimer.Time(func() {
				parseAnnotations(k8sSnapshot)
			})

			reconcileSecretsTimer.Time(func() {
				ReconcileSecrets(ctx, k8sSnapshot)
			})
			reconcileConsulTimer.Time(func() {
				ReconcileConsul(ctx, consul, k8sSnapshot)
			})
		case <-consul.changed():
			dlog.Debugf(ctx, "WATCHER: Consul fired")

			consulUpdateTimer.Time(func() {
				consul.update(consulSnapshot)
			})
		case icertUpdate := <-istio.Changed():
			// The Istio cert has some changes, so we need to handle them.
			istioCertUpdateTimer.Time(func() {
				istio.Update(ctx, icertUpdate, k8sSnapshot)
			})

			reconcileSecretsTimer.Time(func() {
				ReconcileSecrets(ctx, k8sSnapshot)
			})

		// BEFORE ADDING A NEW CHANNEL, READ THE COMMENT AT THE TOP OF THIS
		// FUNCTION so you don't break the short-circuiting logic.

		case <-ctx.Done():
			return
		}

		sn := &snapshot.Snapshot{
			Kubernetes:     k8sSnapshot,
			Consul:         consulSnapshot,
			Invalid:        validator.getInvalid(),
			Deltas:         unsentDeltas,
			AmbassadorMeta: ambassadorMeta,
		}

		if !consul.isBootstrapped() {
			notify(ctx, SnapshotIncomplete, sn)
			continue
		}

		unsentDeltas = nil

		snapshotJSON, err := json.MarshalIndent(sn, "", "  ")
		if err != nil {
			panic(err)
		}

		// ...then stash this snapshot and fire off webhooks.
		encoded.Store(snapshotJSON)
		if firstReconfig {
			dlog.Debugf(ctx, "WATCHER: Bootstrapped! Computing initial configuration...")
			firstReconfig = false
		}

		// Finally, use the reconfigure webhooks to let the rest of Ambassador
		// know about the new configuration.
		notifyWebhooksTimer.Time(func() {
			notify(ctx, SnapshotReady, sn)
		})
	}
}

// The kates aka "real" version of our injected dependencies.
type k8sSource struct {
	client *kates.Client
}

func (k *k8sSource) Watch(ctx context.Context, queries ...kates.Query) K8sWatcher {
	acc := k.client.Watch(ctx, queries...)
	return acc
}

func newK8sSource(client *kates.Client) *k8sSource {
	return &k8sSource{
		client: client,
	}
}
<commit_msg>(from AES) Separate the busness logic of processing snapshots from the plumbing and/or concurrent logic. This should be a noop change with the exception of adding a mutex to protect the loop state.<commit_after>package entrypoint

import (
	"context"
	"encoding/json"
	"sync"
	"sync/atomic"

	"github.com/datawire/ambassador/pkg/acp"
	"github.com/datawire/ambassador/pkg/debug"
	"github.com/datawire/ambassador/pkg/kates"
	"github.com/datawire/ambassador/pkg/snapshot/v1"
	"github.com/datawire/ambassador/pkg/watt"
	"github.com/datawire/dlib/dlog"
)

func watcher(ctx context.Context, ambwatch *acp.AmbassadorWatcher, encoded *atomic.Value, clusterID string, version string) {
	client, err := kates.NewClient(kates.ClientConfig{})
	if err != nil {
		panic(err)
	}

	serverTypeList, err := client.ServerPreferredResources()
	if err != nil {
		// It's possible that an error prevented listing some apigroups, but not all; so
		// process the output even if there is an error.
		dlog.Infof(ctx, "Warning, unable to list api-resources: %v", err)
	}

	interestingTypes := GetInterestingTypes(ctx, serverTypeList)
	queries := GetQueries(ctx, interestingTypes)

	ambassadorMeta := getAmbassadorMeta(GetAmbassadorId(), clusterID, version, client)

	// **** SETUP DONE for the Kubernetes Watcher

	notify := func(ctx context.Context, disposition SnapshotDisposition, snap *snapshot.Snapshot) {
		if disposition == SnapshotReady {
			notifyReconfigWebhooks(ctx, ambwatch)
		}
	}

	k8sSrc := newK8sSource(client)
	consulSrc := &consulWatcher{}
	istioCertSrc := newIstioCertSource()

	watcherLoop(ctx, encoded, k8sSrc, queries, consulSrc, istioCertSrc, notify, ambassadorMeta)
}

func getAmbassadorMeta(ambassadorID string, clusterID string, version string, client *kates.Client) *snapshot.AmbassadorMetaInfo {
	ambMeta := &snapshot.AmbassadorMetaInfo{
		ClusterID:         clusterID,
		AmbassadorID:      ambassadorID,
		AmbassadorVersion: version,
	}
	kubeServerVer, err := client.ServerVersion()
	if err == nil {
		ambMeta.KubeVersion = kubeServerVer.GitVersion
	}
	return ambMeta
}

type SnapshotProcessor func(context.Context, SnapshotDisposition, *snapshot.Snapshot)
type SnapshotDisposition int

const (
	// Indicates the watcher is still in the booting process and the snapshot has dangling pointers.
	SnapshotIncomplete SnapshotDisposition = iota
	// Indicates that the watcher is deferring processing of the snapshot because it is considered
	// to be a product of churn.
	SnapshotDefer
	// Indicates that the watcher is dropping the snapshot because it has determined that it is
	// logically a noop.
	SnapshotDrop
	// Indicates that the snapshot is ready to be processed.
	SnapshotReady
)

// watcher is _the_ thing that watches all the different kinds of Ambassador configuration
// events that we care about. This right here is pretty much the root of everything flowing
// into Ambassador from the outside world, so:
//
// ******** READ THE FOLLOWING COMMENT CAREFULLY! ********
//
// Since this is where _all_ the different kinds of these events (K8s, Consul, filesystem,
// whatever) are brought together and examined, and where we pass judgement on whether or
// not a given event is worth reconfiguring Ambassador or not, the interactions between
// this function and other pieces of the system can be quite a bit more complex than you
// might expect. There are two really huge things you should be bearing in mind if you
// need to work on this:
//
// 1. The set of things we're watching is not static, but it must converge.
//
//    An example: you can set up a Kubernetes watch that finds a KubernetesConsulResolver
//    resource, which will then prompt a new Consul watch to happen. At present, nothing
//    that that Consul watch could find is capable of prompting a new Kubernetes watch to
//    be created. This is important: it would be fairly easy to change things such that
//    there is a feedback loop where the set of things we watch does not converge on a
//    stable set. If such a loop exists, fixing it will probably require grokking this
//    watcher function, kates.Accumulator, and maybe the reconcilers in consul.go and
//    endpoints.go as well.
//
// 2. No one source of input events can be allowed to alter the event stream for another
//    source.
//
//    An example: at one point, a bug in the watcher function resulted in the Kubernetes
//    watcher being able to decide to short-circuit a watcher iteration -- which had the
//    effect of allowing the K8s watcher to cause _Consul_ events to be ignored. That's
//    not OK. To guard against this:
//
//    A. Refrain from adding state to the watcher loop.
//    B. Try very very hard to keep logic that applies to a single source within that
//       source's specific case in the watcher's select statement.
//    C. Don't add any more select statements, so that B. above is unambiguous.
//
// 3. If you add a new channel to watch, you MUST make sure it has a way to let the loop
//    know whether it saw real changes, so that the short-circuit logic works correctly.
//    That said, recognize that the way it works now, with the state for the individual
//    watchers in the watcher() function itself is a crock, and the path forward is to
//    refactor them into classes that can separate things more cleanly.
//
// 4. If you don't fully understand everything above, _do not touch this function without
//    guidance_.
func watcherLoop(ctx context.Context, encoded *atomic.Value, k8sSrc K8sSource, queries []kates.Query,
	consulWatcher Watcher, istioCertSrc IstioCertSource, notify SnapshotProcessor, ambassadorMeta *snapshot.AmbassadorMetaInfo) {
	// Ambassador has three sources of inputs: kubernetes, consul, and the filesystem. The job of
	// the watcherLoop is to read updates from all three of these sources, assemble them into a
	// single coherent configuration, and pass them along to other parts of ambassador for
	// processing.

	// The watcherLoop must decide what information is relevant to solicit from each source. This is
	// decided a bit differently for each source.
	//
	// For kubernetes the set of subscriptions is basically hardcoded to the set of resources
	// defined in interesting_types.go, this is filtered down at boot based on RBAC limitations. The
	// filtered list is used to construct the queries that are passed into this function, and that
	// set of queries remains fixed for the lifetime of the loop, i.e. the lifetime of the
	// abmassador process (unless we are testing, in which case we may run the watcherLoop more than
	// once in a single process).
	//
	// For the consul source we derive the set of resources to watch based on the configuration in
	// kubernetes, i.e. we watch the services defined in Mappings that are configured to use a
	// consul resolver. We use the ConsulResolver that a given Mapping is configured with to find
	// the datacenter to query.
	//
	// The filesystem datasource is for istio secrets. XXX fill in more

	// Each time the wathcerLoop wakes up, it assembles updates from whatever source woke it up into
	// its view of the world. It then determines if enough information has been assembled to
	// consider ambassador "booted" and if so passes the updated view along to its output (the
	// SnapshotProcessor).x

	// Setup our three sources of ambassador inputs: kubernetes, consul, and the filesystem. Each of
	// these have interfaces that enable us to run with the "real" implementation or a mock
	// implementation for our Fake test harness.
	k8sWatcher := k8sSrc.Watch(ctx, queries...)
	consul := newConsul(ctx, consulWatcher)
	istioCertWatcher := istioCertSrc.Watch(ctx)
	istio := newIstioCertWatchManager(ctx, istioCertWatcher)

	// SnapshotHolder tracks all the data structures that get updated by the various sources of
	// information. It also holds the business logic that converts the data as received to a more
	// amenable form for processing. It not only serves to group these together, but it also
	// provides a mutex to protect access to the data.
	snapshots := NewSnapshotHolder(ambassadorMeta)

	for {
		dlog.Debugf(ctx, "WATCHER: --------")

		// XXX Hack: the istioCertWatchManager needs to reset at the start of the
		// loop, for now. A better way, I think, will be to instead track deltas in
		// ReconcileSecrets -- that way we can ditch this crap and Istio-cert changes
		// that somehow don't generate an actual change will still not trigger a
		// reconfigure.
		istio.StartLoop(ctx)

		select {
		case <-k8sWatcher.Changed():
			// Kubernetes has some changes, so we need to handle them.
			changed := snapshots.K8sUpdate(ctx, k8sWatcher, consul)
			if !changed {
				continue
			}
		case <-consul.changed():
			dlog.Debugf(ctx, "WATCHER: Consul fired")
			snapshots.ConsulUpdate(ctx, consul)
		case icertUpdate := <-istio.Changed():
			// The Istio cert has some changes, so we need to handle them.
			snapshots.IstioUpdate(ctx, istio, icertUpdate)

		// BEFORE ADDING A NEW CHANNEL, READ THE COMMENT AT THE TOP OF THIS
		// FUNCTION so you don't break the short-circuiting logic.

		case <-ctx.Done():
			return
		}

		snapshots.Notify(ctx, encoded, consul, notify)
	}
}

// SnapshotHolder is responsible for holding
type SnapshotHolder struct {
	mutex sync.Mutex

	// The thing that knows how to validate kubernetes resources. This is always calling into the
	// kates validator even when we are being driven by the Fake harness.
	validator *resourceValidator

	// Ambassadro meta info to pass along in the snapshot.
	ambassadorMeta *snapshot.AmbassadorMetaInfo

	// These two fields represent the view of the kubernetes world and the view of the consul
	// world. This view is constructed from the raw data given to us from each respective source,
	// plus additional fields that are computed based on the raw data. These are cumulative values,
	// they always represent the entire state of their respective worlds.
	k8sSnapshot    *snapshot.KubernetesSnapshot
	consulSnapshot *watt.ConsulSnapshot
	// XXX: you would expect there to be an analogous snapshot for istio secrets, however the istio
	// source works by directly munging the k8sSnapshot.

	// The unsentDeltas field tracks the stream of deltas that have occured in between each
	// kubernetes snapshot. This is a passthrough of the full stream of deltas reported by kates
	// which is in turn a facade fo the deltas reported by client-go.
	unsentDeltas []*kates.Delta

	// Has the very first reconfig happened?
	firstReconfig bool
}

func NewSnapshotHolder(ambassadorMeta *snapshot.AmbassadorMetaInfo) *SnapshotHolder {
	return &SnapshotHolder{
		validator:      newResourceValidator(),
		ambassadorMeta: ambassadorMeta,
		k8sSnapshot:    NewKubernetesSnapshot(),
		consulSnapshot: &watt.ConsulSnapshot{},
		firstReconfig:  true,
	}
}

// Get the raw update from the kubernetes watcher, then redo our computed view.
func (sh *SnapshotHolder) K8sUpdate(ctx context.Context, watcher K8sWatcher, consul *consul) bool {
	dbg := debug.FromContext(ctx)

	katesUpdateTimer := dbg.Timer("katesUpdate")
	parseAnnotationsTimer := dbg.Timer("parseAnnotations")
	reconcileSecretsTimer := dbg.Timer("reconcileSecrets")
	reconcileConsulTimer := dbg.Timer("reconcileConsul")

	sh.mutex.Lock()
	defer sh.mutex.Unlock()

	// We could probably get a win in some scenarios by using this filtered update thing to
	// pre-exclude based on ambassador-id.
	var deltas []*kates.Delta
	var changed bool
	katesUpdateTimer.Time(func() {
		changed = watcher.FilteredUpdate(sh.k8sSnapshot, &deltas, func(un *kates.Unstructured) bool {
			return sh.validator.isValid(ctx, un)
		})
	})

	if !changed {
		return false
	}

	sh.unsentDeltas = append(sh.unsentDeltas, deltas...)

	parseAnnotationsTimer.Time(func() {
		parseAnnotations(sh.k8sSnapshot)
	})

	reconcileSecretsTimer.Time(func() {
		ReconcileSecrets(ctx, sh.k8sSnapshot)
	})
	reconcileConsulTimer.Time(func() {
		ReconcileConsul(ctx, consul, sh.k8sSnapshot)
	})

	return true
}

func (sh *SnapshotHolder) ConsulUpdate(ctx context.Context, consul *consul) bool {
	sh.mutex.Lock()
	defer sh.mutex.Unlock()
	consul.update(sh.consulSnapshot)
	return true
}

func (sh *SnapshotHolder) IstioUpdate(ctx context.Context, istio *istioCertWatchManager,
	icertUpdate IstioCertUpdate) bool {
	dbg := debug.FromContext(ctx)

	istioCertUpdateTimer := dbg.Timer("istioCertUpdate")
	reconcileSecretsTimer := dbg.Timer("reconcileSecrets")

	sh.mutex.Lock()
	defer sh.mutex.Unlock()

	istioCertUpdateTimer.Time(func() {
		istio.Update(ctx, icertUpdate, sh.k8sSnapshot)
	})

	reconcileSecretsTimer.Time(func() {
		ReconcileSecrets(ctx, sh.k8sSnapshot)
	})

	return true
}

func (sh *SnapshotHolder) Notify(ctx context.Context, encoded *atomic.Value, consul *consul, notify SnapshotProcessor) {
	dbg := debug.FromContext(ctx)

	notifyWebhooksTimer := dbg.Timer("notifyWebhooks")

	sh.mutex.Lock()
	defer sh.mutex.Unlock()

	sn := &snapshot.Snapshot{
		Kubernetes: sh.k8sSnapshot,
		Consul:     sh.consulSnapshot,
		Invalid:    sh.validator.getInvalid(),
		Deltas:     sh.unsentDeltas,
	}

	if !consul.isBootstrapped() {
		notify(ctx, SnapshotIncomplete, sn)
		return
	}

	sh.unsentDeltas = nil

	snapshotJSON, err := json.MarshalIndent(sn, "", "  ")
	if err != nil {
		panic(err)
	}

	// ...then stash this snapshot and fire off webhooks.
	encoded.Store(snapshotJSON)
	if sh.firstReconfig {
		dlog.Debugf(ctx, "WATCHER: Bootstrapped! Computing initial configuration...")
		sh.firstReconfig = false
	}

	// Finally, use the reconfigure webhooks to let the rest of Ambassador
	// know about the new configuration.
	notifyWebhooksTimer.Time(func() {
		notify(ctx, SnapshotReady, sn)
	})
}

// The kates aka "real" version of our injected dependencies.
type k8sSource struct {
	client *kates.Client
}

func (k *k8sSource) Watch(ctx context.Context, queries ...kates.Query) K8sWatcher {
	acc := k.client.Watch(ctx, queries...)
	return acc
}

func newK8sSource(client *kates.Client) *k8sSource {
	return &k8sSource{
		client: client,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/fatih/color"
)

// GitPodsRunner builds, runs, stops and restarts GitPods.
type GitPodsRunner struct {
	name    string
	env     []string
	cmd     *exec.Cmd
	restart chan bool
}

func NewGitPodsRunner(name string, env []string) *GitPodsRunner {
	return &GitPodsRunner{
		name:    name,
		env:     env,
		restart: make(chan bool, 16),
	}
}

func (r *GitPodsRunner) Name() string {
	return r.name
}

func (r *GitPodsRunner) Run() error {
	if err := r.Build(); err == nil {
		r.restart <- true
	}

	for {
		_, more := <-r.restart
		if more {
			if r.cmd != nil {
				r.Stop()
			}

			go func() {
				r.cmd = exec.Command("./dev/" + r.name)
				r.cmd.Env = r.env
				stdout, err := r.cmd.StdoutPipe()
				if err != nil {
					return
				}
				stderr, err := r.cmd.StderrPipe()
				if err != nil {
					return
				}

				multi := io.MultiReader(stdout, stderr)

				if r.cmd.Start() != nil {
					return
				}

				scanner := bufio.NewScanner(multi)

				for scanner.Scan() {
					fmt.Printf("%s\t%s\n", color.HiBlueString(r.name), scanner.Text())
				}

				if err = r.cmd.Wait(); err != nil {
					return
				}
			}()
		} else {
			return nil
		}
	}
}

func (r *GitPodsRunner) Stop() {
	if r.cmd == nil || r.cmd.Process == nil {
		return
	}
	r.cmd.Process.Kill()
}

func (r *GitPodsRunner) Build() error {
	cmd := exec.Command("go", "build", "-v", "-ldflags", "-w -extldflags '-static'", "-o", "./dev/"+r.name, "./cmd/"+r.name)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	fmt.Println(strings.Join(cmd.Args, " "))

	return cmd.Run()
}

func (r *GitPodsRunner) Restart() {
	r.restart <- true
}

func (r *GitPodsRunner) Shutdown() {
	close(r.restart)
	r.Stop()
}

// CaddyRunner runs caddy
type CaddyRunner struct {
	cmd *exec.Cmd
}

func (r *CaddyRunner) Run() error {
	r.cmd = exec.Command(filepath.Join(".", "dev", "caddy"), "-conf", "./dev/Caddyfile")
	stdout, err := r.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := r.cmd.StderrPipe()
	if err != nil {
		return err
	}

	multi := io.MultiReader(stdout, stderr)

	if err := r.cmd.Start(); err != nil {
		return err
	}

	scanner := bufio.NewScanner(multi)

	for scanner.Scan() {
		fmt.Printf("%s\t%s\n", color.HiBlueString("caddy"), scanner.Text())
	}

	return r.cmd.Wait()
}

func (r *CaddyRunner) Stop() {
	if r.cmd == nil || r.cmd.Process == nil {
		return
	}
	r.cmd.Process.Kill()
}
<commit_msg>Use make command to build Go binaries<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/fatih/color"
)

// GitPodsRunner builds, runs, stops and restarts GitPods.
type GitPodsRunner struct {
	name    string
	env     []string
	cmd     *exec.Cmd
	restart chan bool
}

func NewGitPodsRunner(name string, env []string) *GitPodsRunner {
	return &GitPodsRunner{
		name:    name,
		env:     env,
		restart: make(chan bool, 16),
	}
}

func (r *GitPodsRunner) Name() string {
	return r.name
}

func (r *GitPodsRunner) Run() error {
	if err := r.Build(); err == nil {
		r.restart <- true
	}

	for {
		_, more := <-r.restart
		if more {
			if r.cmd != nil {
				r.Stop()
			}

			go func() {
				r.cmd = exec.Command("./dev/" + r.name)
				r.cmd.Env = r.env
				stdout, err := r.cmd.StdoutPipe()
				if err != nil {
					return
				}
				stderr, err := r.cmd.StderrPipe()
				if err != nil {
					return
				}

				multi := io.MultiReader(stdout, stderr)

				if r.cmd.Start() != nil {
					return
				}

				scanner := bufio.NewScanner(multi)

				for scanner.Scan() {
					fmt.Printf("%s\t%s\n", color.HiBlueString(r.name), scanner.Text())
				}

				if err = r.cmd.Wait(); err != nil {
					return
				}
			}()
		} else {
			return nil
		}
	}
}

func (r *GitPodsRunner) Stop() {
	if r.cmd == nil || r.cmd.Process == nil {
		return
	}
	r.cmd.Process.Kill()
}

func (r *GitPodsRunner) Build() error {
	cmd := exec.Command("make", "dev/"+r.name)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	fmt.Println(strings.Join(cmd.Args, " "))

	return cmd.Run()
}

func (r *GitPodsRunner) Restart() {
	r.restart <- true
}

func (r *GitPodsRunner) Shutdown() {
	close(r.restart)
	r.Stop()
}

// CaddyRunner runs caddy
type CaddyRunner struct {
	cmd *exec.Cmd
}

func (r *CaddyRunner) Run() error {
	r.cmd = exec.Command(filepath.Join(".", "dev", "caddy"), "-conf", "./dev/Caddyfile")
	stdout, err := r.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := r.cmd.StderrPipe()
	if err != nil {
		return err
	}

	multi := io.MultiReader(stdout, stderr)

	if err := r.cmd.Start(); err != nil {
		return err
	}

	scanner := bufio.NewScanner(multi)

	for scanner.Scan() {
		fmt.Printf("%s\t%s\n", color.HiBlueString("caddy"), scanner.Text())
	}

	return r.cmd.Wait()
}

func (r *CaddyRunner) Stop() {
	if r.cmd == nil || r.cmd.Process == nil {
		return
	}
	r.cmd.Process.Kill()
}
<|endoftext|>


<commit_before>package main

import (
	"net"
	"net/http"
	"runtime"
	"sync"
	"time"

	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

type webServer struct {
	name       string
	maxClients int
	gw         *Gateway

	httpListener net.Listener
	httpServer   *http.Server

	httpsListener net.Listener
	httpsServer   *http.Server

	router *httprouter.Router

	waitExitFunc waitExitFunc
}

func newWebServer(name string, httpAddr, httpsAddr string, maxClients int,
	gw *Gateway) *webServer {
	this := &webServer{
		name:       name,
		gw:         gw,
		maxClients: maxClients,
		router:     httprouter.New(),
	}

	if httpAddr != "" {
		this.httpServer = &http.Server{
			Addr:           httpAddr,
			Handler:        this.router,
			ReadTimeout:    httpReadTimeout,
			WriteTimeout:   httpWriteTimeout,
			MaxHeaderBytes: httpHeaderMaxBytes,
		}
	}

	if httpsAddr != "" {
		this.httpsServer = &http.Server{
			Addr:           httpsAddr,
			Handler:        this.router,
			ReadTimeout:    httpReadTimeout,
			WriteTimeout:   httpWriteTimeout,
			MaxHeaderBytes: httpHeaderMaxBytes,
		}
	}

	return this
}

func (this *webServer) Start() {
	if this.httpsServer != nil {
		this.startServer(true)
	}

	if this.httpServer != nil {
		this.startServer(false)
	}

}

func (this *webServer) Router() *httprouter.Router {
	return this.router
}

func (this *webServer) startServer(https bool) {
	var err error
	waitListenerUp := make(chan struct{})
	go func() {
		if options.cpuAffinity {
			runtime.LockOSThread()
		}

		var (
			retryDelay         time.Duration
			theListener        net.Listener
			waitListenerUpOnce sync.Once
			addr               string
		)
		for {
			if https {
				this.httpsListener, err = net.Listen("tcp", this.httpsServer.Addr)
				this.httpsListener, err = setupHttpsListener(this.httpsListener,
					this.gw.certFile, this.gw.keyFile)
				if err != nil {
					panic(err)
				}

				theListener = this.httpsListener
				addr = this.httpsServer.Addr
			} else {
				this.httpListener, err = net.Listen("tcp", this.httpServer.Addr)
				theListener = this.httpListener
				addr = this.httpServer.Addr
			}

			if err != nil {
				if retryDelay == 0 {
					retryDelay = 50 * time.Millisecond
				} else {
					retryDelay = 2 * retryDelay
				}
				if maxDelay := time.Second; retryDelay > maxDelay {
					retryDelay = maxDelay
				}
				log.Error("%s listener %v, retry in %v", this.name, err, retryDelay)
				time.Sleep(retryDelay)
				continue
			}

			theListener = LimitListener(this.gw, theListener, this.maxClients)
			waitListenerUpOnce.Do(func() {
				close(waitListenerUp)
			})

			// on non-temporary err, net/http will close the listener
			if https {
				err = this.httpsServer.Serve(theListener)
			} else {
				err = this.httpServer.Serve(theListener)
			}

			select {
			case <-this.gw.shutdownCh:
				log.Trace("%s server stopped on %s", this.name, addr)
				return

			default:
				log.Error("%s server: %v", this.name, err)
			}
		}
	}()

	// FIXME if net.Listen fails, kateway will not be able to stop
	// e,g. start a kateway, then start another, the 2nd will not be able to stop

	// wait for the listener up
	<-waitListenerUp

	this.gw.wg.Add(1)
	if https {
		go this.waitExitFunc(this.httpsServer, this.httpsListener, this.gw.shutdownCh)
	} else {
		go this.waitExitFunc(this.httpServer, this.httpListener, this.gw.shutdownCh)
	}

	if https {
		log.Info("%s https server ready on %s", this.name, this.httpsServer.Addr)
	} else {
		log.Info("%s http server ready on %s", this.name, this.httpServer.Addr)
	}
}
<commit_msg>handles the err of https Listen<commit_after>package main

import (
	"net"
	"net/http"
	"runtime"
	"sync"
	"time"

	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

type webServer struct {
	name       string
	maxClients int
	gw         *Gateway

	httpListener net.Listener
	httpServer   *http.Server

	httpsListener net.Listener
	httpsServer   *http.Server

	router *httprouter.Router

	waitExitFunc waitExitFunc
}

func newWebServer(name string, httpAddr, httpsAddr string, maxClients int,
	gw *Gateway) *webServer {
	this := &webServer{
		name:       name,
		gw:         gw,
		maxClients: maxClients,
		router:     httprouter.New(),
	}

	if httpAddr != "" {
		this.httpServer = &http.Server{
			Addr:           httpAddr,
			Handler:        this.router,
			ReadTimeout:    httpReadTimeout,
			WriteTimeout:   httpWriteTimeout,
			MaxHeaderBytes: httpHeaderMaxBytes,
		}
	}

	if httpsAddr != "" {
		this.httpsServer = &http.Server{
			Addr:           httpsAddr,
			Handler:        this.router,
			ReadTimeout:    httpReadTimeout,
			WriteTimeout:   httpWriteTimeout,
			MaxHeaderBytes: httpHeaderMaxBytes,
		}
	}

	return this
}

func (this *webServer) Start() {
	if this.httpsServer != nil {
		this.startServer(true)
	}

	if this.httpServer != nil {
		this.startServer(false)
	}

}

func (this *webServer) Router() *httprouter.Router {
	return this.router
}

func (this *webServer) startServer(https bool) {
	var err error
	waitListenerUp := make(chan struct{})
	go func() {
		if options.cpuAffinity {
			runtime.LockOSThread()
		}

		var (
			retryDelay         time.Duration
			theListener        net.Listener
			waitListenerUpOnce sync.Once
			addr               string
		)
		for {
			if https {
				this.httpsListener, err = net.Listen("tcp", this.httpsServer.Addr)
				if err != nil {
					if retryDelay == 0 {
						retryDelay = 50 * time.Millisecond
					} else {
						retryDelay = 2 * retryDelay
					}
					if maxDelay := time.Second; retryDelay > maxDelay {
						retryDelay = maxDelay
					}
					log.Error("%s listener %v, retry in %v", this.name, err, retryDelay)
					time.Sleep(retryDelay)
					continue
				}

				this.httpsListener, err = setupHttpsListener(this.httpsListener,
					this.gw.certFile, this.gw.keyFile)
				if err != nil {
					panic(err)
				}

				theListener = this.httpsListener
				addr = this.httpsServer.Addr
			} else {
				this.httpListener, err = net.Listen("tcp", this.httpServer.Addr)
				theListener = this.httpListener
				addr = this.httpServer.Addr
			}

			if err != nil {
				if retryDelay == 0 {
					retryDelay = 50 * time.Millisecond
				} else {
					retryDelay = 2 * retryDelay
				}
				if maxDelay := time.Second; retryDelay > maxDelay {
					retryDelay = maxDelay
				}
				log.Error("%s listener %v, retry in %v", this.name, err, retryDelay)
				time.Sleep(retryDelay)
				continue
			}

			theListener = LimitListener(this.gw, theListener, this.maxClients)
			waitListenerUpOnce.Do(func() {
				close(waitListenerUp)
			})

			// on non-temporary err, net/http will close the listener
			if https {
				err = this.httpsServer.Serve(theListener)
			} else {
				err = this.httpServer.Serve(theListener)
			}

			select {
			case <-this.gw.shutdownCh:
				log.Trace("%s server stopped on %s", this.name, addr)
				return

			default:
				log.Error("%s server: %v", this.name, err)
			}
		}
	}()

	// FIXME if net.Listen fails, kateway will not be able to stop
	// e,g. start a kateway, then start another, the 2nd will not be able to stop

	// wait for the listener up
	<-waitListenerUp

	this.gw.wg.Add(1)
	if https {
		go this.waitExitFunc(this.httpsServer, this.httpsListener, this.gw.shutdownCh)
	} else {
		go this.waitExitFunc(this.httpServer, this.httpListener, this.gw.shutdownCh)
	}

	if https {
		log.Info("%s https server ready on %s", this.name, this.httpsServer.Addr)
	} else {
		log.Info("%s http server ready on %s", this.name, this.httpServer.Addr)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/grafeas/kritis/pkg/attestlib"
	"github.com/grafeas/kritis/pkg/kritis/apis/kritis/v1beta1"
	"github.com/grafeas/kritis/pkg/kritis/crd/vulnzsigningpolicy"
	"github.com/grafeas/kritis/pkg/kritis/metadata/containeranalysis"
	"github.com/grafeas/kritis/pkg/kritis/signer"
	"github.com/grafeas/kritis/pkg/kritis/util"
	"k8s.io/apimachinery/pkg/util/yaml"
)

type SignerMode string

const (
	CheckAndSign  SignerMode = "check-and-sign"
	CheckOnly     SignerMode = "check-only"
	BypassAndSign SignerMode = "bypass-and-sign"
)

var (
	// input flags
	mode               string
	image              string
	vulnzTimeout       string
	policyPath         string
	attestationProject string
	overwrite          bool
	noteName           string
	// input flags: pgp key flags
	pgpPriKeyPath string
	pgpPassphrase string
	// input flags: kms flags
	kmsKeyName   string
	kmsDigestAlg string

	// helper global variables
	modeFlags   *flag.FlagSet
	modeExample string
)

func init() {
	// need to add all flags to avoid "flag not provided error"
	addBasicFlags(flag.CommandLine)
	addCheckFlags(flag.CommandLine)
	addSignFlags(flag.CommandLine)
}

func addBasicFlags(fs *flag.FlagSet) {
	fs.StringVar(&mode, "mode", "check-and-sign", "(required) mode of operation, check-and-sign|check-only|bypass-and-sign")
	fs.StringVar(&image, "image", "", "(required) image url, e.g., gcr.io/foo/bar@sha256:abcd")
}

func addCheckFlags(fs *flag.FlagSet) {
	fs.StringVar(&policyPath, "policy", "", "(required for check) vulnerability signing policy file path, e.g., /tmp/vulnz_signing_policy.yaml")
	fs.StringVar(&vulnzTimeout, "vulnz_timeout", "5m", "timeout for polling image vulnerability , e.g., 600s, 5m")
}

func addSignFlags(fs *flag.FlagSet) {
	fs.StringVar(&noteName, "note_name", "", "(required for sign) note name that created attestations are attached to, in the form of projects/[PROVIDER_ID]/notes/[NOTE_ID]")
	fs.StringVar(&attestationProject, "attestation_project", "", "project id for GCP project that stores attestation, use image project if set to empty")
	fs.BoolVar(&overwrite, "overwrite", false, "overwrite attestation if already existed")
	fs.StringVar(&kmsKeyName, "kms_key_name", "", "kms key name, in the format of in the format projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*")
	fs.StringVar(&kmsDigestAlg, "kms_digest_alg", "", "kms digest algorithm, must be one of SHA256|SHA384|SHA512, and the same as specified by the key version's algorithm")
	fs.StringVar(&pgpPriKeyPath, "pgp_private_key", "", "pgp private signing key path, e.g., /dev/shm/key.pgp")
	fs.StringVar(&pgpPassphrase, "pgp_passphrase", "", "passphrase for pgp private key, if any")
}

// parseSignerMode creates mode-specific flagset and analyze actions (check, sign) for given mode
func parseSignerMode(mode SignerMode) (doCheck bool, doSign bool, err error) {
	modeFlags, doCheck, doSign, err = flag.NewFlagSet("", flag.ExitOnError), false, false, nil
	addBasicFlags(modeFlags)
	switch mode {
	case CheckAndSign:
		addCheckFlags(modeFlags)
		addSignFlags(modeFlags)
		doCheck, doSign = true, true
		modeExample = `	./signer \
	-mode=check-and-sign \
	-image=gcr.io/my-image-repo/image-1@sha256:123 \
	-policy=policy.yaml \
	-note_name=projects/$NOTE_PROJECT/NOTES/$NOTE_ID \
	-kms_key_name=projects/$KMS_PROJECT/locations/$KMS_KEYLOCATION/keyRings/$KMS_KEYRING/cryptoKeys/$KMS_KEYNAME/cryptoKeyVersions/$KMS_KEYVERSION \
	-kms_digest_alg=SHA512`
	case BypassAndSign:
		addSignFlags(modeFlags)
		doSign = true
		modeExample = `	./signer \
	-mode=bypass-and-sign \
	-image=gcr.io/my-image-repo/image-1@sha256:123 \
	-note_name=projects/$NOTE_PROJECT/NOTES/$NOTE_ID \
	-kms_key_name=projects/$KMS_PROJECT/locations/$KMS_KEYLOCATION/keyRings/$KMS_KEYRING/cryptoKeys/$KMS_KEYNAME/cryptoKeyVersions/$KMS_KEYVERSION \
	-kms_digest_alg=SHA512`
	case CheckOnly:
		addCheckFlags(modeFlags)
		doCheck = true
		modeExample = `	./signer \
	-mode=check-only \
	-image=gcr.io/my-image-repo/image-1@sha256:123 \
	-policy=policy.yaml`
	default:
		return false, false, fmt.Errorf("unrecognized mode %s, must be one of check-and-sign|check-only|bypass-and-sign", mode)
	}
	modeFlags.Parse(os.Args[1:])
	return doCheck, doSign, err
}

func exitOnBadFlags(mode SignerMode, err string) {
	fmt.Fprintf(modeFlags.Output(), "Usage of signer's %s mode:\n", mode)
	modeFlags.PrintDefaults()
	fmt.Fprintf(modeFlags.Output(), "Example (%s mode):\n %s\n", mode, modeExample)
	fmt.Fprintf(modeFlags.Output(), "Bad flags for mode %s: %v. \n", mode, err)
	os.Exit(1)
}

func main() {
	flag.Parse()
	glog.Infof("Signer mode: %s.", mode)

	doCheck, doSign, err := parseSignerMode(SignerMode(mode))
	if err != nil {
		glog.Fatalf("Parse mode err %v.", err)
	}

	// Check image url is non-empty
	// TODO: check and format image url to
	//  gcr.io/project-id/rest-of-image-path@sha256:[sha-value]
	if image == "" {
		exitOnBadFlags(SignerMode(mode), "image url is empty")
	}

	// Create a client
	client, err := containeranalysis.New()
	if err != nil {
		glog.Fatalf("Could not initialize the client %v", err)
	}

	if doCheck {
		// Read the vulnz signing policy
		if policyPath == "" {
			exitOnBadFlags(SignerMode(mode), "policy path is empty")
		}
		policy := v1beta1.VulnzSigningPolicy{}
		policyFile, err := os.Open(policyPath)
		if err != nil {
			glog.Fatalf("Fail to load vulnz signing policy: %v", err)
		}
		defer policyFile.Close()
		// err = json.Unmarshal(policyFile, &policy)
		if err := yaml.NewYAMLToJSONDecoder(policyFile).Decode(&policy); err != nil {
			glog.Fatalf("Fail to parse policy file: %v", err)
		} else {
			glog.Infof("Policy req: %v\n", policy.Spec.ImageVulnerabilityRequirements)
		}

		timeout, err := time.ParseDuration(vulnzTimeout)
		if err != nil {
			glog.Fatalf("Fail to parse timeout %v", err)
		}
		err = client.WaitForVulnzAnalysis(image, timeout)
		if err != nil {
			glog.Fatalf("Error waiting for vulnerability analysis %v", err)
		}

		// Read the vulnz scanning events
		vulnz, err := client.Vulnerabilities(image)
		if err != nil {
			glog.Fatalf("Found err %s", err)
		}
		if vulnz == nil {
			glog.Fatalf("Expected some vulnerabilities. Nil found")
		}

		violations, err := vulnzsigningpolicy.ValidateVulnzSigningPolicy(policy, image, vulnz)
		if err != nil {
			glog.Fatalf("Error when evaluating image %q against policy %q", image, policy.Name)
		}
		if violations != nil && len(violations) != 0 {
			glog.Errorf("Image %q does not pass VulnzSigningPolicy %q:", image, policy.Name)
			glog.Errorf("Found %d violations in image %s:", len(violations), image)
			for _, v := range violations {
				glog.Error(v.Reason())
			}
			os.Exit(1)
		}
		glog.Infof("Image %q passes VulnzSigningPolicy %s.", image, policy.Name)
	}

	if doSign {
		// Read the signing credentials
		// Either kmsKeyName or pgpPriKeyPath needs to be set
		if kmsKeyName == "" && pgpPriKeyPath == "" {
			exitOnBadFlags(SignerMode(mode), "neither kms_key_name or private_key is specified")
		}
		var cSigner attestlib.Signer
		if kmsKeyName != "" {
			glog.Infof("Using kms key %s for signing.", kmsKeyName)
			if kmsDigestAlg == "" {
				glog.Fatalf("kms_digest_alg is unspecified, must be one of SHA256|SHA384|SHA512, and the same as specified by the key version's algorithm")
			}
			cSigner, err = signer.NewCloudKmsSigner(kmsKeyName, signer.DigestAlgorithm(kmsDigestAlg))
			if err != nil {
				glog.Fatalf("Creating kms signer failed: %v\n", err)
			}
		} else {
			glog.Infof("Using pgp key for signing.")
			signerKey, err := ioutil.ReadFile(pgpPriKeyPath)
			if err != nil {
				glog.Fatalf("Fail to read signer key: %v\n", err)
			}
			// Create a cryptolib signer
			cSigner, err = attestlib.NewPgpSigner(signerKey, pgpPassphrase)
			if err != nil {
				glog.Fatalf("Creating pgp signer failed: %v\n", err)
			}
		}

		// Check note name
		err = util.CheckNoteName(noteName)
		if err != nil {
			exitOnBadFlags(SignerMode(mode), fmt.Sprintf("note name is invalid %v", err))
		}

		// Parse attestation project
		if attestationProject == "" {
			attestationProject = util.GetProjectFromContainerImage(image)
			glog.Infof("Using image project as attestation project: %s\n", attestationProject)
		} else {
			glog.Infof("Using specified attestation project: %s\n", attestationProject)
		}

		// Create signer
		r := signer.New(client, cSigner, noteName, attestationProject, overwrite)
		// Sign image
		err := r.SignImage(image)
		if err != nil {
			glog.Fatalf("Signing image failed %v", err)
		}
	}
}
<commit_msg>Fix test.<commit_after>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/grafeas/kritis/pkg/attestlib"
	"github.com/grafeas/kritis/pkg/kritis/apis/kritis/v1beta1"
	"github.com/grafeas/kritis/pkg/kritis/crd/vulnzsigningpolicy"
	"github.com/grafeas/kritis/pkg/kritis/metadata/containeranalysis"
	"github.com/grafeas/kritis/pkg/kritis/signer"
	"github.com/grafeas/kritis/pkg/kritis/util"
	"k8s.io/apimachinery/pkg/util/yaml"
)

type SignerMode string

const (
	CheckAndSign  SignerMode = "check-and-sign"
	CheckOnly     SignerMode = "check-only"
	BypassAndSign SignerMode = "bypass-and-sign"
)

var (
	// input flags
	mode               string
	image              string
	vulnzTimeout       string
	policyPath         string
	attestationProject string
	overwrite          bool
	noteName           string
	// input flags: pgp key flags
	pgpPriKeyPath string
	pgpPassphrase string
	// input flags: kms flags
	kmsKeyName   string
	kmsDigestAlg string

	// helper global variables
	modeFlags   *flag.FlagSet
	modeExample string
)

func init() {
	// need to add all flags to avoid "flag not provided error"
	addBasicFlags(flag.CommandLine)
	addCheckFlags(flag.CommandLine)
	addSignFlags(flag.CommandLine)
}

func addBasicFlags(fs *flag.FlagSet) {
	fs.StringVar(&mode, "mode", "check-and-sign", "(required) mode of operation, check-and-sign|check-only|bypass-and-sign")
	fs.StringVar(&image, "image", "", "(required) image url, e.g., gcr.io/foo/bar@sha256:abcd")
}

func addCheckFlags(fs *flag.FlagSet) {
	fs.StringVar(&policyPath, "policy", "", "(required for check) vulnerability signing policy file path, e.g., /tmp/vulnz_signing_policy.yaml")
	fs.StringVar(&vulnzTimeout, "vulnz_timeout", "5m", "timeout for polling image vulnerability , e.g., 600s, 5m")
}

func addSignFlags(fs *flag.FlagSet) {
	fs.StringVar(&noteName, "note_name", "", "(required for sign) note name that created attestations are attached to, in the form of projects/[PROVIDER_ID]/notes/[NOTE_ID]")
	fs.StringVar(&attestationProject, "attestation_project", "", "project id for GCP project that stores attestation, use image project if set to empty")
	fs.BoolVar(&overwrite, "overwrite", false, "overwrite attestation if already existed")
	fs.StringVar(&kmsKeyName, "kms_key_name", "", "kms key name, in the format of in the format projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*")
	fs.StringVar(&kmsDigestAlg, "kms_digest_alg", "", "kms digest algorithm, must be one of SHA256|SHA384|SHA512, and the same as specified by the key version's algorithm")
	fs.StringVar(&pgpPriKeyPath, "pgp_private_key", "", "pgp private signing key path, e.g., /dev/shm/key.pgp")
	fs.StringVar(&pgpPassphrase, "pgp_passphrase", "", "passphrase for pgp private key, if any")
}

// parseSignerMode creates mode-specific flagset and analyze actions (check, sign) for given mode
func parseSignerMode(mode SignerMode) (doCheck bool, doSign bool, err error) {
	modeFlags, doCheck, doSign, err = flag.NewFlagSet("", flag.ExitOnError), false, false, nil
	addBasicFlags(modeFlags)
	switch mode {
	case CheckAndSign:
		addCheckFlags(modeFlags)
		addSignFlags(modeFlags)
		doCheck, doSign = true, true
		modeExample = `	./signer \
	-mode=check-and-sign \
	-image=gcr.io/my-image-repo/image-1@sha256:123 \
	-policy=policy.yaml \
	-note_name=projects/$NOTE_PROJECT/NOTES/$NOTE_ID \
	-kms_key_name=projects/$KMS_PROJECT/locations/$KMS_KEYLOCATION/keyRings/$KMS_KEYRING/cryptoKeys/$KMS_KEYNAME/cryptoKeyVersions/$KMS_KEYVERSION \
	-kms_digest_alg=SHA512`
	case BypassAndSign:
		addSignFlags(modeFlags)
		doSign = true
		modeExample = `	./signer \
	-mode=bypass-and-sign \
	-image=gcr.io/my-image-repo/image-1@sha256:123 \
	-note_name=projects/$NOTE_PROJECT/NOTES/$NOTE_ID \
	-kms_key_name=projects/$KMS_PROJECT/locations/$KMS_KEYLOCATION/keyRings/$KMS_KEYRING/cryptoKeys/$KMS_KEYNAME/cryptoKeyVersions/$KMS_KEYVERSION \
	-kms_digest_alg=SHA512`
	case CheckOnly:
		addCheckFlags(modeFlags)
		doCheck = true
		modeExample = `	./signer \
	-mode=check-only \
	-image=gcr.io/my-image-repo/image-1@sha256:123 \
	-policy=policy.yaml`
	default:
		return false, false, fmt.Errorf("unrecognized mode %s, must be one of check-and-sign|check-only|bypass-and-sign", mode)
	}
	flag.Parse()
	return doCheck, doSign, err
}

func exitOnBadFlags(mode SignerMode, err string) {
	fmt.Fprintf(modeFlags.Output(), "Usage of signer's %s mode:\n", mode)
	modeFlags.PrintDefaults()
	fmt.Fprintf(modeFlags.Output(), "Example (%s mode):\n %s\n", mode, modeExample)
	fmt.Fprintf(modeFlags.Output(), "Bad flags for mode %s: %v. \n", mode, err)
	os.Exit(1)
}

func main() {
	flag.Parse()
	glog.Infof("Signer mode: %s.", mode)

	doCheck, doSign, err := parseSignerMode(SignerMode(mode))
	if err != nil {
		glog.Fatalf("Parse mode err %v.", err)
	}

	// Check image url is non-empty
	// TODO: check and format image url to
	//  gcr.io/project-id/rest-of-image-path@sha256:[sha-value]
	if image == "" {
		exitOnBadFlags(SignerMode(mode), "image url is empty")
	}

	// Create a client
	client, err := containeranalysis.New()
	if err != nil {
		glog.Fatalf("Could not initialize the client %v", err)
	}

	if doCheck {
		// Read the vulnz signing policy
		if policyPath == "" {
			exitOnBadFlags(SignerMode(mode), "policy path is empty")
		}
		policy := v1beta1.VulnzSigningPolicy{}
		policyFile, err := os.Open(policyPath)
		if err != nil {
			glog.Fatalf("Fail to load vulnz signing policy: %v", err)
		}
		defer policyFile.Close()
		// err = json.Unmarshal(policyFile, &policy)
		if err := yaml.NewYAMLToJSONDecoder(policyFile).Decode(&policy); err != nil {
			glog.Fatalf("Fail to parse policy file: %v", err)
		} else {
			glog.Infof("Policy req: %v\n", policy.Spec.ImageVulnerabilityRequirements)
		}

		timeout, err := time.ParseDuration(vulnzTimeout)
		if err != nil {
			glog.Fatalf("Fail to parse timeout %v", err)
		}
		err = client.WaitForVulnzAnalysis(image, timeout)
		if err != nil {
			glog.Fatalf("Error waiting for vulnerability analysis %v", err)
		}

		// Read the vulnz scanning events
		vulnz, err := client.Vulnerabilities(image)
		if err != nil {
			glog.Fatalf("Found err %s", err)
		}
		if vulnz == nil {
			glog.Fatalf("Expected some vulnerabilities. Nil found")
		}

		violations, err := vulnzsigningpolicy.ValidateVulnzSigningPolicy(policy, image, vulnz)
		if err != nil {
			glog.Fatalf("Error when evaluating image %q against policy %q", image, policy.Name)
		}
		if violations != nil && len(violations) != 0 {
			glog.Errorf("Image %q does not pass VulnzSigningPolicy %q:", image, policy.Name)
			glog.Errorf("Found %d violations in image %s:", len(violations), image)
			for _, v := range violations {
				glog.Error(v.Reason())
			}
			os.Exit(1)
		}
		glog.Infof("Image %q passes VulnzSigningPolicy %s.", image, policy.Name)
	}

	if doSign {
		// Read the signing credentials
		// Either kmsKeyName or pgpPriKeyPath needs to be set
		if kmsKeyName == "" && pgpPriKeyPath == "" {
			exitOnBadFlags(SignerMode(mode), "neither kms_key_name or private_key is specified")
		}
		var cSigner attestlib.Signer
		if kmsKeyName != "" {
			glog.Infof("Using kms key %s for signing.", kmsKeyName)
			if kmsDigestAlg == "" {
				glog.Fatalf("kms_digest_alg is unspecified, must be one of SHA256|SHA384|SHA512, and the same as specified by the key version's algorithm")
			}
			cSigner, err = signer.NewCloudKmsSigner(kmsKeyName, signer.DigestAlgorithm(kmsDigestAlg))
			if err != nil {
				glog.Fatalf("Creating kms signer failed: %v\n", err)
			}
		} else {
			glog.Infof("Using pgp key for signing.")
			signerKey, err := ioutil.ReadFile(pgpPriKeyPath)
			if err != nil {
				glog.Fatalf("Fail to read signer key: %v\n", err)
			}
			// Create a cryptolib signer
			cSigner, err = attestlib.NewPgpSigner(signerKey, pgpPassphrase)
			if err != nil {
				glog.Fatalf("Creating pgp signer failed: %v\n", err)
			}
		}

		// Check note name
		err = util.CheckNoteName(noteName)
		if err != nil {
			exitOnBadFlags(SignerMode(mode), fmt.Sprintf("note name is invalid %v", err))
		}

		// Parse attestation project
		if attestationProject == "" {
			attestationProject = util.GetProjectFromContainerImage(image)
			glog.Infof("Using image project as attestation project: %s\n", attestationProject)
		} else {
			glog.Infof("Using specified attestation project: %s\n", attestationProject)
		}

		// Create signer
		r := signer.New(client, cSigner, noteName, attestationProject, overwrite)
		// Sign image
		err := r.SignImage(image)
		if err != nil {
			glog.Fatalf("Signing image failed %v", err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mesh

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"

	"istio.io/operator/pkg/object"
	"istio.io/pkg/log"
)

// YAMLSuffix is the suffix of a YAML file.
const YAMLSuffix = ".yaml"

type manifestDiffArgs struct {
	// compareDir indicates comparison between directory.
	compareDir bool
}

func addManifestDiffFlags(cmd *cobra.Command, diffArgs *manifestDiffArgs) {
	cmd.PersistentFlags().BoolVarP(&diffArgs.compareDir, "directory", "r",
		false, "compare directory")
}

func manifestDiffCmd(rootArgs *rootArgs, diffArgs *manifestDiffArgs) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "diff",
		Short: "Compare manifests and generate diff.",
		Long:  "The diff-manifest subcommand is used to compare manifest from two files or directories.",
		Args:  cobra.ExactArgs(2),
		Run: func(cmd *cobra.Command, args []string) {
			if diffArgs.compareDir {
				compareManifestsFromDirs(args[0], args[1])
			} else {
				compareManifestsFromFiles(rootArgs, args)
			}
		}}
	return cmd
}

//compareManifestsFromFiles compares two manifest files
func compareManifestsFromFiles(rootArgs *rootArgs, args []string) {
	if err := configLogs(rootArgs); err != nil {
		_, _ = fmt.Fprintf(os.Stderr, "Could not configure logs: %s", err)
		os.Exit(1)
	}
	a, err := ioutil.ReadFile(args[0])
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	b, err := ioutil.ReadFile(args[1])
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	diff, err := object.ManifestDiff(string(a), string(b))
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	if diff == "" {
		fmt.Println("Manifests are identical")
	} else {
		fmt.Printf("Difference of manifests are:\n%s", diff)
	}
}

func readFromDir(dirName string) (string, error) {
	var fileList []string
	err := filepath.Walk(dirName, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() || filepath.Ext(path) != YAMLSuffix {
			return nil
		}
		fileList = append(fileList, path)
		return nil
	})
	if err != nil {
		return "", err
	}
	var sb strings.Builder
	for _, file := range fileList {
		a, err := ioutil.ReadFile(file)
		if err != nil {
			return "", err
		}
		if _, err := sb.WriteString(string(a)); err != nil {
			return "", err
		}
	}
	return sb.String(), nil
}

//compareManifestsFromDirs compares manifests from two directories
func compareManifestsFromDirs(dirName1 string, dirName2 string) {
	mf1, err := readFromDir(dirName1)
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	mf2, err := readFromDir(dirName2)
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	diff, err := object.ManifestDiff(mf1, mf2)
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	if diff == "" {
		fmt.Println("Manifests are identical")
	} else {
		fmt.Printf("Difference of manifests are:\n%s", diff)
	}
}
<commit_msg>Fix return code of manifest-diff (#145)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mesh

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"

	"istio.io/operator/pkg/object"
	"istio.io/pkg/log"
)

// YAMLSuffix is the suffix of a YAML file.
const YAMLSuffix = ".yaml"

type manifestDiffArgs struct {
	// compareDir indicates comparison between directory.
	compareDir bool
}

func addManifestDiffFlags(cmd *cobra.Command, diffArgs *manifestDiffArgs) {
	cmd.PersistentFlags().BoolVarP(&diffArgs.compareDir, "directory", "r",
		false, "compare directory")
}

func manifestDiffCmd(rootArgs *rootArgs, diffArgs *manifestDiffArgs) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "diff",
		Short: "Compare manifests and generate diff.",
		Long:  "The diff-manifest subcommand is used to compare manifest from two files or directories.",
		Args:  cobra.ExactArgs(2),
		Run: func(cmd *cobra.Command, args []string) {
			if diffArgs.compareDir {
				compareManifestsFromDirs(rootArgs, args[0], args[1])
			} else {
				compareManifestsFromFiles(rootArgs, args)
			}
		}}
	return cmd
}

//compareManifestsFromFiles compares two manifest files
func compareManifestsFromFiles(rootArgs *rootArgs, args []string) {
	checkLogsOrExit(rootArgs)

	a, err := ioutil.ReadFile(args[0])
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	b, err := ioutil.ReadFile(args[1])
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	diff, err := object.ManifestDiff(string(a), string(b))
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	if diff == "" {
		fmt.Println("Manifests are identical")
	} else {
		fmt.Printf("Difference of manifests are:\n%s", diff)
		os.Exit(1)
	}
}

func readFromDir(dirName string) (string, error) {
	var fileList []string
	err := filepath.Walk(dirName, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() || filepath.Ext(path) != YAMLSuffix {
			return nil
		}
		fileList = append(fileList, path)
		return nil
	})
	if err != nil {
		return "", err
	}
	var sb strings.Builder
	for _, file := range fileList {
		a, err := ioutil.ReadFile(file)
		if err != nil {
			return "", err
		}
		if _, err := sb.WriteString(string(a)); err != nil {
			return "", err
		}
	}
	return sb.String(), nil
}

//compareManifestsFromDirs compares manifests from two directories
func compareManifestsFromDirs(rootArgs *rootArgs, dirName1 string, dirName2 string) {
	checkLogsOrExit(rootArgs)

	mf1, err := readFromDir(dirName1)
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	mf2, err := readFromDir(dirName2)
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	diff, err := object.ManifestDiff(mf1, mf2)
	if err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
	if diff == "" {
		fmt.Println("Manifests are identical")
	} else {
		fmt.Printf("Difference of manifests are:\n%s", diff)
	}
}
<|endoftext|>


<commit_before>//
// Copyright 2019 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package server

import (
	"context"
	"testing"

	"google3/third_party/golang/grpctunnel/tunnel/tunnel"
)

func TestListen(t *testing.T) {
	for _, test := range []struct {
		name, lAddr string
	}{
		{
			name:  "invalidListenAddress",
			lAddr: "invalid:999999999",
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			if err := listen(context.Background(), nil, test.lAddr, &[]tunnel.Target{}); err == nil {
				t.Fatalf("listen() got success, want error")
			}
		})
	}
}

func TestRun(t *testing.T) {
	for _, test := range []struct {
		name, lAddr, tAddr, certFile, keyFile string
	}{
		{
			name:     "InvalidCertFile",
			certFile: "does/Not\\Exist//",
			keyFile:  "does/Not\\Exist//",
		},
		{
			name:  "invalidTunnelAddress",
			lAddr: "invalid:999999999",
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			conf := Config{
				TunnelAddress: test.tAddr,
				ListenAddress: test.lAddr,
				CertFile:      test.certFile,
				KeyFile:       test.keyFile,
			}
			if err := Run(context.Background(), conf); err == nil {
				t.Fatalf("Run() got success, want error")
			}
		})
	}
}
<commit_msg>fix import<commit_after>//
// Copyright 2019 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package server

import (
	"context"
	"testing"

	"github.com/openconfig/grpctunnel/tunnel"
)

func TestListen(t *testing.T) {
	for _, test := range []struct {
		name, lAddr string
	}{
		{
			name:  "invalidListenAddress",
			lAddr: "invalid:999999999",
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			if err := listen(context.Background(), nil, test.lAddr, &[]tunnel.Target{}); err == nil {
				t.Fatalf("listen() got success, want error")
			}
		})
	}
}

func TestRun(t *testing.T) {
	for _, test := range []struct {
		name, lAddr, tAddr, certFile, keyFile string
	}{
		{
			name:     "InvalidCertFile",
			certFile: "does/Not\\Exist//",
			keyFile:  "does/Not\\Exist//",
		},
		{
			name:  "invalidTunnelAddress",
			lAddr: "invalid:999999999",
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			conf := Config{
				TunnelAddress: test.tAddr,
				ListenAddress: test.lAddr,
				CertFile:      test.certFile,
				KeyFile:       test.keyFile,
			}
			if err := Run(context.Background(), conf); err == nil {
				t.Fatalf("Run() got success, want error")
			}
		})
	}
}
<|endoftext|>


<commit_before>package sudoku

import (
	"math"
	"math/rand"
)

//ProbabilityDistribution represents a distribution of probabilities over
//indexes.
type ProbabilityDistribution []float64

//probabiliyDistributionTweak represents tweaks to make to a
//ProbabilityDistribution via tweak(). 1.0 is no effect.
type probabilityDistributionTweak []float64

//Normalized returns true if the distribution is normalized: that is, the
//distribution sums to 1.0
func (d ProbabilityDistribution) normalized() bool {
	var sum float64
	for _, weight := range d {
		if weight < 0 {
			return false
		}
		sum += weight
	}
	return sum == 1.0
}

//Normalize returns a new probability distribution based on this one that is
//normalized.
func (d ProbabilityDistribution) normalize() ProbabilityDistribution {
	if d.normalized() {
		return d
	}
	var sum float64

	fixedWeights := d.denegativize()

	for _, weight := range fixedWeights {
		sum += weight
	}

	result := make(ProbabilityDistribution, len(d))
	for i, weight := range fixedWeights {
		result[i] = weight / sum
	}
	return result
}

//Denegativize returns a new probability distribution like this one, but with
//no negatives. If a negative is found, the entire distribution is shifted up
//by that amount.
func (d ProbabilityDistribution) denegativize() ProbabilityDistribution {
	var lowestNegative float64

	//Check for a negative.
	for _, weight := range d {
		if weight < 0 {
			if weight < lowestNegative {
				lowestNegative = weight
			}
		}
	}

	fixedWeights := make(ProbabilityDistribution, len(d))

	copy(fixedWeights, d)

	if lowestNegative != 0 {
		//Found a negative. Need to fix up all weights.
		for i := 0; i < len(d); i++ {
			fixedWeights[i] = d[i] - lowestNegative
		}

	}

	return fixedWeights
}

//tweak takes an amount to tweak each probability and returns a new
//ProbabilityDistribution where each probability is multiplied by tweak and
//the entire distribution is normalized. Useful for strengthening some
//probabilities and reducing others.
func (d ProbabilityDistribution) tweak(tweak probabilityDistributionTweak) ProbabilityDistribution {
	//sanity check that the tweak amounts are same length
	if len(d) != len(tweak) {
		return d
	}
	result := make(ProbabilityDistribution, len(d))

	for i, num := range d {
		result[i] = num * tweak[i]
	}

	return result.normalize()
}

//invert returns a new probability distribution like this one, but "flipped"
//so low values have a high chance of occurring and high values have a low
//chance. The curve used to invert is expoential.
func (d ProbabilityDistribution) invert() ProbabilityDistribution {
	//TODO: this function means that the worst weighted item will have a weight of 0.0. Isn't that wrong? Maybe it should be +1 to everythign?
	weights := make(ProbabilityDistribution, len(d))

	invertedWeights := d.denegativize()

	//This inversion isn't really an inversion, and it's tied closely to the shape of the weightings we expect to be given in the sudoku problem domain.

	//In sudoku, the weights for different techniques go up exponentially. So when we invert, we want to see a similar exponential shape of the
	//flipped values.

	//We flip with 1/x, and the bottom is an exponent, but softened some.

	//I don't know if this math makes any sense, but in the test distributions the outputs FEEL right.

	//Invert
	for i, weight := range invertedWeights {
		weights[i] = 1 / math.Exp(weight/20)
	}

	//But now you need to renormalize since they won't sum to 1.
	return weights.normalize()
}

//RandomIndex returns a random index based on the probability distribution.
func (d ProbabilityDistribution) RandomIndex() int {
	weightsToUse := d
	if !d.normalized() {
		weightsToUse = d.normalize()
	}
	sample := rand.Float64()
	var counter float64
	for i, weight := range weightsToUse {
		counter += weight
		if sample <= counter {
			return i
		}
	}
	//This shouldn't happen if the weights are properly normalized.
	return len(weightsToUse) - 1
}
<commit_msg>Factored out the invertWeight logic from ProbabilityDistribution so I have it availabile outside of probabily distribution.<commit_after>package sudoku

import (
	"math"
	"math/rand"
)

//ProbabilityDistribution represents a distribution of probabilities over
//indexes.
type ProbabilityDistribution []float64

//probabiliyDistributionTweak represents tweaks to make to a
//ProbabilityDistribution via tweak(). 1.0 is no effect.
type probabilityDistributionTweak []float64

//Normalized returns true if the distribution is normalized: that is, the
//distribution sums to 1.0
func (d ProbabilityDistribution) normalized() bool {
	var sum float64
	for _, weight := range d {
		if weight < 0 {
			return false
		}
		sum += weight
	}
	return sum == 1.0
}

//Normalize returns a new probability distribution based on this one that is
//normalized.
func (d ProbabilityDistribution) normalize() ProbabilityDistribution {
	if d.normalized() {
		return d
	}
	var sum float64

	fixedWeights := d.denegativize()

	for _, weight := range fixedWeights {
		sum += weight
	}

	result := make(ProbabilityDistribution, len(d))
	for i, weight := range fixedWeights {
		result[i] = weight / sum
	}
	return result
}

//Denegativize returns a new probability distribution like this one, but with
//no negatives. If a negative is found, the entire distribution is shifted up
//by that amount.
func (d ProbabilityDistribution) denegativize() ProbabilityDistribution {
	var lowestNegative float64

	//Check for a negative.
	for _, weight := range d {
		if weight < 0 {
			if weight < lowestNegative {
				lowestNegative = weight
			}
		}
	}

	fixedWeights := make(ProbabilityDistribution, len(d))

	copy(fixedWeights, d)

	if lowestNegative != 0 {
		//Found a negative. Need to fix up all weights.
		for i := 0; i < len(d); i++ {
			fixedWeights[i] = d[i] - lowestNegative
		}

	}

	return fixedWeights
}

//tweak takes an amount to tweak each probability and returns a new
//ProbabilityDistribution where each probability is multiplied by tweak and
//the entire distribution is normalized. Useful for strengthening some
//probabilities and reducing others.
func (d ProbabilityDistribution) tweak(tweak probabilityDistributionTweak) ProbabilityDistribution {
	//sanity check that the tweak amounts are same length
	if len(d) != len(tweak) {
		return d
	}
	result := make(ProbabilityDistribution, len(d))

	for i, num := range d {
		result[i] = num * tweak[i]
	}

	return result.normalize()
}

//invert returns a new probability distribution like this one, but "flipped"
//so low values have a high chance of occurring and high values have a low
//chance. The curve used to invert is expoential.
func (d ProbabilityDistribution) invert() ProbabilityDistribution {
	//TODO: this function means that the worst weighted item will have a weight of 0.0. Isn't that wrong? Maybe it should be +1 to everythign?
	weights := make(ProbabilityDistribution, len(d))

	invertedWeights := d.denegativize()

	//This inversion isn't really an inversion, and it's tied closely to the shape of the weightings we expect to be given in the sudoku problem domain.

	//In sudoku, the weights for different techniques go up exponentially. So when we invert, we want to see a similar exponential shape of the
	//flipped values.

	//We flip with 1/x, and the bottom is an exponent, but softened some.

	//I don't know if this math makes any sense, but in the test distributions the outputs FEEL right.

	//Invert
	for i, weight := range invertedWeights {
		weights[i] = invertWeight(weight)
	}

	//But now you need to renormalize since they won't sum to 1.
	return weights.normalize()
}

//invertWeight is the primary logic used to invert a positive weight.
func invertWeight(inverted float64) float64 {
	return 1 / math.Exp(inverted/20)
}

//RandomIndex returns a random index based on the probability distribution.
func (d ProbabilityDistribution) RandomIndex() int {
	weightsToUse := d
	if !d.normalized() {
		weightsToUse = d.normalize()
	}
	sample := rand.Float64()
	var counter float64
	for i, weight := range weightsToUse {
		counter += weight
		if sample <= counter {
			return i
		}
	}
	//This shouldn't happen if the weights are properly normalized.
	return len(weightsToUse) - 1
}
<|endoftext|>


<commit_before>package proc

import (
	"fmt"
	"net"
	"os/exec"
	"strconv"
	"strings"
)

type reader struct{}

// NewReader returns a Darwin (lsof-based) '/proc' reader
func NewReader(proc Dir) Reader {
	return &reader{}
}

const (
	lsofBinary    = "lsof"
	lsofFields    = "cn" // parseLSOF() depends on the order
	netstatBinary = "netstat"
	lsofBinary    = "lsof"
)

func (reader) Processes(f func(Process)) error {
	output, err := exec.Command(
		lsofBinary,
		"-i",       // only Internet files
		"-n", "-P", // no number resolving
		"-w",             // no warnings
		"-F", lsofFields, // \n based output of only the fields we want.
	).CombinedOutput()
	if err != nil {
		return err
	}

	processes, err := parseLSOF(string(output))
	if err != nil {
		return err
	}

	for _, process := range processes {
		f(process)
	}
	return nil
}

func (r *reader) Connections(withProcs bool, f func(Connection)) error {
	out, err := exec.Command(
		netstatBinary,
		"-n", // no number resolving
		"-W", // Wide output
		// "-l", // full IPv6 addresses // What does this do?
		"-p", "tcp", // only TCP
	).CombinedOutput()
	if err != nil {
		return err
	}
	connections := parseDarwinNetstat(string(out))

	if withProcs {
		out, err := exec.Command(
			lsofBinary,
			"-i",       // only Internet files
			"-n", "-P", // no number resolving
			"-w",             // no warnings
			"-F", lsofFields, // \n based output of only the fields we want.
		).CombinedOutput()
		if err != nil {
			return err
		}

		procs, err := parseLSOF(string(out))
		if err != nil {
			return err
		}
		for local, proc := range procs {
			for i, c := range connections {
				localAddr := net.JoinHostPort(
					c.LocalAddress.String(),
					strconv.Itoa(int(c.LocalPort)),
				)
				if localAddr == local {
					connections[i].Proc = proc
				}
			}
		}
	}

	for _, c := range connections {
		f(c)
	}
	return nil
}

// Close closes the Darwin "/proc" reader
func (reader) Close() error {
	return nil
}

func parseLSOF(output string) (map[string]Process, error) {
	var (
		processes = map[string]Process{} // Local addr -> Proc
		process   Process
	)
	for _, line := range strings.Split(output, "\n") {
		if len(line) <= 1 {
			continue
		}

		var (
			field = line[0]
			value = line[1:]
		)
		switch field {
		case 'p':
			pid, err := strconv.Atoi(value)
			if err != nil {
				return nil, fmt.Errorf("invalid 'p' field in lsof output: %#v", value)
			}
			process.PID = pid

		case 'c':
			process.Comm = value

		case 'n':
			// 'n' is the last field, with '-F cn'
			// format examples:
			// "192.168.2.111:44013->54.229.241.196:80"
			// "[2003:45:2b57:8900:1869:2947:f942:aba7]:55711->[2a00:1450:4008:c01::11]:443"
			// "*:111" <- a listen
			addresses := strings.SplitN(value, "->", 2)
			if len(addresses) != 2 {
				// That's a listen entry.
				continue
			}
			processes[addresses[0]] = Process{
				PID:  process.PID,
				Comm: process.Comm,
			}

		default:
			return nil, fmt.Errorf("unexpected lsof field: %c in %#v", field, value)
		}
	}
	return processes, nil
}
<commit_msg>Fix bad merge on Darwin<commit_after>package proc

import (
	"fmt"
	"net"
	"os/exec"
	"strconv"
	"strings"
)

type reader struct{}

// NewReader returns a Darwin (lsof-based) '/proc' reader
func NewReader(proc Dir) Reader {
	return &reader{}
}

const (
	lsofBinary    = "lsof"
	lsofFields    = "cn" // parseLSOF() depends on the order
	netstatBinary = "netstat"
)

func (reader) Processes(f func(Process)) error {
	output, err := exec.Command(
		lsofBinary,
		"-i",       // only Internet files
		"-n", "-P", // no number resolving
		"-w",             // no warnings
		"-F", lsofFields, // \n based output of only the fields we want.
	).CombinedOutput()
	if err != nil {
		return err
	}

	processes, err := parseLSOF(string(output))
	if err != nil {
		return err
	}

	for _, process := range processes {
		f(process)
	}
	return nil
}

func (r *reader) Connections(withProcs bool, f func(Connection)) error {
	out, err := exec.Command(
		netstatBinary,
		"-n", // no number resolving
		"-W", // Wide output
		// "-l", // full IPv6 addresses // What does this do?
		"-p", "tcp", // only TCP
	).CombinedOutput()
	if err != nil {
		return err
	}
	connections := parseDarwinNetstat(string(out))

	if withProcs {
		out, err := exec.Command(
			lsofBinary,
			"-i",       // only Internet files
			"-n", "-P", // no number resolving
			"-w",             // no warnings
			"-F", lsofFields, // \n based output of only the fields we want.
		).CombinedOutput()
		if err != nil {
			return err
		}

		procs, err := parseLSOF(string(out))
		if err != nil {
			return err
		}
		for local, proc := range procs {
			for i, c := range connections {
				localAddr := net.JoinHostPort(
					c.LocalAddress.String(),
					strconv.Itoa(int(c.LocalPort)),
				)
				if localAddr == local {
					connections[i].Process = proc
				}
			}
		}
	}

	for _, c := range connections {
		f(c)
	}
	return nil
}

// Close closes the Darwin "/proc" reader
func (reader) Close() error {
	return nil
}

func parseLSOF(output string) (map[string]Process, error) {
	var (
		processes = map[string]Process{} // Local addr -> Proc
		process   Process
	)
	for _, line := range strings.Split(output, "\n") {
		if len(line) <= 1 {
			continue
		}

		var (
			field = line[0]
			value = line[1:]
		)
		switch field {
		case 'p':
			pid, err := strconv.Atoi(value)
			if err != nil {
				return nil, fmt.Errorf("invalid 'p' field in lsof output: %#v", value)
			}
			process.PID = pid

		case 'c':
			process.Comm = value

		case 'n':
			// 'n' is the last field, with '-F cn'
			// format examples:
			// "192.168.2.111:44013->54.229.241.196:80"
			// "[2003:45:2b57:8900:1869:2947:f942:aba7]:55711->[2a00:1450:4008:c01::11]:443"
			// "*:111" <- a listen
			addresses := strings.SplitN(value, "->", 2)
			if len(addresses) != 2 {
				// That's a listen entry.
				continue
			}
			processes[addresses[0]] = Process{
				PID:  process.PID,
				Comm: process.Comm,
			}

		default:
			return nil, fmt.Errorf("unexpected lsof field: %c in %#v", field, value)
		}
	}
	return processes, nil
}
<|endoftext|>


<commit_before>package kafkamdm

import (
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"
)

func TestLagLogger(t *testing.T) {
	logger := newLagLogger(5)
	Convey("with 0 measurements", t, func() {
		So(logger.Min(), ShouldEqual, 0)
	})
	Convey("with 1 measurements", t, func() {
		logger.Store(10)
		So(logger.Min(), ShouldEqual, 10)
	})
	Convey("with 2 measurements", t, func() {
		logger.Store(5)
		So(logger.Min(), ShouldEqual, 5)
	})
	Convey("with lots of measurements", t, func() {
		for i := 0; i < 100; i++ {
			logger.Store(i)
		}
		So(logger.Min(), ShouldEqual, 95)
	})
}

func TestRateLogger(t *testing.T) {
	logger := newRateLogger()
	now := time.Now()
	Convey("with 0 measurements", t, func() {
		So(logger.Rate(), ShouldEqual, 0)
	})
	Convey("after 1st measurements", t, func() {
		logger.Store(10, now)
		So(logger.Rate(), ShouldEqual, 0)
	})
	Convey("with 2nd measurements", t, func() {
		logger.Store(15, now.Add(time.Second))
		So(logger.Rate(), ShouldEqual, 5)
	})
	Convey("with old ts", t, func() {
		logger.Store(25, now)
		So(logger.Rate(), ShouldEqual, 5)
	})
	Convey("with less then 1per second", t, func() {
		logger.Store(30, now.Add(time.Second*10))
		So(logger.Rate(), ShouldEqual, 0)
	})
}

func TestLagMonitor(t *testing.T) {
	mon := NewLagMonitor(10, []int32{0, 1, 2, 3})
	Convey("with 0 measurements", t, func() {
		So(mon.Metric(), ShouldEqual, 0)
	})
	Convey("with lots of measurements", t, func() {
		now := time.Now()
		for part := range mon.lag {
			for i := 0; i < 100; i++ {
				mon.StoreLag(part, i)
				mon.StoreOffset(part, int64(i), now.Add(time.Second*time.Duration(i)))
			}
		}
		So(mon.Metric(), ShouldEqual, 90)
	})
	Convey("metric should be worst partition", t, func() {
		for part := range mon.lag {
			mon.StoreLag(part, 10+int(part))
		}
		So(mon.Metric(), ShouldEqual, 13)
	})
}
<commit_msg>add failing test<commit_after>package kafkamdm

import (
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"
)

func TestLagLogger(t *testing.T) {
	logger := newLagLogger(5)
	Convey("with 0 measurements", t, func() {
		So(logger.Min(), ShouldEqual, 0)
	})
	Convey("with 1 measurements", t, func() {
		logger.Store(10)
		So(logger.Min(), ShouldEqual, 10)
	})
	Convey("with 2 measurements", t, func() {
		logger.Store(5)
		So(logger.Min(), ShouldEqual, 5)
	})
	Convey("with lots of measurements", t, func() {
		for i := 0; i < 100; i++ {
			logger.Store(i)
		}
		So(logger.Min(), ShouldEqual, 95)
	})
}

func TestRateLogger(t *testing.T) {
	logger := newRateLogger()
	now := time.Now()
	Convey("with 0 measurements", t, func() {
		So(logger.Rate(), ShouldEqual, 0)
	})
	Convey("after 1st measurements", t, func() {
		logger.Store(10, now)
		So(logger.Rate(), ShouldEqual, 0)
	})
	Convey("with 2nd measurements", t, func() {
		logger.Store(15, now.Add(time.Second))
		So(logger.Rate(), ShouldEqual, 5)
	})
	Convey("with old ts", t, func() {
		logger.Store(25, now)
		So(logger.Rate(), ShouldEqual, 5)
	})

	//	metrics 15 duration 9s rate is 1

	Convey("with less then 1per second", t, func() {
		logger.Store(30, now.Add(time.Second*10))
		So(logger.Rate(), ShouldEqual, 0)
	})
}

func TestRateLoggerSmallIncrements(t *testing.T) {
	logger := newRateLogger()
	now := time.Now()
	Convey("after 1st measurements", t, func() {
		logger.Store(10, now)
		So(logger.Rate(), ShouldEqual, 0)
	})
	Convey("with 2nd measurements", t, func() {
		logger.Store(20, now.Add(200*time.Millisecond))
		So(logger.Rate(), ShouldEqual, 0)
	})
	Convey("with 3rd measurements", t, func() {
		logger.Store(30, now.Add(400*time.Millisecond))
		So(logger.Rate(), ShouldEqual, 0)
	})
	Convey("with 4th measurements", t, func() {
		logger.Store(40, now.Add(600*time.Millisecond))
		So(logger.Rate(), ShouldEqual, 0)
	})
	Convey("with 5th measurements", t, func() {
		logger.Store(50, now.Add(800*time.Millisecond))
		So(logger.Rate(), ShouldEqual, 0)
	})
	Convey("with 6th measurements", t, func() {
		logger.Store(60, now.Add(1000*time.Millisecond))
		So(logger.Rate(), ShouldEqual, 60-10)
	})
	Convey("with 7th measurements", t, func() {
		logger.Store(80, now.Add(1200*time.Millisecond))
		So(logger.Rate(), ShouldEqual, 60-10)
	})
	Convey("with 8th measurements", t, func() {
		logger.Store(100, now.Add(1400*time.Millisecond))
		So(logger.Rate(), ShouldEqual, 60-10)
	})
	Convey("with 9th measurements", t, func() {
		logger.Store(120, now.Add(1600*time.Millisecond))
		So(logger.Rate(), ShouldEqual, 60-10)
	})
	Convey("with 10th measurements", t, func() {
		logger.Store(140, now.Add(1800*time.Millisecond))
		So(logger.Rate(), ShouldEqual, 60-10)
	})
	Convey("with 11th measurements", t, func() {
		logger.Store(160, now.Add(2000*time.Millisecond))
		So(logger.Rate(), ShouldEqual, 160-60)
	})
}

func TestLagMonitor(t *testing.T) {
	mon := NewLagMonitor(10, []int32{0, 1, 2, 3})
	Convey("with 0 measurements", t, func() {
		So(mon.Metric(), ShouldEqual, 0)
	})
	Convey("with lots of measurements", t, func() {
		now := time.Now()
		for part := range mon.lag {
			for i := 0; i < 100; i++ {
				mon.StoreLag(part, i)
				mon.StoreOffset(part, int64(i), now.Add(time.Second*time.Duration(i)))
			}
		}
		So(mon.Metric(), ShouldEqual, 90)
	})
	Convey("metric should be worst partition", t, func() {
		for part := range mon.lag {
			mon.StoreLag(part, 10+int(part))
		}
		So(mon.Metric(), ShouldEqual, 13)
	})
}
<|endoftext|>