<commit_before>package metrics

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/mackerelio/golib/logging"
	"github.com/mackerelio/mackerel-agent/config"
	"github.com/mackerelio/mackerel-agent/mackerel"
)

// pluginGenerator collects user-defined metrics.
// mackerel-agent runs specified command and parses the result for the metric names and values.
type pluginGenerator struct {
	Config *config.MetricPlugin
	Meta   *pluginMeta
}

// pluginMeta is generated from plugin command. (not the configuration file)
type pluginMeta struct {
	Graphs map[string]customGraphDef
}

type customGraphDef struct {
	Label   string
	Unit    string
	Metrics []customGraphMetricDef
}

type customGraphMetricDef struct {
	Name    string
	Label   string
	Stacked bool
}

var pluginLogger = logging.GetLogger("metrics.plugin")

const pluginPrefix = "custom."

var pluginConfigurationEnvName = "MACKEREL_AGENT_PLUGIN_META"

// NewPluginGenerator XXX
func NewPluginGenerator(conf *config.MetricPlugin) PluginGenerator {
	return &pluginGenerator{Config: conf}
}

func (g *pluginGenerator) Generate() (Values, error) {
	results, err := g.collectValues()
	if err != nil {
		return nil, err
	}
	return results, nil
}

func (g *pluginGenerator) PrepareGraphDefs() ([]mackerel.CreateGraphDefsPayload, error) {
	err := g.loadPluginMeta()
	if err != nil {
		return nil, err
	}

	payload := g.makeCreateGraphDefsPayload()
	return payload, nil
}

func (g *pluginGenerator) CustomIdentifier() *string {
	return g.Config.CustomIdentifier
}

var pluginMetaHeadlineReg = regexp.MustCompile(`^#\s*mackerel-agent-plugin\b(.*)`)

// loadPluginMeta obtains plugin information (e.g. graph visuals, metric
// namespaces, etc) from the command specified.
// mackerel-agent runs the command with MACKEREL_AGENT_PLUGIN_META
// environment variable set.  The command is supposed to output like below:
//
// 	# mackerel-agent-plugin
// 	{
// 	  "graphs": {
// 	    GRAPH_NAME: {
// 	      "label": GRAPH_LABEL,
// 	      "unit": UNIT_TYPE
// 	      "metrics": [
// 	        {
// 	          "name": METRIC_NAME,
// 	          "label": METRIC_LABEL
// 	        },
// 	        ...
// 	      ]
// 	    },
// 	    GRAPH_NAME: ...
// 	  }
// 	}
//
// Valid UNIT_TYPEs are: "float", "integer", "percentage", "bytes", "bytes/sec", "iops"
//
// The output should start with a line beginning with '#', which contains
// meta-info of the configuration. (eg. plugin schema version)
//
// Below is a working example where the plugin emits metrics named "dice.d6" and "dice.d20":
//
// 	{
// 	  "graphs": {
// 	    "dice": {
// 	      "metrics": [
// 	        {
// 	          "name": "d6",
// 	          "label": "Die (d6)"
// 	        },
// 	        {
// 	          "name": "d20",
// 	          "label": "Die (d20)"
// 	        }
// 	      ],
// 	      "unit": "integer",
// 	      "label": "My Dice"
// 	    }
// 	  }
// 	}
func (g *pluginGenerator) loadPluginMeta() error {
	// Set environment variable to make the plugin command generate its configuration
	os.Setenv(pluginConfigurationEnvName, "1")
	defer os.Setenv(pluginConfigurationEnvName, "")

	stdout, stderr, exitCode, err := g.Config.Command.Run()
	if err != nil {
		return fmt.Errorf("running %s failed: %s, exit=%d stderr=%q", g.Config.Command.CommandString(), err, exitCode, stderr)
	}

	outBuffer := bufio.NewReader(strings.NewReader(stdout))
	// Read the plugin configuration meta (version etc)

	headerLine, err := outBuffer.ReadString('\n')
	if err != nil {
		return fmt.Errorf("while reading the first line of command %s: %s", g.Config.Command.CommandString(), err)
	}

	// Parse the header line of format:
	// # mackerel-agent-plugin [key=value]...
	pluginMetaHeader := map[string]string{}

	m := pluginMetaHeadlineReg.FindStringSubmatch(headerLine)
	if m == nil {
		return fmt.Errorf("bad format of first line: %q", headerLine)
	}

	for _, field := range strings.Fields(m[1]) {
		keyValue := strings.Split(field, "=")
		var value string
		if len(keyValue) > 1 {
			value = keyValue[1]
		} else {
			value = ""
		}
		pluginMetaHeader[keyValue[0]] = value
	}

	// Check schema version
	version, ok := pluginMetaHeader["version"]
	if !ok {
		version = "1"
	}

	if version != "1" {
		return fmt.Errorf("unsupported plugin meta version: %q", version)
	}

	conf := &pluginMeta{}
	err = json.NewDecoder(outBuffer).Decode(conf)

	if err != nil {
		return fmt.Errorf("while reading plugin configuration: %s", err)
	}

	g.Meta = conf

	return nil
}

func (g *pluginGenerator) makeCreateGraphDefsPayload() []mackerel.CreateGraphDefsPayload {
	return makeCreateGraphDefsPayload(g.Meta)
}

func makeCreateGraphDefsPayload(meta *pluginMeta) []mackerel.CreateGraphDefsPayload {
	if meta == nil {
		return nil
	}

	payloads := []mackerel.CreateGraphDefsPayload{}

	for key, graph := range meta.Graphs {
		payload := mackerel.CreateGraphDefsPayload{
			Name:        pluginPrefix + key,
			DisplayName: graph.Label,
			Unit:        graph.Unit,
		}
		if payload.Unit == "" {
			payload.Unit = "float"
		}

		for _, metric := range graph.Metrics {
			metricPayload := mackerel.CreateGraphDefsPayloadMetric{
				Name:        pluginPrefix + key + "." + metric.Name,
				DisplayName: metric.Label,
				IsStacked:   metric.Stacked,
			}
			payload.Metrics = append(payload.Metrics, metricPayload)
		}

		payloads = append(payloads, payload)
	}

	return payloads
}

func (g *pluginGenerator) collectValues() (Values, error) {
	os.Setenv(pluginConfigurationEnvName, "")
	stdout, stderr, _, err := g.Config.Command.Run()

	if stderr != "" {
		pluginLogger.Infof("command %s outputted to STDERR: %q", g.Config.Command.CommandString(), stderr)
	}
	if err != nil {
		pluginLogger.Errorf("Failed to execute command %s (skip these metrics):\n", g.Config.Command.CommandString())
		return nil, err
	}

	results := make(map[string]float64, 0)
	for _, line := range strings.Split(stdout, "\n") {
		// Key, value, timestamp
		// ex.) tcp.CLOSING 0 1397031808
		items := strings.Fields(line)
		if len(items) < 3 {
			continue
		}

		key := items[0]

		if g.Config.IncludePattern != nil && !g.Config.IncludePattern.MatchString(key) {
			continue
		}

		if g.Config.ExcludePattern != nil && g.Config.ExcludePattern.MatchString(key) {
			continue
		}

		value, err := strconv.ParseFloat(items[1], 64)
		if err != nil {
			pluginLogger.Warningf("Failed to parse values: %s", err)
			continue
		}

		results[pluginPrefix+key] = value
	}

	return results, nil
}
<commit_msg>use RunWithEnv instead of os.Setenv to avoid environment variable races<commit_after>package metrics

import (
	"bufio"
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/mackerelio/golib/logging"
	"github.com/mackerelio/mackerel-agent/config"
	"github.com/mackerelio/mackerel-agent/mackerel"
)

// pluginGenerator collects user-defined metrics.
// mackerel-agent runs specified command and parses the result for the metric names and values.
type pluginGenerator struct {
	Config *config.MetricPlugin
	Meta   *pluginMeta
}

// pluginMeta is generated from plugin command. (not the configuration file)
type pluginMeta struct {
	Graphs map[string]customGraphDef
}

type customGraphDef struct {
	Label   string
	Unit    string
	Metrics []customGraphMetricDef
}

type customGraphMetricDef struct {
	Name    string
	Label   string
	Stacked bool
}

var pluginLogger = logging.GetLogger("metrics.plugin")

const pluginPrefix = "custom."

var pluginConfigurationEnvName = "MACKEREL_AGENT_PLUGIN_META"

// NewPluginGenerator XXX
func NewPluginGenerator(conf *config.MetricPlugin) PluginGenerator {
	return &pluginGenerator{Config: conf}
}

func (g *pluginGenerator) Generate() (Values, error) {
	results, err := g.collectValues()
	if err != nil {
		return nil, err
	}
	return results, nil
}

func (g *pluginGenerator) PrepareGraphDefs() ([]mackerel.CreateGraphDefsPayload, error) {
	err := g.loadPluginMeta()
	if err != nil {
		return nil, err
	}

	payload := g.makeCreateGraphDefsPayload()
	return payload, nil
}

func (g *pluginGenerator) CustomIdentifier() *string {
	return g.Config.CustomIdentifier
}

var pluginMetaHeadlineReg = regexp.MustCompile(`^#\s*mackerel-agent-plugin\b(.*)`)

// loadPluginMeta obtains plugin information (e.g. graph visuals, metric
// namespaces, etc) from the command specified.
// mackerel-agent runs the command with MACKEREL_AGENT_PLUGIN_META
// environment variable set.  The command is supposed to output like below:
//
// 	# mackerel-agent-plugin
// 	{
// 	  "graphs": {
// 	    GRAPH_NAME: {
// 	      "label": GRAPH_LABEL,
// 	      "unit": UNIT_TYPE
// 	      "metrics": [
// 	        {
// 	          "name": METRIC_NAME,
// 	          "label": METRIC_LABEL
// 	        },
// 	        ...
// 	      ]
// 	    },
// 	    GRAPH_NAME: ...
// 	  }
// 	}
//
// Valid UNIT_TYPEs are: "float", "integer", "percentage", "bytes", "bytes/sec", "iops"
//
// The output should start with a line beginning with '#', which contains
// meta-info of the configuration. (eg. plugin schema version)
//
// Below is a working example where the plugin emits metrics named "dice.d6" and "dice.d20":
//
// 	{
// 	  "graphs": {
// 	    "dice": {
// 	      "metrics": [
// 	        {
// 	          "name": "d6",
// 	          "label": "Die (d6)"
// 	        },
// 	        {
// 	          "name": "d20",
// 	          "label": "Die (d20)"
// 	        }
// 	      ],
// 	      "unit": "integer",
// 	      "label": "My Dice"
// 	    }
// 	  }
// 	}
func (g *pluginGenerator) loadPluginMeta() error {
	// Set environment variable to make the plugin command generate its configuration
	pluginMetaEnv := pluginConfigurationEnvName + "=1"
	stdout, stderr, exitCode, err := g.Config.Command.RunWithEnv([]string{pluginMetaEnv})
	if err != nil {
		return fmt.Errorf("running %s failed: %s, exit=%d stderr=%q", g.Config.Command.CommandString(), err, exitCode, stderr)
	}

	outBuffer := bufio.NewReader(strings.NewReader(stdout))
	// Read the plugin configuration meta (version etc)

	headerLine, err := outBuffer.ReadString('\n')
	if err != nil {
		return fmt.Errorf("while reading the first line of command %s: %s", g.Config.Command.CommandString(), err)
	}

	// Parse the header line of format:
	// # mackerel-agent-plugin [key=value]...
	pluginMetaHeader := map[string]string{}

	m := pluginMetaHeadlineReg.FindStringSubmatch(headerLine)
	if m == nil {
		return fmt.Errorf("bad format of first line: %q", headerLine)
	}

	for _, field := range strings.Fields(m[1]) {
		keyValue := strings.Split(field, "=")
		var value string
		if len(keyValue) > 1 {
			value = keyValue[1]
		} else {
			value = ""
		}
		pluginMetaHeader[keyValue[0]] = value
	}

	// Check schema version
	version, ok := pluginMetaHeader["version"]
	if !ok {
		version = "1"
	}

	if version != "1" {
		return fmt.Errorf("unsupported plugin meta version: %q", version)
	}

	conf := &pluginMeta{}
	err = json.NewDecoder(outBuffer).Decode(conf)

	if err != nil {
		return fmt.Errorf("while reading plugin configuration: %s", err)
	}

	g.Meta = conf

	return nil
}

func (g *pluginGenerator) makeCreateGraphDefsPayload() []mackerel.CreateGraphDefsPayload {
	return makeCreateGraphDefsPayload(g.Meta)
}

func makeCreateGraphDefsPayload(meta *pluginMeta) []mackerel.CreateGraphDefsPayload {
	if meta == nil {
		return nil
	}

	payloads := []mackerel.CreateGraphDefsPayload{}

	for key, graph := range meta.Graphs {
		payload := mackerel.CreateGraphDefsPayload{
			Name:        pluginPrefix + key,
			DisplayName: graph.Label,
			Unit:        graph.Unit,
		}
		if payload.Unit == "" {
			payload.Unit = "float"
		}

		for _, metric := range graph.Metrics {
			metricPayload := mackerel.CreateGraphDefsPayloadMetric{
				Name:        pluginPrefix + key + "." + metric.Name,
				DisplayName: metric.Label,
				IsStacked:   metric.Stacked,
			}
			payload.Metrics = append(payload.Metrics, metricPayload)
		}

		payloads = append(payloads, payload)
	}

	return payloads
}

func (g *pluginGenerator) collectValues() (Values, error) {
	pluginMetaEnv := pluginConfigurationEnvName + "="
	stdout, stderr, _, err := g.Config.Command.RunWithEnv([]string{pluginMetaEnv})

	if stderr != "" {
		pluginLogger.Infof("command %s outputted to STDERR: %q", g.Config.Command.CommandString(), stderr)
	}
	if err != nil {
		pluginLogger.Errorf("Failed to execute command %s (skip these metrics):\n", g.Config.Command.CommandString())
		return nil, err
	}

	results := make(map[string]float64, 0)
	for _, line := range strings.Split(stdout, "\n") {
		// Key, value, timestamp
		// ex.) tcp.CLOSING 0 1397031808
		items := strings.Fields(line)
		if len(items) < 3 {
			continue
		}

		key := items[0]

		if g.Config.IncludePattern != nil && !g.Config.IncludePattern.MatchString(key) {
			continue
		}

		if g.Config.ExcludePattern != nil && g.Config.ExcludePattern.MatchString(key) {
			continue
		}

		value, err := strconv.ParseFloat(items[1], 64)
		if err != nil {
			pluginLogger.Warningf("Failed to parse values: %s", err)
			continue
		}

		results[pluginPrefix+key] = value
	}

	return results, nil
}
<|endoftext|>


<commit_before>package middleware

import (
	"net"
	"time"

	"github.com/webx-top/echo"
)

func Log() echo.MiddlewareFunc {
	return func(h echo.Handler) echo.Handler {
		return echo.HandlerFunc(func(c echo.Context) error {
			req := c.Request()
			res := c.Response()
			logger := c.Logger()

			start := time.Now()
			if err := h.Handle(c); err != nil {
				c.Error(err)
			}

			remoteAddr := req.RealIP()
			stop := time.Now()
			method := req.Method()
			uri := req.URI()
			size := res.Size()
			code := res.Status()
			logger.Infof("%s %s %s %v %s %d", remoteAddr, method, uri, code, stop.Sub(start), size)
			return nil
		})
	}
}
<commit_msg>remove unused package<commit_after>package middleware

import (
	"time"

	"github.com/webx-top/echo"
)

func Log() echo.MiddlewareFunc {
	return func(h echo.Handler) echo.Handler {
		return echo.HandlerFunc(func(c echo.Context) error {
			req := c.Request()
			res := c.Response()
			logger := c.Logger()

			start := time.Now()
			if err := h.Handle(c); err != nil {
				c.Error(err)
			}

			remoteAddr := req.RealIP()
			stop := time.Now()
			method := req.Method()
			uri := req.URI()
			size := res.Size()
			code := res.Status()
			logger.Infof("%s %s %s %v %s %d", remoteAddr, method, uri, code, stop.Sub(start), size)
			return nil
		})
	}
}
<|endoftext|>


<commit_before>package middleware

import (
	"errors"
	"html/template"
	"net/http"
	"strings"

	"github.com/urandom/webfw"
	"github.com/urandom/webfw/context"
)

/*
The Url middleware provides helper template functions, registered on the base
renderer template. The following functions are currently provided:

 * "url" - expectes a series of string arguments, which will be used to generate
   a url. A trailing request object must be provided. The first string argument
   will be used as the url path, and the subsequent ones will be joined as a
   query string. If the path argument does not begin with a '/', it will be
   treated as a relative path, and the current request path will be prepended.
   The same goes if the the path argument is the empty string. In both cases,
   if the current request has query params, those will be added to the generated
   url. The string arguments may also contain one or more ':here:' strings, which
   will be replaces with the current RequestURI. Finally, if a previous middleware
   has set the language in the context, it will be added to the url.
   Examples for a request /example/request and language "en":
    - {{ url "" .base.r }} -> /en/example/request
    - {{ url "test" .base.r }} -> /en/example/request/test
    - {{ url "/test" .base.r }} -> /en/test
    - {{ url "/test/:here:" .base.r }} -> /en/test/example/request
    - {{ url "/test" "backto=:here:" .base.r }} ->
         /en/test?backto=/example/request
 * "localizedUrl" - very similar to the "url" function. The only difference is that
   the last string argument will be treated as the language to be used when building
   the url. An example for the same request:
    - {{ localizedUrl "/foo" "de" .base.r }} -> /de/foo
*/
type Url struct {
	Pattern string
}

func (mw Url) Handler(ph http.Handler, c context.Context) http.Handler {
	renderer := webfw.GetRenderer(c)
	renderer.Funcs(mw.TemplateFuncMap(c))

	return ph
}

func (mw Url) TemplateFuncMap(c context.Context) template.FuncMap {
	return template.FuncMap{
		"url": func(data ...interface{}) (string, error) {
			r, parts, err := handleParts(data)
			if err != nil {
				return "", err
			}

			return URL(c, r, mw.Pattern, parts)
		},
		"localizedUrl": func(data ...interface{}) (string, error) {
			var lang string
			r, parts, err := handleParts(data)
			if err != nil {
				return "", err
			}

			lang, parts = parts[len(parts)-1], parts[:len(parts)-1]

			return LocalizedURL(c, r, mw.Pattern, lang, parts)
		},
	}
}

// The URL function provides the functionality of the url template functions
// for use outside of the template context. The dispatcherPattern is the
// pattern used by the dispatcher responsible for handling the resulting url.
// In most cases it will probably be "/".
func URL(c context.Context, r *http.Request, dispatcherPattern string, parts []string) (string, error) {
	base, err := unlocalizedUrl(r, parts)

	if err != nil {
		return "", err
	}

	if lang, ok := c.Get(r, context.BaseCtxKey("lang")); ok && len(lang.(string)) > 0 {
		base = "/" + lang.(string) + base
	}
	if len(dispatcherPattern) > 1 {
		base = dispatcherPattern[:len(dispatcherPattern)-1] + base
	}

	if base[len(base)-1] == '/' {
		base = base[:len(base)-1]
	}
	return base, nil
}

// The LocalizedURL function is equivalent to the 'localizedUrl' template
// function.
func LocalizedURL(c context.Context, r *http.Request, dispatcherPattern, language string, parts []string) (string, error) {
	base, err := unlocalizedUrl(r, parts)

	if err != nil {
		return "", err
	}

	base = language + base
	if len(dispatcherPattern) > 1 {
		base = dispatcherPattern + base
	} else {
		base = "/" + base
	}

	if base[len(base)-1] == '/' {
		base = base[:len(base)-1]
	}
	return base, nil
}

func handleParts(data []interface{}) (*http.Request, []string, error) {
	if len(data) < 2 {
		return nil, nil, errors.New("Insufficient arguments")
	}

	if r, ok := data[len(data)-1].(*http.Request); ok {
		data = data[:len(data)-1]

		parts := make([]string, len(data))

		for i, d := range data {
			if p, ok := d.(string); ok {
				parts[i] = p
			} else {
				return nil, nil, errors.New("The leading arguments must be strings")
			}
		}

		return r, parts, nil
	} else {
		return nil, nil, errors.New("The last argument must be a request object")
	}
}

func unlocalizedUrl(r *http.Request, parts []string) (string, error) {
	if len(parts) == 0 {
		return "", errors.New("No base url given")
	}

	uriParts := strings.SplitN(r.RequestURI, "?", 2)
	if uriParts[0] == "" {
		uriParts[0] = r.URL.Path
	}
	for i, part := range parts {
		if r.RequestURI[0] == '/' {
			part = strings.Replace(part, "/:here:", r.RequestURI, -1)
		}
		part = strings.Replace(part, ":here:", r.RequestURI, -1)

		parts[i] = part
	}

	base := parts[0]
	query := strings.Join(parts[1:], "&")

	if len(uriParts) > 1 && uriParts[1] != "" && (base == "" || base[0] != '/') {
		query = uriParts[1] + "&" + query
		if query[len(query)-1] == '&' {
			query = query[:len(query)-1]
		}
	}

	if base == "" {
		base = uriParts[0]
	} else if base[0] != '/' {
		if uriParts[0][len(uriParts[0])-1] == '/' {
			base = uriParts[0] + base
		} else {
			base = uriParts[0] + "/" + base
		}
	}

	if query != "" {
		if strings.Contains(base, "?") {
			base += "&" + query
		} else {
			base += "?" + query
		}
	}

	return base, nil
}
<commit_msg>make sure to leave '/' alone if that's the only thing in the url<commit_after>package middleware

import (
	"errors"
	"html/template"
	"net/http"
	"strings"

	"github.com/urandom/webfw"
	"github.com/urandom/webfw/context"
)

/*
The Url middleware provides helper template functions, registered on the base
renderer template. The following functions are currently provided:

 * "url" - expectes a series of string arguments, which will be used to generate
   a url. A trailing request object must be provided. The first string argument
   will be used as the url path, and the subsequent ones will be joined as a
   query string. If the path argument does not begin with a '/', it will be
   treated as a relative path, and the current request path will be prepended.
   The same goes if the the path argument is the empty string. In both cases,
   if the current request has query params, those will be added to the generated
   url. The string arguments may also contain one or more ':here:' strings, which
   will be replaces with the current RequestURI. Finally, if a previous middleware
   has set the language in the context, it will be added to the url.
   Examples for a request /example/request and language "en":
    - {{ url "" .base.r }} -> /en/example/request
    - {{ url "test" .base.r }} -> /en/example/request/test
    - {{ url "/test" .base.r }} -> /en/test
    - {{ url "/test/:here:" .base.r }} -> /en/test/example/request
    - {{ url "/test" "backto=:here:" .base.r }} ->
         /en/test?backto=/example/request
 * "localizedUrl" - very similar to the "url" function. The only difference is that
   the last string argument will be treated as the language to be used when building
   the url. An example for the same request:
    - {{ localizedUrl "/foo" "de" .base.r }} -> /de/foo
*/
type Url struct {
	Pattern string
}

func (mw Url) Handler(ph http.Handler, c context.Context) http.Handler {
	renderer := webfw.GetRenderer(c)
	renderer.Funcs(mw.TemplateFuncMap(c))

	return ph
}

func (mw Url) TemplateFuncMap(c context.Context) template.FuncMap {
	return template.FuncMap{
		"url": func(data ...interface{}) (string, error) {
			r, parts, err := handleParts(data)
			if err != nil {
				return "", err
			}

			return URL(c, r, mw.Pattern, parts)
		},
		"localizedUrl": func(data ...interface{}) (string, error) {
			var lang string
			r, parts, err := handleParts(data)
			if err != nil {
				return "", err
			}

			lang, parts = parts[len(parts)-1], parts[:len(parts)-1]

			return LocalizedURL(c, r, mw.Pattern, lang, parts)
		},
	}
}

// The URL function provides the functionality of the url template functions
// for use outside of the template context. The dispatcherPattern is the
// pattern used by the dispatcher responsible for handling the resulting url.
// In most cases it will probably be "/".
func URL(c context.Context, r *http.Request, dispatcherPattern string, parts []string) (string, error) {
	base, err := unlocalizedUrl(r, parts)

	if err != nil {
		return "", err
	}

	if lang, ok := c.Get(r, context.BaseCtxKey("lang")); ok && len(lang.(string)) > 0 {
		base = "/" + lang.(string) + base
	}
	if len(dispatcherPattern) > 1 {
		base = dispatcherPattern[:len(dispatcherPattern)-1] + base
	}

	if len(base) > 1 && base[len(base)-1] == '/' {
		base = base[:len(base)-1]
	}
	return base, nil
}

// The LocalizedURL function is equivalent to the 'localizedUrl' template
// function.
func LocalizedURL(c context.Context, r *http.Request, dispatcherPattern, language string, parts []string) (string, error) {
	base, err := unlocalizedUrl(r, parts)

	if err != nil {
		return "", err
	}

	base = language + base
	if len(dispatcherPattern) > 1 {
		base = dispatcherPattern + base
	} else {
		base = "/" + base
	}

	if len(base) > 1 && base[len(base)-1] == '/' {
		base = base[:len(base)-1]
	}
	return base, nil
}

func handleParts(data []interface{}) (*http.Request, []string, error) {
	if len(data) < 2 {
		return nil, nil, errors.New("Insufficient arguments")
	}

	if r, ok := data[len(data)-1].(*http.Request); ok {
		data = data[:len(data)-1]

		parts := make([]string, len(data))

		for i, d := range data {
			if p, ok := d.(string); ok {
				parts[i] = p
			} else {
				return nil, nil, errors.New("The leading arguments must be strings")
			}
		}

		return r, parts, nil
	} else {
		return nil, nil, errors.New("The last argument must be a request object")
	}
}

func unlocalizedUrl(r *http.Request, parts []string) (string, error) {
	if len(parts) == 0 {
		return "", errors.New("No base url given")
	}

	uriParts := strings.SplitN(r.RequestURI, "?", 2)
	if uriParts[0] == "" {
		uriParts[0] = r.URL.Path
	}
	for i, part := range parts {
		if r.RequestURI[0] == '/' {
			part = strings.Replace(part, "/:here:", r.RequestURI, -1)
		}
		part = strings.Replace(part, ":here:", r.RequestURI, -1)

		parts[i] = part
	}

	base := parts[0]
	query := strings.Join(parts[1:], "&")

	if len(uriParts) > 1 && uriParts[1] != "" && (base == "" || base[0] != '/') {
		query = uriParts[1] + "&" + query
		if query[len(query)-1] == '&' {
			query = query[:len(query)-1]
		}
	}

	if base == "" {
		base = uriParts[0]
	} else if base[0] != '/' {
		if uriParts[0][len(uriParts[0])-1] == '/' {
			base = uriParts[0] + base
		} else {
			base = uriParts[0] + "/" + base
		}
	}

	if query != "" {
		if strings.Contains(base, "?") {
			base += "&" + query
		} else {
			base += "?" + query
		}
	}

	return base, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package anago

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"k8s.io/release/pkg/build"
	"k8s.io/release/pkg/changelog"
	"k8s.io/release/pkg/command"
	"k8s.io/release/pkg/git"
	"k8s.io/release/pkg/release"
)

// stageClient is a client for staging releases.
//counterfeiter:generate . stageClient
type stageClient interface {
	// Validate if the provided `ReleaseOptions` are correctly set.
	ValidateOptions() error

	// CheckPrerequisites verifies that a valid GITHUB_TOKEN environment
	// variable is set. It also checks for the existence and version of
	// required packages and if the correct Google Cloud project is set. A
	// basic hardware check will ensure that enough disk space is available,
	// too.
	CheckPrerequisites() error

	// SetBuildCandidate discovers the release branch, parent branch (if
	// available) and build version for this release.
	SetBuildCandidate() error

	// GenerateReleaseVersion discovers the next versions to be released.
	GenerateReleaseVersion(parentBranch string) (*release.Versions, error)

	// PrepareWorkspace verifies that the working directory is in the desired
	// state. This means that the build directory is cleaned up and the checked
	// out repository is in a clean state.
	PrepareWorkspace() error

	// TagRepository creates all necessary git objects by tagging the
	// repository for the provided `versions`.
	TagRepository(versions []string) error

	// Build runs 'make cross-in-a-container' by using the latest kubecross
	// container image. This step also build all necessary release tarballs.
	Build(versions []string) error

	// GenerateChangelog builds the CHANGELOG-x.y.md file and commits it
	// into the local repository.
	GenerateChangelog(version string) error

	// StageArtifacts copies the build artifacts to a Google Cloud Bucket.
	StageArtifacts(versions []string) error
}

// DefaultStage is the default staging implementation used in production.
type DefaultStage struct {
	impl    stageImpl
	options *StageOptions
}

// NewDefaultStage creates a new defaultStage instance.
func NewDefaultStage(options *StageOptions) *DefaultStage {
	return &DefaultStage{&defaultStageImpl{}, options}
}

// SetClient can be used to set the internal stage implementation.
func (d *DefaultStage) SetClient(impl stageImpl) {
	d.impl = impl
}

// defaultStageImpl is the default internal stage client implementation.
type defaultStageImpl struct{}

// stageImpl is the implementation of the stage client.
//counterfeiter:generate . stageImpl
type stageImpl interface {
	PrepareWorkspaceStage() error
	GenerateReleaseVersion(
		releaseType, version, branch string, branchFromMaster bool,
	) (*release.Versions, error)
	CheckReleaseBucket(options *build.Options) error
	Tag(releaseType, version string) error
	MakeCross(version string) error
	GenerateChangelog(options *changelog.Options) error
	StageLocalSourceTree(
		options *build.Options, workDir, buildVersion string,
	) error
	StageLocalArtifacts(options *build.Options) error
	PushReleaseArtifacts(
		options *build.Options, srcPath, gcsPath string,
	) error
	PushContainerImages(options *build.Options) error
}

func (d *defaultStageImpl) GenerateReleaseVersion(
	releaseType, version, branch string, branchFromMaster bool,
) (*release.Versions, error) {
	return release.GenerateReleaseVersion(
		releaseType, version, branch, branchFromMaster,
	)
}

func (d *defaultStageImpl) PrepareWorkspaceStage() error {
	if err := release.PrepareWorkspaceStage(gitRoot); err != nil {
		return err
	}
	return os.Chdir(gitRoot)
}

// TODO: see TagRepository()
func (d *defaultStageImpl) Tag(releaseType, version string) error {
	if err := command.New(
		"git", "config", "--global", "user.email", "nobody@k8s.io",
	).RunSuccess(); err != nil {
		return err
	}
	if err := command.New(
		"git", "config", "--global", "user.name", "Anago GCB",
	).RunSuccess(); err != nil {
		return err
	}
	return command.New(
		"git", "tag", "-a", "-m",
		fmt.Sprintf("Kubernetes %s release %s", releaseType, version),
		version,
	).RunSuccess()
}

func (d *defaultStageImpl) MakeCross(version string) error {
	return build.NewMake().MakeCross(version)
}

func (d *defaultStageImpl) GenerateChangelog(options *changelog.Options) error {
	return changelog.New(options).Run()
}

func (d *defaultStageImpl) CheckReleaseBucket(
	options *build.Options,
) error {
	return build.NewInstance(options).CheckReleaseBucket()
}

func (d *defaultStageImpl) StageLocalSourceTree(
	options *build.Options, workDir, buildVersion string,
) error {
	return build.NewInstance(options).StageLocalSourceTree(workDir, buildVersion)
}

func (d *defaultStageImpl) StageLocalArtifacts(
	options *build.Options,
) error {
	return build.NewInstance(options).StageLocalArtifacts()
}

func (d *defaultStageImpl) PushReleaseArtifacts(
	options *build.Options, srcPath, gcsPath string,
) error {
	return build.NewInstance(options).PushReleaseArtifacts(srcPath, gcsPath)
}

func (d *defaultStageImpl) PushContainerImages(
	options *build.Options,
) error {
	return build.NewInstance(options).PushContainerImages()
}

func (d *DefaultStage) ValidateOptions() error {
	return d.options.Validate()
}

func (d *DefaultStage) CheckPrerequisites() error { return nil }

func (d *DefaultStage) SetBuildCandidate() error { return nil }

func (d *DefaultStage) GenerateReleaseVersion(
	parentBranch string,
) (*release.Versions, error) {
	return d.impl.GenerateReleaseVersion(
		d.options.ReleaseType,
		d.options.BuildVersion,
		d.options.ReleaseBranch,
		parentBranch == git.DefaultBranch,
	)
}

func (d *DefaultStage) PrepareWorkspace() error {
	if err := d.impl.PrepareWorkspaceStage(); err != nil {
		return errors.Wrap(err, "prepare workspace")
	}
	return nil
}

func (d *DefaultStage) TagRepository(versions []string) error {
	// TODO: this is a workaround to make `Build work`, it needs to have the
	// same implementation like anago:prepare_tree.
	for _, version := range versions {
		if err := d.impl.Tag(d.options.ReleaseType, version); err != nil {
			return errors.Wrap(err, "tag version")
		}
	}
	return nil
}

func (d *DefaultStage) Build(versions []string) error {
	for _, version := range versions {
		if err := d.impl.MakeCross(version); err != nil {
			return errors.Wrap(err, "build artifacts")
		}
	}
	return nil
}

func (d *DefaultStage) GenerateChangelog(version string) error {
	return d.impl.GenerateChangelog(&changelog.Options{
		RepoPath:     gitRoot,
		Tag:          version,
		Branch:       d.options.ReleaseBranch,
		Bucket:       d.options.Bucket(),
		HTMLFile:     filepath.Join(workspaceDir, "src/release-notes.html"),
		Dependencies: true,
		Tars: filepath.Join(
			gitRoot,
			fmt.Sprintf("%s-%s", release.BuildDir, version),
			release.ReleaseTarsPath,
		),
	})
}

func (d *DefaultStage) StageArtifacts(versions []string) error {
	for _, version := range versions {
		logrus.Infof("Staging artifacts for version %s", version)
		buildDir := filepath.Join(
			gitRoot, fmt.Sprintf("%s-%s", release.BuildDir, version),
		)
		bucket := d.options.Bucket()
		containerRegistry := d.options.ContainerRegistry()
		pushBuildOptions := &build.Options{
			Bucket:                     bucket,
			BuildDir:                   buildDir,
			DockerRegistry:             containerRegistry,
			Version:                    version,
			AllowDup:                   true,
			ValidateRemoteImageDigests: true,
		}
		if err := d.impl.CheckReleaseBucket(pushBuildOptions); err != nil {
			return errors.Wrap(err, "check release bucket access")
		}

		// Stage the local source tree
		if err := d.impl.StageLocalSourceTree(
			pushBuildOptions,
			workspaceDir,
			d.options.BuildVersion,
		); err != nil {
			return errors.Wrap(err, "staging local source tree")
		}

		// Stage local artifacts and write checksums
		if err := d.impl.StageLocalArtifacts(pushBuildOptions); err != nil {
			return errors.Wrap(err, "staging local artifacts")
		}
		gcsPath := filepath.Join("stage", d.options.BuildVersion, version)

		// Push gcs-stage to GCS
		if err := d.impl.PushReleaseArtifacts(
			pushBuildOptions,
			filepath.Join(buildDir, release.GCSStagePath, version),
			filepath.Join(gcsPath, release.GCSStagePath, version),
		); err != nil {
			return errors.Wrap(err, "pushing release artifacts")
		}

		// Push container release-images to GCS
		if err := d.impl.PushReleaseArtifacts(
			pushBuildOptions,
			filepath.Join(buildDir, release.ImagesPath),
			filepath.Join(gcsPath, release.ImagesPath),
		); err != nil {
			return errors.Wrap(err, "pushing release artifacts")
		}

		// Push container images into registry
		if err := d.impl.PushContainerImages(pushBuildOptions); err != nil {
			return errors.Wrap(err, "pushing container images")
		}
	}
	return nil
}
<commit_msg>Add release message to krel stage<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package anago

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"k8s.io/release/pkg/build"
	"k8s.io/release/pkg/changelog"
	"k8s.io/release/pkg/command"
	"k8s.io/release/pkg/git"
	"k8s.io/release/pkg/release"
)

// stageClient is a client for staging releases.
//counterfeiter:generate . stageClient
type stageClient interface {
	// Validate if the provided `ReleaseOptions` are correctly set.
	ValidateOptions() error

	// CheckPrerequisites verifies that a valid GITHUB_TOKEN environment
	// variable is set. It also checks for the existence and version of
	// required packages and if the correct Google Cloud project is set. A
	// basic hardware check will ensure that enough disk space is available,
	// too.
	CheckPrerequisites() error

	// SetBuildCandidate discovers the release branch, parent branch (if
	// available) and build version for this release.
	SetBuildCandidate() error

	// GenerateReleaseVersion discovers the next versions to be released.
	GenerateReleaseVersion(parentBranch string) (*release.Versions, error)

	// PrepareWorkspace verifies that the working directory is in the desired
	// state. This means that the build directory is cleaned up and the checked
	// out repository is in a clean state.
	PrepareWorkspace() error

	// TagRepository creates all necessary git objects by tagging the
	// repository for the provided `versions`.
	TagRepository(versions []string) error

	// Build runs 'make cross-in-a-container' by using the latest kubecross
	// container image. This step also build all necessary release tarballs.
	Build(versions []string) error

	// GenerateChangelog builds the CHANGELOG-x.y.md file and commits it
	// into the local repository.
	GenerateChangelog(version string) error

	// StageArtifacts copies the build artifacts to a Google Cloud Bucket.
	StageArtifacts(versions []string) error
}

// DefaultStage is the default staging implementation used in production.
type DefaultStage struct {
	impl    stageImpl
	options *StageOptions
}

// NewDefaultStage creates a new defaultStage instance.
func NewDefaultStage(options *StageOptions) *DefaultStage {
	return &DefaultStage{&defaultStageImpl{}, options}
}

// SetClient can be used to set the internal stage implementation.
func (d *DefaultStage) SetClient(impl stageImpl) {
	d.impl = impl
}

// defaultStageImpl is the default internal stage client implementation.
type defaultStageImpl struct{}

// stageImpl is the implementation of the stage client.
//counterfeiter:generate . stageImpl
type stageImpl interface {
	PrepareWorkspaceStage() error
	GenerateReleaseVersion(
		releaseType, version, branch string, branchFromMaster bool,
	) (*release.Versions, error)
	CheckReleaseBucket(options *build.Options) error
	Tag(releaseType, version string) error
	MakeCross(version string) error
	GenerateChangelog(options *changelog.Options) error
	StageLocalSourceTree(
		options *build.Options, workDir, buildVersion string,
	) error
	StageLocalArtifacts(options *build.Options) error
	PushReleaseArtifacts(
		options *build.Options, srcPath, gcsPath string,
	) error
	PushContainerImages(options *build.Options) error
}

func (d *defaultStageImpl) GenerateReleaseVersion(
	releaseType, version, branch string, branchFromMaster bool,
) (*release.Versions, error) {
	return release.GenerateReleaseVersion(
		releaseType, version, branch, branchFromMaster,
	)
}

func (d *defaultStageImpl) PrepareWorkspaceStage() error {
	if err := release.PrepareWorkspaceStage(gitRoot); err != nil {
		return err
	}
	return os.Chdir(gitRoot)
}

// TODO: see TagRepository()
func (d *defaultStageImpl) Tag(releaseType, version string) error {
	if err := command.New(
		"git", "config", "--global", "user.email", "nobody@k8s.io",
	).RunSuccess(); err != nil {
		return err
	}
	if err := command.New(
		"git", "config", "--global", "user.name", "Anago GCB",
	).RunSuccess(); err != nil {
		return err
	}
	return command.New(
		"git", "tag", "-a", "-m",
		fmt.Sprintf("Kubernetes %s release %s", releaseType, version),
		version,
	).RunSuccess()
}

func (d *defaultStageImpl) MakeCross(version string) error {
	return build.NewMake().MakeCross(version)
}

func (d *defaultStageImpl) GenerateChangelog(options *changelog.Options) error {
	return changelog.New(options).Run()
}

func (d *defaultStageImpl) CheckReleaseBucket(
	options *build.Options,
) error {
	return build.NewInstance(options).CheckReleaseBucket()
}

func (d *defaultStageImpl) StageLocalSourceTree(
	options *build.Options, workDir, buildVersion string,
) error {
	return build.NewInstance(options).StageLocalSourceTree(workDir, buildVersion)
}

func (d *defaultStageImpl) StageLocalArtifacts(
	options *build.Options,
) error {
	return build.NewInstance(options).StageLocalArtifacts()
}

func (d *defaultStageImpl) PushReleaseArtifacts(
	options *build.Options, srcPath, gcsPath string,
) error {
	return build.NewInstance(options).PushReleaseArtifacts(srcPath, gcsPath)
}

func (d *defaultStageImpl) PushContainerImages(
	options *build.Options,
) error {
	return build.NewInstance(options).PushContainerImages()
}

func (d *DefaultStage) ValidateOptions() error {
	return d.options.Validate()
}

func (d *DefaultStage) CheckPrerequisites() error { return nil }

func (d *DefaultStage) SetBuildCandidate() error { return nil }

func (d *DefaultStage) GenerateReleaseVersion(
	parentBranch string,
) (*release.Versions, error) {
	return d.impl.GenerateReleaseVersion(
		d.options.ReleaseType,
		d.options.BuildVersion,
		d.options.ReleaseBranch,
		parentBranch == git.DefaultBranch,
	)
}

func (d *DefaultStage) PrepareWorkspace() error {
	if err := d.impl.PrepareWorkspaceStage(); err != nil {
		return errors.Wrap(err, "prepare workspace")
	}
	return nil
}

func (d *DefaultStage) TagRepository(versions []string) error {
	// TODO: this is a workaround to make `Build work`, it needs to have the
	// same implementation like anago:prepare_tree.
	for _, version := range versions {
		if err := d.impl.Tag(d.options.ReleaseType, version); err != nil {
			return errors.Wrap(err, "tag version")
		}
	}
	return nil
}

func (d *DefaultStage) Build(versions []string) error {
	for _, version := range versions {
		if err := d.impl.MakeCross(version); err != nil {
			return errors.Wrap(err, "build artifacts")
		}
	}
	return nil
}

func (d *DefaultStage) GenerateChangelog(version string) error {
	return d.impl.GenerateChangelog(&changelog.Options{
		RepoPath:     gitRoot,
		Tag:          version,
		Branch:       d.options.ReleaseBranch,
		Bucket:       d.options.Bucket(),
		HTMLFile:     filepath.Join(workspaceDir, "src/release-notes.html"),
		Dependencies: true,
		Tars: filepath.Join(
			gitRoot,
			fmt.Sprintf("%s-%s", release.BuildDir, version),
			release.ReleaseTarsPath,
		),
	})
}

func (d *DefaultStage) StageArtifacts(versions []string) error {
	for _, version := range versions {
		logrus.Infof("Staging artifacts for version %s", version)
		buildDir := filepath.Join(
			gitRoot, fmt.Sprintf("%s-%s", release.BuildDir, version),
		)
		bucket := d.options.Bucket()
		containerRegistry := d.options.ContainerRegistry()
		pushBuildOptions := &build.Options{
			Bucket:                     bucket,
			BuildDir:                   buildDir,
			DockerRegistry:             containerRegistry,
			Version:                    version,
			AllowDup:                   true,
			ValidateRemoteImageDigests: true,
		}
		if err := d.impl.CheckReleaseBucket(pushBuildOptions); err != nil {
			return errors.Wrap(err, "check release bucket access")
		}

		// Stage the local source tree
		if err := d.impl.StageLocalSourceTree(
			pushBuildOptions,
			workspaceDir,
			d.options.BuildVersion,
		); err != nil {
			return errors.Wrap(err, "staging local source tree")
		}

		// Stage local artifacts and write checksums
		if err := d.impl.StageLocalArtifacts(pushBuildOptions); err != nil {
			return errors.Wrap(err, "staging local artifacts")
		}
		gcsPath := filepath.Join("stage", d.options.BuildVersion, version)

		// Push gcs-stage to GCS
		if err := d.impl.PushReleaseArtifacts(
			pushBuildOptions,
			filepath.Join(buildDir, release.GCSStagePath, version),
			filepath.Join(gcsPath, release.GCSStagePath, version),
		); err != nil {
			return errors.Wrap(err, "pushing release artifacts")
		}

		// Push container release-images to GCS
		if err := d.impl.PushReleaseArtifacts(
			pushBuildOptions,
			filepath.Join(buildDir, release.ImagesPath),
			filepath.Join(gcsPath, release.ImagesPath),
		); err != nil {
			return errors.Wrap(err, "pushing release artifacts")
		}

		// Push container images into registry
		if err := d.impl.PushContainerImages(pushBuildOptions); err != nil {
			return errors.Wrap(err, "pushing container images")
		}
	}

	noMockFlag := ""
	if d.options.NoMock {
		noMockFlag = "--nomock"
	}

	logrus.Infof(
		"To release this staged build, run:\n\n"+
			"$ krel gcbmgr --no-anago --release "+
			"--type %s "+
			"--branch %s "+
			"--build-version=%s %s",
		d.options.ReleaseType,
		d.options.ReleaseBranch,
		d.options.BuildVersion,
		noMockFlag,
	)
	return nil
}
<|endoftext|>


<commit_before>package driver

import (
	"fmt"
	"log"
	"math/rand"
	"os"

	"github.com/docker/go-plugins-helpers/volume"
)

const (
	cfgFile      = "/etc/minfs/config.json"
	cfgDir       = "/etc/minfs/"
	vers         = "1"
	volumePrefix = "miniovol-"
	bucketPrefix = "miniobucket-"
	location     = "us-east-1"
)

type minfsCfg struct {
	Version   string `json:"version"`
	AccessKey string `json:"accessKey"`
	SecretKey string `json:"secretKey"`
}

// EnvVarError implements the error interface and uses it to return a generic
// validation error for environment variables that are mandatory.
type EnvVarError struct {
	envVar string
}

func (e EnvVarError) Error() string {
	return fmt.Sprintf("environment variable %s cannot be empty", e.envVar)
}

func newErrEmptyEnvVar(v string) error {
	return EnvVarError{
		envVar: v,
	}
}

// VolumeError implements the error interface to eliminate message duplication
// when the driver checks for a specific volume
type VolumeError struct {
	volumeName string
}

func (e VolumeError) Error() string {
	return fmt.Sprintf("volume %s not found", e.volumeName)
}

func newErrVolNotFound(v string) error {
	return VolumeError{
		volumeName: v,
	}
}

// ProvisionConfig updates the minfs config with the Minio instance details
// (accessKeyID, secretAccessKey, serverURI)
// This is necessary for minfs to autheticate with the Minio instance.
// NOTE: move this to the driver to streamline testing?
// NOTE: if the API is correct, it should be possible to do this via env vars.
func provisionConfig(m *MinioDriver) error {
	if _, err := os.Stat(cfgDir); os.IsNotExist(err) {
		if err = os.MkdirAll(cfgDir, 0755); err != nil {
			glog.V(1).Infof("Error while creating MinFS config dir: %s", err)
			return err
		}
	} else if err != nil {
		glog.V(1).Infof("Error while writing MinFS config: %s", err)
		return err
	}

	details := fmt.Sprintf(`{"version":"%s","accessKey":"%s","secretKey":"%s"}`,
		vers,
		m.accessKey,
		m.secretKey,
	)

	fh, err := os.Create(cfgFile)
	if err != nil {
		return err
	}
	defer fh.Close()

	fh.WriteString(details)
	if err != nil {
		return err
	}

	return nil
}

func newCfg(accessKey, secretKey, version string) *minfsCfg {
	return &minfsCfg{
		Version:   version,
		AccessKey: accessKey,
		SecretKey: secretKey,
	}
}

func createName(prefix string) string {
	return fmt.Sprintf("%s%08x", prefix, rand.Uint32())
}

func checkParam(param string, opts map[string]string) (string, error) {
	stringParam, exists := opts[param]
	if stringParam == "" || !exists {
		return "", fmt.Errorf("%s option is required", param)
	}
	return stringParam, nil
}

func volumeResp(mountPoint, rName string, volumes []*volume.Volume, capabilities volume.Capability, err string) volume.Response {
	return volume.Response{
		Err: err,
		Volume: &volume.Volume{
			Mountpoint: mountPoint,
			Name:       rName,
		},
		Volumes:      volumes,
		Capabilities: capabilities,
	}
}
<commit_msg>add missing glog import<commit_after>package driver

import (
	"fmt"
	"math/rand"
	"os"

	"github.com/docker/go-plugins-helpers/volume"
	"github.com/golang/glog"
)

const (
	cfgFile      = "/etc/minfs/config.json"
	cfgDir       = "/etc/minfs/"
	vers         = "1"
	volumePrefix = "miniovol-"
	bucketPrefix = "miniobucket-"
	location     = "us-east-1"
)

type minfsCfg struct {
	Version   string `json:"version"`
	AccessKey string `json:"accessKey"`
	SecretKey string `json:"secretKey"`
}

// EnvVarError implements the error interface and uses it to return a generic
// validation error for environment variables that are mandatory.
type EnvVarError struct {
	envVar string
}

func (e EnvVarError) Error() string {
	return fmt.Sprintf("environment variable %s cannot be empty", e.envVar)
}

func newErrEmptyEnvVar(v string) error {
	return EnvVarError{
		envVar: v,
	}
}

// VolumeError implements the error interface to eliminate message duplication
// when the driver checks for a specific volume
type VolumeError struct {
	volumeName string
}

func (e VolumeError) Error() string {
	return fmt.Sprintf("volume %s not found", e.volumeName)
}

func newErrVolNotFound(v string) error {
	return VolumeError{
		volumeName: v,
	}
}

// ProvisionConfig updates the minfs config with the Minio instance details
// (accessKeyID, secretAccessKey, serverURI)
// This is necessary for minfs to autheticate with the Minio instance.
// NOTE: move this to the driver to streamline testing?
// NOTE: if the API is correct, it should be possible to do this via env vars.
func provisionConfig(m *MinioDriver) error {
	if _, err := os.Stat(cfgDir); os.IsNotExist(err) {
		if err = os.MkdirAll(cfgDir, 0755); err != nil {
			glog.V(1).Infof("Error while creating MinFS config dir: %s", err)
			return err
		}
	} else if err != nil {
		glog.V(1).Infof("Error while writing MinFS config: %s", err)
		return err
	}

	details := fmt.Sprintf(`{"version":"%s","accessKey":"%s","secretKey":"%s"}`,
		vers,
		m.accessKey,
		m.secretKey,
	)

	fh, err := os.Create(cfgFile)
	if err != nil {
		return err
	}
	defer fh.Close()

	fh.WriteString(details)
	if err != nil {
		return err
	}

	return nil
}

func newCfg(accessKey, secretKey, version string) *minfsCfg {
	return &minfsCfg{
		Version:   version,
		AccessKey: accessKey,
		SecretKey: secretKey,
	}
}

func createName(prefix string) string {
	return fmt.Sprintf("%s%08x", prefix, rand.Uint32())
}

func checkParam(param string, opts map[string]string) (string, error) {
	stringParam, exists := opts[param]
	if stringParam == "" || !exists {
		return "", fmt.Errorf("%s option is required", param)
	}
	return stringParam, nil
}

func volumeResp(mountPoint, rName string, volumes []*volume.Volume, capabilities volume.Capability, err string) volume.Response {
	return volume.Response{
		Err: err,
		Volume: &volume.Volume{
			Mountpoint: mountPoint,
			Name:       rName,
		},
		Volumes:      volumes,
		Capabilities: capabilities,
	}
}
<|endoftext|>


<commit_before>package envoy

import (
	"context"
	"kourier/pkg/kubernetes"
	"fmt"
	envoyv2 "github.com/envoyproxy/go-control-plane/envoy/api/v2"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/listener"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
	accesslogv2 "github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2"
	"github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
	discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
	"github.com/envoyproxy/go-control-plane/pkg/cache"
	xds "github.com/envoyproxy/go-control-plane/pkg/server"
	"github.com/envoyproxy/go-control-plane/pkg/util"
	"github.com/gogo/protobuf/types"
	"github.com/google/uuid"
	log "github.com/sirupsen/logrus"
	"google.golang.org/grpc"
	kubev1 "k8s.io/api/core/v1"
	v1alpha12 "knative.dev/serving/pkg/apis/networking/v1alpha1"
	"net"
	"net/http"
	"strconv"
	"time"
)

const (
	grpcMaxConcurrentStreams = 1000000
)

type EnvoyXdsServer struct {
	gatewayPort    uint
	managementPort uint
	kubeClient     kubernetes.KubernetesClient // TODO: let's try to remove this coupling later
	ctx            context.Context
	server         xds.Server
	snapshotCache  cache.SnapshotCache
}

// Hasher returns node ID as an ID
type Hasher struct {
}

func (h Hasher) ID(node *core.Node) string {
	if node == nil {
		return "unknown"
	}
	return node.Id
}

func NewEnvoyXdsServer(gatewayPort uint, managementPort uint, kubeClient kubernetes.KubernetesClient) EnvoyXdsServer {
	ctx := context.Background()
	snapshotCache := cache.NewSnapshotCache(true, Hasher{}, nil)
	srv := xds.NewServer(snapshotCache, nil)

	return EnvoyXdsServer{
		gatewayPort:    gatewayPort,
		managementPort: managementPort,
		kubeClient:     kubeClient,
		ctx:            ctx,
		server:         srv,
		snapshotCache:  snapshotCache,
	}
}

// RunManagementServer starts an xDS server at the given Port.
func (envoyXdsServer *EnvoyXdsServer) RunManagementServer() {
	port := envoyXdsServer.managementPort
	server := envoyXdsServer.server

	var grpcOptions []grpc.ServerOption
	grpcOptions = append(grpcOptions, grpc.MaxConcurrentStreams(grpcMaxConcurrentStreams))
	grpcServer := grpc.NewServer(grpcOptions...)
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		log.Error("Failed to listen")
	}

	// register services
	discovery.RegisterAggregatedDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterEndpointDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterClusterDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterRouteDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterListenerDiscoveryServiceServer(grpcServer, server)

	log.Printf("Starting Management Server on Port %d\n", port)
	go func() {
		if err = grpcServer.Serve(lis); err != nil {
			log.Errorf("%s", err)
		}
	}()
	<-envoyXdsServer.ctx.Done()
	grpcServer.GracefulStop()
}

// RunManagementGateway starts an HTTP gateway to an xDS server.
func (envoyXdsServer *EnvoyXdsServer) RunGateway() {
	port := envoyXdsServer.gatewayPort
	server := envoyXdsServer.server
	ctx := envoyXdsServer.ctx

	log.Printf("Starting HTTP/1.1 gateway on Port %d\n", port)
	httpServer := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: &xds.HTTPGateway{Server: server}}
	go func() {
		if err := httpServer.ListenAndServe(); err != nil {
			panic(err)
		}
	}()

	<-ctx.Done()
	if err := httpServer.Shutdown(ctx); err != nil {
		panic(err)
	}
}

func (envoyXdsServer *EnvoyXdsServer) SetSnapshotForClusterIngresses(nodeId string, Ingresses []v1alpha12.IngressAccessor) {
	var virtualHosts []*route.VirtualHost
	var routeCache []cache.Resource
	var clusterCache []cache.Resource

	for i, ingress := range Ingresses {

		routeName := getRouteName(ingress)
		routeNamespace := getRouteNamespace(ingress)

		log.WithFields(log.Fields{"name": routeName, "namespace": routeNamespace}).Info("Knative Ingress found")

		for _, rule := range ingress.GetSpec().Rules {

			var ruleRoute []*route.Route
			domains := rule.Hosts

			for _, httpPath := range rule.HTTP.Paths {

				path := "/"
				if httpPath.Path != "" {
					path = httpPath.Path
				}

				headers := httpPath.AppendHeaders

				var wrs []*route.WeightedCluster_ClusterWeight

				for _, split := range httpPath.Splits {

					endpointList, err := envoyXdsServer.kubeClient.EndpointsForRevision(split.ServiceNamespace, split.ServiceName)

					if err != nil {
						log.Errorf("%s", err)
						break
					}

					service, err := envoyXdsServer.kubeClient.ServiceForRevision(split.ServiceNamespace, split.ServiceName)

					if err != nil {
						log.Errorf("%s", err)
						break
					}

					var targetPort int32
					http2 := false
					for _, port := range service.Spec.Ports {
						if port.Port == split.ServicePort.IntVal || port.Name == split.ServicePort.StrVal {
							targetPort = port.TargetPort.IntVal
							http2 = port.Name == "http2" || port.Name == "h2c"
						}
					}

					lbEndpoints := lbEndpointsForKubeEndpoints(endpointList, targetPort)

					connectTimeout := 5 * time.Second
					cluster := clusterForRevision(split.ServiceName, connectTimeout, lbEndpoints, http2, path)
					clusterCache = append(clusterCache, &cluster)

					weightedCluster := weightedCluster(split.ServiceName, uint32(split.Percent), path, headers)

					wrs = append(wrs, &weightedCluster)

				}

				r := createRouteForRevision(routeName, i, path, wrs)

				ruleRoute = append(ruleRoute, &r)
				routeCache = append(routeCache, &r)

			}

			virtualHost := route.VirtualHost{
				Name:    routeName,
				Domains: domains,
				Routes:  ruleRoute,
			}

			virtualHosts = append(virtualHosts, &virtualHost)
		}

	}

	manager := httpConnectionManager(virtualHosts)
	l := envoyListener(&manager)
	listenerCache := []cache.Resource{&l}

	snapshotVersion, errUUID := uuid.NewUUID()
	if errUUID != nil {
		log.Error(errUUID)
		return
	}

	snapshot := cache.NewSnapshot(snapshotVersion.String(), nil, clusterCache, routeCache, listenerCache)

	err := envoyXdsServer.snapshotCache.SetSnapshot(nodeId, snapshot)
	if err != nil {
		log.Error(err)
	}
}

func createRouteForRevision(routeName string, i int, path string, wrs []*route.WeightedCluster_ClusterWeight) route.Route {
	r := route.Route{
		Name: routeName + "_" + strconv.Itoa(i),
		Match: &route.RouteMatch{
			PathSpecifier: &route.RouteMatch_Prefix{
				Prefix: path,
			},
		},
		Action: &route.Route_Route{Route: &route.RouteAction{
			ClusterSpecifier: &route.RouteAction_WeightedClusters{
				WeightedClusters: &route.WeightedCluster{
					Clusters: wrs,
				},
			},
		}},
	}
	return r
}

func getRouteNamespace(ingress v1alpha12.IngressAccessor) string {
	return ingress.GetLabels()["serving.knative.dev/routeNamespace"]
}

func getRouteName(ingress v1alpha12.IngressAccessor) string {
	return ingress.GetLabels()["serving.knative.dev/route"]
}

func lbEndpointsForKubeEndpoints(kubeEndpoints *kubev1.EndpointsList, targetPort int32) []*endpoint.LbEndpoint {
	var result []*endpoint.LbEndpoint

	for _, kubeEndpoint := range kubeEndpoints.Items {
		for _, subset := range kubeEndpoint.Subsets {

			for _, address := range subset.Addresses {

				serviceEndpoint := &core.Address{
					Address: &core.Address_SocketAddress{
						SocketAddress: &core.SocketAddress{
							Protocol: core.TCP,
							Address:  address.IP,
							PortSpecifier: &core.SocketAddress_PortValue{
								PortValue: uint32(targetPort),
							},
							Ipv4Compat: true,
						},
					},
				}

				lbEndpoint := endpoint.LbEndpoint{
					HostIdentifier: &endpoint.LbEndpoint_Endpoint{
						Endpoint: &endpoint.Endpoint{
							Address: serviceEndpoint,
						},
					},
				}
				result = append(result, &lbEndpoint)
			}
		}
	}

	return result
}

func clusterForRevision(revisionName string, connectTimeout time.Duration, lbEndpoints []*endpoint.LbEndpoint, http2 bool, path string) envoyv2.Cluster {

	cluster := envoyv2.Cluster{
		Name: revisionName + path,
		ClusterDiscoveryType: &envoyv2.Cluster_Type{
			Type: envoyv2.Cluster_STRICT_DNS,
		},
		ConnectTimeout: &connectTimeout,
		LoadAssignment: &envoyv2.ClusterLoadAssignment{
			ClusterName: revisionName + path,
			Endpoints: []*endpoint.LocalityLbEndpoints{
				{
					LbEndpoints: lbEndpoints,
				},
			},
		},
	}

	if http2 {
		cluster.Http2ProtocolOptions = &core.Http2ProtocolOptions{}
	}

	return cluster
}

func weightedCluster(revisionName string, trafficPerc uint32, path string, headersToAdd map[string]string) route.WeightedCluster_ClusterWeight {

	var headers []*core.HeaderValueOption

	for k, v := range headersToAdd {

		header := core.HeaderValueOption{
			Header: &core.HeaderValue{
				Key:   k,
				Value: v,
			},
			Append: &types.BoolValue{
				Value: true,
			},
		}

		headers = append(headers, &header)

	}

	return route.WeightedCluster_ClusterWeight{
		Name: revisionName + path,
		Weight: &types.UInt32Value{
			Value: trafficPerc,
		},
		RequestHeadersToAdd: headers,
	}
}

func httpConnectionManager(virtualHosts []*route.VirtualHost) v2.HttpConnectionManager {
	return v2.HttpConnectionManager{
		CodecType:  v2.AUTO,
		StatPrefix: "ingress_http",
		RouteSpecifier: &v2.HttpConnectionManager_RouteConfig{
			RouteConfig: &envoyv2.RouteConfiguration{
				Name:         "local_route",
				VirtualHosts: virtualHosts,
			},
		},
		HttpFilters: []*v2.HttpFilter{
			{
				Name: util.Router,
			},
		},

		AccessLog: accessLogs(),
	}
}

// Outputs to /dev/stdout using the default format
func accessLogs() []*accesslogv2.AccessLog {
	accessLogConfigFields := make(map[string]*types.Value)
	accessLogConfigFields["path"] = &types.Value{
		Kind: &types.Value_StringValue{
			StringValue: "/dev/stdout",
		},
	}

	return []*accesslogv2.AccessLog{
		{
			Name: "envoy.file_access_log",
			ConfigType: &accesslogv2.AccessLog_Config{
				Config: &types.Struct{Fields: accessLogConfigFields},
			},
		},
	}
}

func envoyListener(httpConnectionManager *v2.HttpConnectionManager) envoyv2.Listener {
	pbst, err := util.MessageToStruct(httpConnectionManager)
	if err != nil {
		panic(err)
	}

	return envoyv2.Listener{
		Name: "listener_0",
		Address: &core.Address{
			Address: &core.Address_SocketAddress{
				SocketAddress: &core.SocketAddress{
					Protocol: core.TCP,
					Address:  "0.0.0.0",
					PortSpecifier: &core.SocketAddress_PortValue{
						PortValue: uint32(8080),
					},
				},
			},
		},
		FilterChains: []*listener.FilterChain{{
			Filters: []*listener.Filter{{
				Name:       util.HTTPConnectionManager,
				ConfigType: &listener.Filter_Config{Config: pbst},
			}},
		}},
	}
}
<commit_msg>envoy: add retries<commit_after>package envoy

import (
	"context"
	"fmt"
	envoyv2 "github.com/envoyproxy/go-control-plane/envoy/api/v2"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/listener"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
	accesslogv2 "github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2"
	"github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
	discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2"
	"github.com/envoyproxy/go-control-plane/pkg/cache"
	xds "github.com/envoyproxy/go-control-plane/pkg/server"
	"github.com/envoyproxy/go-control-plane/pkg/util"
	"github.com/gogo/protobuf/types"
	"github.com/google/uuid"
	log "github.com/sirupsen/logrus"
	"google.golang.org/grpc"
	kubev1 "k8s.io/api/core/v1"
	v1alpha12 "knative.dev/serving/pkg/apis/networking/v1alpha1"
	"kourier/pkg/kubernetes"
	"net"
	"net/http"
	"strconv"
	"time"
)

const (
	grpcMaxConcurrentStreams = 1000000
)

type EnvoyXdsServer struct {
	gatewayPort    uint
	managementPort uint
	kubeClient     kubernetes.KubernetesClient // TODO: let's try to remove this coupling later
	ctx            context.Context
	server         xds.Server
	snapshotCache  cache.SnapshotCache
}

// Hasher returns node ID as an ID
type Hasher struct {
}

func (h Hasher) ID(node *core.Node) string {
	if node == nil {
		return "unknown"
	}
	return node.Id
}

func NewEnvoyXdsServer(gatewayPort uint, managementPort uint, kubeClient kubernetes.KubernetesClient) EnvoyXdsServer {
	ctx := context.Background()
	snapshotCache := cache.NewSnapshotCache(true, Hasher{}, nil)
	srv := xds.NewServer(snapshotCache, nil)

	return EnvoyXdsServer{
		gatewayPort:    gatewayPort,
		managementPort: managementPort,
		kubeClient:     kubeClient,
		ctx:            ctx,
		server:         srv,
		snapshotCache:  snapshotCache,
	}
}

// RunManagementServer starts an xDS server at the given Port.
func (envoyXdsServer *EnvoyXdsServer) RunManagementServer() {
	port := envoyXdsServer.managementPort
	server := envoyXdsServer.server

	var grpcOptions []grpc.ServerOption
	grpcOptions = append(grpcOptions, grpc.MaxConcurrentStreams(grpcMaxConcurrentStreams))
	grpcServer := grpc.NewServer(grpcOptions...)
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		log.Error("Failed to listen")
	}

	// register services
	discovery.RegisterAggregatedDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterEndpointDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterClusterDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterRouteDiscoveryServiceServer(grpcServer, server)
	envoyv2.RegisterListenerDiscoveryServiceServer(grpcServer, server)

	log.Printf("Starting Management Server on Port %d\n", port)
	go func() {
		if err = grpcServer.Serve(lis); err != nil {
			log.Errorf("%s", err)
		}
	}()
	<-envoyXdsServer.ctx.Done()
	grpcServer.GracefulStop()
}

// RunManagementGateway starts an HTTP gateway to an xDS server.
func (envoyXdsServer *EnvoyXdsServer) RunGateway() {
	port := envoyXdsServer.gatewayPort
	server := envoyXdsServer.server
	ctx := envoyXdsServer.ctx

	log.Printf("Starting HTTP/1.1 gateway on Port %d\n", port)
	httpServer := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: &xds.HTTPGateway{Server: server}}
	go func() {
		if err := httpServer.ListenAndServe(); err != nil {
			panic(err)
		}
	}()

	<-ctx.Done()
	if err := httpServer.Shutdown(ctx); err != nil {
		panic(err)
	}
}

func (envoyXdsServer *EnvoyXdsServer) SetSnapshotForClusterIngresses(nodeId string, Ingresses []v1alpha12.IngressAccessor) {
	var virtualHosts []*route.VirtualHost
	var routeCache []cache.Resource
	var clusterCache []cache.Resource

	for i, ingress := range Ingresses {

		routeName := getRouteName(ingress)
		routeNamespace := getRouteNamespace(ingress)

		log.WithFields(log.Fields{"name": routeName, "namespace": routeNamespace}).Info("Knative Ingress found")

		for _, rule := range ingress.GetSpec().Rules {

			var ruleRoute []*route.Route
			domains := rule.Hosts

			for _, httpPath := range rule.HTTP.Paths {

				path := "/"
				if httpPath.Path != "" {
					path = httpPath.Path
				}

				headers := httpPath.AppendHeaders

				var wrs []*route.WeightedCluster_ClusterWeight

				for _, split := range httpPath.Splits {

					endpointList, err := envoyXdsServer.kubeClient.EndpointsForRevision(split.ServiceNamespace, split.ServiceName)

					if err != nil {
						log.Errorf("%s", err)
						break
					}

					service, err := envoyXdsServer.kubeClient.ServiceForRevision(split.ServiceNamespace, split.ServiceName)

					if err != nil {
						log.Errorf("%s", err)
						break
					}

					var targetPort int32
					http2 := false
					for _, port := range service.Spec.Ports {
						if port.Port == split.ServicePort.IntVal || port.Name == split.ServicePort.StrVal {
							targetPort = port.TargetPort.IntVal
							http2 = port.Name == "http2" || port.Name == "h2c"
						}
					}

					lbEndpoints := lbEndpointsForKubeEndpoints(endpointList, targetPort)

					connectTimeout := 5 * time.Second
					cluster := clusterForRevision(split.ServiceName, connectTimeout, lbEndpoints, http2, path)
					clusterCache = append(clusterCache, &cluster)

					weightedCluster := weightedCluster(split.ServiceName, uint32(split.Percent), path, headers)

					wrs = append(wrs, &weightedCluster)

				}

				attempts := 0
				var perTryTimeout time.Duration
				if httpPath.Retries != nil {
					attempts = httpPath.Retries.Attempts

					if httpPath.Retries.PerTryTimeout != nil {
						perTryTimeout = httpPath.Retries.PerTryTimeout.Duration
					}
				}

				r := createRouteForRevision(routeName, i, path, wrs, attempts, perTryTimeout)

				ruleRoute = append(ruleRoute, &r)
				routeCache = append(routeCache, &r)

			}

			virtualHost := route.VirtualHost{
				Name:    routeName,
				Domains: domains,
				Routes:  ruleRoute,
			}

			virtualHosts = append(virtualHosts, &virtualHost)
		}

	}

	manager := httpConnectionManager(virtualHosts)
	l := envoyListener(&manager)
	listenerCache := []cache.Resource{&l}

	snapshotVersion, errUUID := uuid.NewUUID()
	if errUUID != nil {
		log.Error(errUUID)
		return
	}

	snapshot := cache.NewSnapshot(snapshotVersion.String(), nil, clusterCache, routeCache, listenerCache)

	err := envoyXdsServer.snapshotCache.SetSnapshot(nodeId, snapshot)
	if err != nil {
		log.Error(err)
	}
}

func createRouteForRevision(routeName string, i int, path string, wrs []*route.WeightedCluster_ClusterWeight, attempts int, perTryTimeout time.Duration) route.Route {
	r := route.Route{
		Name: routeName + "_" + strconv.Itoa(i),
		Match: &route.RouteMatch{
			PathSpecifier: &route.RouteMatch_Prefix{
				Prefix: path,
			},
		},
		Action: &route.Route_Route{Route: &route.RouteAction{
			ClusterSpecifier: &route.RouteAction_WeightedClusters{
				WeightedClusters: &route.WeightedCluster{
					Clusters: wrs,
				},
			},
			RetryPolicy: createRetryPolicy(attempts, perTryTimeout),
		}},
	}

	return r
}

func createRetryPolicy(attempts int, perTryTimeout time.Duration) *route.RetryPolicy {
	if attempts > 0 {
		return &route.RetryPolicy{
			RetryOn: "5xx",
			NumRetries: &types.UInt32Value{
				Value: uint32(attempts),
			},
			PerTryTimeout: &perTryTimeout,
		}
	} else {
		return nil
	}
}

func getRouteNamespace(ingress v1alpha12.IngressAccessor) string {
	return ingress.GetLabels()["serving.knative.dev/routeNamespace"]
}

func getRouteName(ingress v1alpha12.IngressAccessor) string {
	return ingress.GetLabels()["serving.knative.dev/route"]
}

func lbEndpointsForKubeEndpoints(kubeEndpoints *kubev1.EndpointsList, targetPort int32) []*endpoint.LbEndpoint {
	var result []*endpoint.LbEndpoint

	for _, kubeEndpoint := range kubeEndpoints.Items {
		for _, subset := range kubeEndpoint.Subsets {

			for _, address := range subset.Addresses {

				serviceEndpoint := &core.Address{
					Address: &core.Address_SocketAddress{
						SocketAddress: &core.SocketAddress{
							Protocol: core.TCP,
							Address:  address.IP,
							PortSpecifier: &core.SocketAddress_PortValue{
								PortValue: uint32(targetPort),
							},
							Ipv4Compat: true,
						},
					},
				}

				lbEndpoint := endpoint.LbEndpoint{
					HostIdentifier: &endpoint.LbEndpoint_Endpoint{
						Endpoint: &endpoint.Endpoint{
							Address: serviceEndpoint,
						},
					},
				}
				result = append(result, &lbEndpoint)
			}
		}
	}

	return result
}

func clusterForRevision(revisionName string, connectTimeout time.Duration, lbEndpoints []*endpoint.LbEndpoint, http2 bool, path string) envoyv2.Cluster {

	cluster := envoyv2.Cluster{
		Name: revisionName + path,
		ClusterDiscoveryType: &envoyv2.Cluster_Type{
			Type: envoyv2.Cluster_STRICT_DNS,
		},
		ConnectTimeout: &connectTimeout,
		LoadAssignment: &envoyv2.ClusterLoadAssignment{
			ClusterName: revisionName + path,
			Endpoints: []*endpoint.LocalityLbEndpoints{
				{
					LbEndpoints: lbEndpoints,
				},
			},
		},
	}

	if http2 {
		cluster.Http2ProtocolOptions = &core.Http2ProtocolOptions{}
	}

	return cluster
}

func weightedCluster(revisionName string, trafficPerc uint32, path string, headersToAdd map[string]string) route.WeightedCluster_ClusterWeight {

	var headers []*core.HeaderValueOption

	for k, v := range headersToAdd {

		header := core.HeaderValueOption{
			Header: &core.HeaderValue{
				Key:   k,
				Value: v,
			},
			Append: &types.BoolValue{
				Value: true,
			},
		}

		headers = append(headers, &header)

	}

	return route.WeightedCluster_ClusterWeight{
		Name: revisionName + path,
		Weight: &types.UInt32Value{
			Value: trafficPerc,
		},
		RequestHeadersToAdd: headers,
	}
}

func httpConnectionManager(virtualHosts []*route.VirtualHost) v2.HttpConnectionManager {
	return v2.HttpConnectionManager{
		CodecType:  v2.AUTO,
		StatPrefix: "ingress_http",
		RouteSpecifier: &v2.HttpConnectionManager_RouteConfig{
			RouteConfig: &envoyv2.RouteConfiguration{
				Name:         "local_route",
				VirtualHosts: virtualHosts,
			},
		},
		HttpFilters: []*v2.HttpFilter{
			{
				Name: util.Router,
			},
		},

		AccessLog: accessLogs(),
	}
}

// Outputs to /dev/stdout using the default format
func accessLogs() []*accesslogv2.AccessLog {
	accessLogConfigFields := make(map[string]*types.Value)
	accessLogConfigFields["path"] = &types.Value{
		Kind: &types.Value_StringValue{
			StringValue: "/dev/stdout",
		},
	}

	return []*accesslogv2.AccessLog{
		{
			Name: "envoy.file_access_log",
			ConfigType: &accesslogv2.AccessLog_Config{
				Config: &types.Struct{Fields: accessLogConfigFields},
			},
		},
	}
}

func envoyListener(httpConnectionManager *v2.HttpConnectionManager) envoyv2.Listener {
	pbst, err := util.MessageToStruct(httpConnectionManager)
	if err != nil {
		panic(err)
	}

	return envoyv2.Listener{
		Name: "listener_0",
		Address: &core.Address{
			Address: &core.Address_SocketAddress{
				SocketAddress: &core.SocketAddress{
					Protocol: core.TCP,
					Address:  "0.0.0.0",
					PortSpecifier: &core.SocketAddress_PortValue{
						PortValue: uint32(8080),
					},
				},
			},
		},
		FilterChains: []*listener.FilterChain{{
			Filters: []*listener.Filter{{
				Name:       util.HTTPConnectionManager,
				ConfigType: &listener.Filter_Config{Config: pbst},
			}},
		}},
	}
}
<|endoftext|>


<commit_before>package helm

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/Masterminds/semver"
	"sigs.k8s.io/yaml"
)

// LoadChartfile opens a Chartfile tree
func LoadChartfile(projectRoot string) (*Charts, error) {
	// make sure project root is valid
	abs, err := filepath.Abs(projectRoot)
	if err != nil {
		return nil, err
	}

	// open chartfile
	chartfile := filepath.Join(abs, Filename)
	data, err := ioutil.ReadFile(chartfile)
	if err != nil {
		return nil, err
	}

	// parse it
	c := Chartfile{
		Version:   Version,
		Directory: DefaultDir,
	}
	if err := yaml.UnmarshalStrict(data, &c); err != nil {
		return nil, err
	}

	for i, r := range c.Requires {
		if r.Chart == "" {
			return nil, fmt.Errorf("requirements[%v]: 'chart' must be set", i)
		}
	}

	// return Charts handle
	charts := &Charts{
		Manifest:    c,
		projectRoot: abs,

		// default to ExecHelm, but allow injecting from the outside
		Helm: ExecHelm{},
	}
	return charts, nil
}

// Charts exposes the central Chartfile management functions
type Charts struct {
	// Manifest are the chartfile.yaml contents. It holds data about the developers intentions
	Manifest Chartfile

	// projectRoot is the enclosing directory of chartfile.yaml
	projectRoot string

	// Helm is the helm implementation underneath. ExecHelm is the default, but
	// any implementation of the Helm interface may be used
	Helm Helm
}

// ChartDir returns the directory pulled charts are saved in
func (c Charts) ChartDir() string {
	return filepath.Join(c.projectRoot, c.Manifest.Directory)
}

// ManifestFile returns the full path to the chartfile.yaml
func (c Charts) ManifestFile() string {
	return filepath.Join(c.projectRoot, Filename)
}

// Vendor pulls all Charts specified in the manifest into the local charts
// directory. It fetches the repository index before doing so.
func (c Charts) Vendor() error {
	dir := c.ChartDir()
	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
		return err
	}

	log.Println("Syncing Repositories ...")
	if err := c.Helm.RepoUpdate(Opts{Repositories: c.Manifest.Repositories}); err != nil {
		return err
	}

	log.Println("Pulling Charts ...")
	for _, r := range c.Manifest.Requires {
		chartName := parseReqName(r.Chart)
		chartPath := filepath.Join(dir, chartName)
		if _, err := os.Stat(chartPath); err == nil {
			log.Printf(" %s@%s exists", r.Chart, r.Version.String())
			continue
		}

		err := c.Helm.Pull(r.Chart, r.Version.String(), PullOpts{
			Destination: dir,
			Opts:        Opts{Repositories: c.Manifest.Repositories},
		})
		if err != nil {
			return err
		}

		log.Printf(" %s@%s downloaded", r.Chart, r.Version.String())
	}

	return nil
}

// Add adds every Chart in reqs to the Manifest after validation, and runs
// Vendor afterwards
func (c Charts) Add(reqs []string) error {
	log.Printf("Adding %v Charts ...", len(reqs))

	skip := func(s string, err error) {
		log.Printf(" Skipping %s: %s.", s, err)
	}

	// parse new charts, append in memory
	added := 0
	for _, s := range reqs {
		r, err := parseReq(s)
		if err != nil {
			skip(s, err)
			continue
		}

		if c.Manifest.Requires.Has(*r) {
			skip(s, fmt.Errorf("already exists"))
			continue
		}

		c.Manifest.Requires = append(c.Manifest.Requires, *r)
		added++
		log.Println(" OK:", s)
	}

	// write out
	if err := write(c.Manifest, c.ManifestFile()); err != nil {
		return err
	}

	// skipped some? fail then
	if added != len(reqs) {
		return fmt.Errorf("%v Charts were skipped. Please check above logs for details", len(reqs)-added)
	}

	// worked fine? vendor it
	log.Printf("Added %v Charts to helmfile.yaml. Vendoring ...", added)
	return c.Vendor()
}

func InitChartfile(path string) (*Charts, error) {
	c := Chartfile{
		Version: Version,
		Repositories: []Repo{{
			Name: "stable",
			URL:  "https://charts.helm.sh/stable",
		}},
		Requires: make(Requirements, 0),
	}

	if err := write(c, path); err != nil {
		return nil, err
	}

	return LoadChartfile(filepath.Dir(path))
}

// write saves a Chartfile to dest
func write(c Chartfile, dest string) error {
	data, err := yaml.Marshal(c)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(dest, data, 0644)
}

var chartExp = regexp.MustCompile(`\w+\/.+@.+`)

// parseReq parses a requirement from a string of the format `repo/name@version`
func parseReq(s string) (*Requirement, error) {
	if !chartExp.MatchString(s) {
		return nil, fmt.Errorf("not of form 'repo/chart@version'")
	}

	elems := strings.Split(s, "@")
	chart := elems[0]
	ver, err := semver.NewVersion(elems[1])
	if errors.Is(err, semver.ErrInvalidSemVer) {
		return nil, fmt.Errorf("version is invalid")
	} else if err != nil {
		return nil, fmt.Errorf("version is invalid: %s", err)
	}

	return &Requirement{
		Chart:   chart,
		Version: *ver,
	}, nil
}

// parseReqName parses a name from a string of the format `repo/name`
func parseReqName(s string) string {
	elems := strings.Split(s, "/")
	name := elems[1]
	return name
}
<commit_msg>feat(tool/charts): Check chart versions and update accordingly (#420)<commit_after>package helm

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/Masterminds/semver"
	"sigs.k8s.io/yaml"
)

// LoadChartfile opens a Chartfile tree
func LoadChartfile(projectRoot string) (*Charts, error) {
	// make sure project root is valid
	abs, err := filepath.Abs(projectRoot)
	if err != nil {
		return nil, err
	}

	// open chartfile
	chartfile := filepath.Join(abs, Filename)
	data, err := ioutil.ReadFile(chartfile)
	if err != nil {
		return nil, err
	}

	// parse it
	c := Chartfile{
		Version:   Version,
		Directory: DefaultDir,
	}
	if err := yaml.UnmarshalStrict(data, &c); err != nil {
		return nil, err
	}

	for i, r := range c.Requires {
		if r.Chart == "" {
			return nil, fmt.Errorf("requirements[%v]: 'chart' must be set", i)
		}
	}

	// return Charts handle
	charts := &Charts{
		Manifest:    c,
		projectRoot: abs,

		// default to ExecHelm, but allow injecting from the outside
		Helm: ExecHelm{},
	}
	return charts, nil
}

// Charts exposes the central Chartfile management functions
type Charts struct {
	// Manifest are the chartfile.yaml contents. It holds data about the developers intentions
	Manifest Chartfile

	// projectRoot is the enclosing directory of chartfile.yaml
	projectRoot string

	// Helm is the helm implementation underneath. ExecHelm is the default, but
	// any implementation of the Helm interface may be used
	Helm Helm
}

// chartManifest represents a Helm chart's Chart.yaml
type chartManifest struct {
	Version string `yaml:"version"`
}

// ChartDir returns the directory pulled charts are saved in
func (c Charts) ChartDir() string {
	return filepath.Join(c.projectRoot, c.Manifest.Directory)
}

// ManifestFile returns the full path to the chartfile.yaml
func (c Charts) ManifestFile() string {
	return filepath.Join(c.projectRoot, Filename)
}

// Vendor pulls all Charts specified in the manifest into the local charts
// directory. It fetches the repository index before doing so.
func (c Charts) Vendor() error {
	dir := c.ChartDir()
	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
		return err
	}

	log.Println("Syncing Repositories ...")
	if err := c.Helm.RepoUpdate(Opts{Repositories: c.Manifest.Repositories}); err != nil {
		return err
	}

	log.Println("Pulling Charts ...")
	for _, r := range c.Manifest.Requires {
		chartName := parseReqName(r.Chart)
		chartPath := filepath.Join(dir, chartName)

		_, err := os.Stat(chartPath)
		if err == nil {
			chartManifestPath := filepath.Join(chartPath, "Chart.yaml")
			chartManifestBytes, err := ioutil.ReadFile(chartManifestPath)
			if err != nil {
				return fmt.Errorf("reading chart manifest: %w", err)
			}
			var chartYAML chartManifest
			if err := yaml.Unmarshal(chartManifestBytes, &chartYAML); err != nil {
				return fmt.Errorf("unmarshalling chart manifest: %w", err)
			}

			if chartYAML.Version == r.Version.String() {
				log.Printf(" %s@%s exists", r.Chart, r.Version.String())
				continue
			} else {
				log.Printf("Removing %s@%s", r.Chart, r.Version.String())
				if err := os.RemoveAll(chartPath); err != nil {
					return err
				}
			}
		} else if !os.IsNotExist(err) {
			return err
		}

		err = c.Helm.Pull(r.Chart, r.Version.String(), PullOpts{
			Destination: dir,
			Opts:        Opts{Repositories: c.Manifest.Repositories},
		})
		if err != nil {
			return err
		}

		log.Printf(" %s@%s downloaded", r.Chart, r.Version.String())
	}

	return nil
}

// Add adds every Chart in reqs to the Manifest after validation, and runs
// Vendor afterwards
func (c Charts) Add(reqs []string) error {
	log.Printf("Adding %v Charts ...", len(reqs))

	skip := func(s string, err error) {
		log.Printf(" Skipping %s: %s.", s, err)
	}

	// parse new charts, append in memory
	added := 0
	for _, s := range reqs {
		r, err := parseReq(s)
		if err != nil {
			skip(s, err)
			continue
		}

		if c.Manifest.Requires.Has(*r) {
			skip(s, fmt.Errorf("already exists"))
			continue
		}

		c.Manifest.Requires = append(c.Manifest.Requires, *r)
		added++
		log.Println(" OK:", s)
	}

	// write out
	if err := write(c.Manifest, c.ManifestFile()); err != nil {
		return err
	}

	// skipped some? fail then
	if added != len(reqs) {
		return fmt.Errorf("%v Charts were skipped. Please check above logs for details", len(reqs)-added)
	}

	// worked fine? vendor it
	log.Printf("Added %v Charts to helmfile.yaml. Vendoring ...", added)
	return c.Vendor()
}

func InitChartfile(path string) (*Charts, error) {
	c := Chartfile{
		Version: Version,
		Repositories: []Repo{{
			Name: "stable",
			URL:  "https://charts.helm.sh/stable",
		}},
		Requires: make(Requirements, 0),
	}

	if err := write(c, path); err != nil {
		return nil, err
	}

	return LoadChartfile(filepath.Dir(path))
}

// write saves a Chartfile to dest
func write(c Chartfile, dest string) error {
	data, err := yaml.Marshal(c)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(dest, data, 0644)
}

var chartExp = regexp.MustCompile(`\w+\/.+@.+`)

// parseReq parses a requirement from a string of the format `repo/name@version`
func parseReq(s string) (*Requirement, error) {
	if !chartExp.MatchString(s) {
		return nil, fmt.Errorf("not of form 'repo/chart@version'")
	}

	elems := strings.Split(s, "@")
	chart := elems[0]
	ver, err := semver.NewVersion(elems[1])
	if errors.Is(err, semver.ErrInvalidSemVer) {
		return nil, fmt.Errorf("version is invalid")
	} else if err != nil {
		return nil, fmt.Errorf("version is invalid: %s", err)
	}

	return &Requirement{
		Chart:   chart,
		Version: *ver,
	}, nil
}

// parseReqName parses a name from a string of the format `repo/name`
func parseReqName(s string) string {
	elems := strings.Split(s, "/")
	name := elems[1]
	return name
}
<|endoftext|>


<commit_before>// Package pools provides a collection of pools which provide various
// data types with buffers. These can be used to lower the number of
// memory allocations and reuse buffers.
//
// New pools should be added to this package to allow them to be
// shared across packages.
//
// Utility functions which operate on pools should be added to this
// package to allow them to be reused.
package pools

import (
	"bufio"
	"io"
	"sync"

	"github.com/docker/docker/pkg/ioutils"
)

var (
	// BufioReader32KPool is a pool which returns bufio.Reader with a 32K buffer.
	BufioReader32KPool *BufioReaderPool
	// BufioWriter32KPool is a pool which returns bufio.Writer with a 32K buffer.
	BufioWriter32KPool *BufioWriterPool
)

const buffer32K = 32 * 1024

// BufioReaderPool is a bufio reader that uses sync.Pool.
type BufioReaderPool struct {
	pool sync.Pool
}

func init() {
	BufioReader32KPool = newBufioReaderPoolWithSize(buffer32K)
	BufioWriter32KPool = newBufioWriterPoolWithSize(buffer32K)
}

// newBufioReaderPoolWithSize is unexported because new pools should be
// added here to be shared where required.
func newBufioReaderPoolWithSize(size int) *BufioReaderPool {
	pool := sync.Pool{
		New: func() interface{} { return bufio.NewReaderSize(nil, size) },
	}
	return &BufioReaderPool{pool: pool}
}

// Get returns a bufio.Reader which reads from r. The buffer size is that of the pool.
func (bufPool *BufioReaderPool) Get(r io.Reader) *bufio.Reader {
	buf := bufPool.pool.Get().(*bufio.Reader)
	buf.Reset(r)
	return buf
}

// Put puts the bufio.Reader back into the pool.
func (bufPool *BufioReaderPool) Put(b *bufio.Reader) {
	b.Reset(nil)
	bufPool.pool.Put(b)
}

// Copy is a convenience wrapper which uses a buffer to avoid allocation in io.Copy.
func Copy(dst io.Writer, src io.Reader) (written int64, err error) {
	buf := BufioReader32KPool.Get(src)
	written, err = io.Copy(dst, buf)
	BufioReader32KPool.Put(buf)
	return
}

// NewReadCloserWrapper returns a wrapper which puts the bufio.Reader back
// into the pool and closes the reader if it's an io.ReadCloser.
func (bufPool *BufioReaderPool) NewReadCloserWrapper(buf *bufio.Reader, r io.Reader) io.ReadCloser {
	return ioutils.NewReadCloserWrapper(r, func() error {
		if readCloser, ok := r.(io.ReadCloser); ok {
			readCloser.Close()
		}
		bufPool.Put(buf)
		return nil
	})
}

// BufioWriterPool is a bufio writer that uses sync.Pool.
type BufioWriterPool struct {
	pool sync.Pool
}

// newBufioWriterPoolWithSize is unexported because new pools should be
// added here to be shared where required.
func newBufioWriterPoolWithSize(size int) *BufioWriterPool {
	pool := sync.Pool{
		New: func() interface{} { return bufio.NewWriterSize(nil, size) },
	}
	return &BufioWriterPool{pool: pool}
}

// Get returns a bufio.Writer which writes to w. The buffer size is that of the pool.
func (bufPool *BufioWriterPool) Get(w io.Writer) *bufio.Writer {
	buf := bufPool.pool.Get().(*bufio.Writer)
	buf.Reset(w)
	return buf
}

// Put puts the bufio.Writer back into the pool.
func (bufPool *BufioWriterPool) Put(b *bufio.Writer) {
	b.Reset(nil)
	bufPool.pool.Put(b)
}

// NewWriteCloserWrapper returns a wrapper which puts the bufio.Writer back
// into the pool and closes the writer if it's an io.Writecloser.
func (bufPool *BufioWriterPool) NewWriteCloserWrapper(buf *bufio.Writer, w io.Writer) io.WriteCloser {
	return ioutils.NewWriteCloserWrapper(w, func() error {
		buf.Flush()
		if writeCloser, ok := w.(io.WriteCloser); ok {
			writeCloser.Close()
		}
		bufPool.Put(buf)
		return nil
	})
}
<commit_msg>pkg/pools: avoid copy of sync.Pool<commit_after>// Package pools provides a collection of pools which provide various
// data types with buffers. These can be used to lower the number of
// memory allocations and reuse buffers.
//
// New pools should be added to this package to allow them to be
// shared across packages.
//
// Utility functions which operate on pools should be added to this
// package to allow them to be reused.
package pools

import (
	"bufio"
	"io"
	"sync"

	"github.com/docker/docker/pkg/ioutils"
)

var (
	// BufioReader32KPool is a pool which returns bufio.Reader with a 32K buffer.
	BufioReader32KPool *BufioReaderPool
	// BufioWriter32KPool is a pool which returns bufio.Writer with a 32K buffer.
	BufioWriter32KPool *BufioWriterPool
)

const buffer32K = 32 * 1024

// BufioReaderPool is a bufio reader that uses sync.Pool.
type BufioReaderPool struct {
	pool *sync.Pool
}

func init() {
	BufioReader32KPool = newBufioReaderPoolWithSize(buffer32K)
	BufioWriter32KPool = newBufioWriterPoolWithSize(buffer32K)
}

// newBufioReaderPoolWithSize is unexported because new pools should be
// added here to be shared where required.
func newBufioReaderPoolWithSize(size int) *BufioReaderPool {
	pool := &sync.Pool{
		New: func() interface{} { return bufio.NewReaderSize(nil, size) },
	}
	return &BufioReaderPool{pool: pool}
}

// Get returns a bufio.Reader which reads from r. The buffer size is that of the pool.
func (bufPool *BufioReaderPool) Get(r io.Reader) *bufio.Reader {
	buf := bufPool.pool.Get().(*bufio.Reader)
	buf.Reset(r)
	return buf
}

// Put puts the bufio.Reader back into the pool.
func (bufPool *BufioReaderPool) Put(b *bufio.Reader) {
	b.Reset(nil)
	bufPool.pool.Put(b)
}

// Copy is a convenience wrapper which uses a buffer to avoid allocation in io.Copy.
func Copy(dst io.Writer, src io.Reader) (written int64, err error) {
	buf := BufioReader32KPool.Get(src)
	written, err = io.Copy(dst, buf)
	BufioReader32KPool.Put(buf)
	return
}

// NewReadCloserWrapper returns a wrapper which puts the bufio.Reader back
// into the pool and closes the reader if it's an io.ReadCloser.
func (bufPool *BufioReaderPool) NewReadCloserWrapper(buf *bufio.Reader, r io.Reader) io.ReadCloser {
	return ioutils.NewReadCloserWrapper(r, func() error {
		if readCloser, ok := r.(io.ReadCloser); ok {
			readCloser.Close()
		}
		bufPool.Put(buf)
		return nil
	})
}

// BufioWriterPool is a bufio writer that uses sync.Pool.
type BufioWriterPool struct {
	pool *sync.Pool
}

// newBufioWriterPoolWithSize is unexported because new pools should be
// added here to be shared where required.
func newBufioWriterPoolWithSize(size int) *BufioWriterPool {
	pool := &sync.Pool{
		New: func() interface{} { return bufio.NewWriterSize(nil, size) },
	}
	return &BufioWriterPool{pool: pool}
}

// Get returns a bufio.Writer which writes to w. The buffer size is that of the pool.
func (bufPool *BufioWriterPool) Get(w io.Writer) *bufio.Writer {
	buf := bufPool.pool.Get().(*bufio.Writer)
	buf.Reset(w)
	return buf
}

// Put puts the bufio.Writer back into the pool.
func (bufPool *BufioWriterPool) Put(b *bufio.Writer) {
	b.Reset(nil)
	bufPool.pool.Put(b)
}

// NewWriteCloserWrapper returns a wrapper which puts the bufio.Writer back
// into the pool and closes the writer if it's an io.Writecloser.
func (bufPool *BufioWriterPool) NewWriteCloserWrapper(buf *bufio.Writer, w io.Writer) io.WriteCloser {
	return ioutils.NewWriteCloserWrapper(w, func() error {
		buf.Flush()
		if writeCloser, ok := w.(io.WriteCloser); ok {
			writeCloser.Close()
		}
		bufPool.Put(buf)
		return nil
	})
}
<|endoftext|>


<commit_before>package logmux

import (
	"errors"
	"net"
	"sync"
	"time"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/technoweenie/grohl"

	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/stream"
)

// serviceConn dials and watches a connection to a discoverd service. Writes
// are blocked when the service is down and during a leader change.
type serviceConn struct {
	net.Conn
	cond *sync.Cond

	closed bool

	donec  chan struct{}
	closec chan struct{}
	errc   chan error
}

func connect(discd *discoverd.Client, name string, donec chan struct{}) (*serviceConn, error) {
	srv := discd.Service(name)
	eventc := make(chan *discoverd.Event)

	stream, err := srv.Watch(eventc)
	if err != nil {
		return nil, err
	}

	sc := &serviceConn{
		cond:   &sync.Cond{L: &sync.Mutex{}},
		donec:  donec,
		closec: make(chan struct{}),
		errc:   make(chan error),
	}

	if err := sc.connect(srv); err != nil {
		grohl.Log(grohl.Data{"service": name, "status": "connect-error", "err": err.Error()})
	}

	go sc.watch(srv, eventc, stream)
	return sc, nil
}

// Write writes data to the connection.
// Write blocks while the service is unavailable. Errors from the internal
// connection are returned.
func (c *serviceConn) Write(p []byte) (int, error) {
	c.cond.L.Lock()
	defer c.cond.L.Unlock()

	for {
		if c.Conn != nil {
			n, err := c.Conn.Write(p)
			if err == nil {
				return n, nil
			}
			if c.closed {
				return 0, errors.New("connection closed")
			}

			c.errc <- err
		}

		c.cond.Wait()
	}
}

func (c *serviceConn) watch(srv discoverd.Service, eventc <-chan *discoverd.Event, stream stream.Stream) {
	g := grohl.NewContext(grohl.Data{"at": "logmux_service_watch"})

	var (
		resetc                  = make(chan time.Time)
		reconc <-chan time.Time = resetc
	)
	defer close(resetc)

	for {
		select {
		case event, ok := <-eventc:
			if !ok {
				c.hangup()
				return
			}
			g.Log(grohl.Data{"status": "event", "event": event.Kind.String()})

			switch event.Kind {
			case discoverd.EventKindLeader:
				reconc = resetc

				if err := c.reset(); err != nil {
					g.Log(grohl.Data{"status": "error", "err": err.Error()})
				}

				if err := c.connect(srv); err != nil {
					g.Log(grohl.Data{"status": "error", "err": err.Error()})
					reconc = time.After(100 * time.Millisecond)
				}
			default:
			}
		case err := <-c.errc:
			g.Log(grohl.Data{"status": "write-error", "err": err.Error()})
			reconc = resetc

			if err := c.reset(); err != nil {
				g.Log(grohl.Data{"status": "error", "err": err.Error()})
			}

			if err := c.connect(srv); err != nil {
				g.Log(grohl.Data{"status": "error", "err": err.Error()})
				reconc = time.After(100 * time.Millisecond)
			}
		case <-reconc:
			if err := c.connect(srv); err != nil {
				g.Log(grohl.Data{"status": "reconnect-error", "err": err.Error()})
				reconc = time.After(100 * time.Millisecond)
			}
		case <-c.donec:
			if err := stream.Close(); err != nil {
				g.Log(grohl.Data{"status": "error", "err": err.Error()})
			}
			if err := c.reset(); err != nil {
				g.Log(grohl.Data{"status": "error", "err": err.Error()})
			}

			return
		case <-c.closec:
			if err := stream.Close(); err != nil {
				g.Log(grohl.Data{"status": "error", "err": err.Error()})
			}

			c.hangup()
			return
		}
	}
}

func (c *serviceConn) connect(srv discoverd.Service) error {
	ldr, err := srv.Leader()
	if err != nil {
		return err
	}

	conn, err := net.Dial("tcp", ldr.Addr)
	if err != nil {
		return err
	}

	c.cond.L.Lock()
	c.Conn = conn
	c.cond.L.Unlock()
	c.cond.Broadcast()

	return nil
}

func (c *serviceConn) reset() error {
	c.cond.L.Lock()
	conn := c.Conn
	c.Conn = nil
	c.cond.L.Unlock()

	if conn == nil {
		return nil
	}
	return conn.Close()
}

func (c *serviceConn) hangup() {
	c.reset()

	c.cond.L.Lock()
	c.closed = true
	c.cond.L.Unlock()
	c.cond.Broadcast()
}
<commit_msg>logmux: fix check for closed serviceConn<commit_after>package logmux

import (
	"errors"
	"net"
	"sync"
	"time"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/technoweenie/grohl"

	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/stream"
)

// serviceConn dials and watches a connection to a discoverd service. Writes
// are blocked when the service is down and during a leader change.
type serviceConn struct {
	net.Conn
	cond *sync.Cond

	closed bool

	donec  chan struct{}
	closec chan struct{}
	errc   chan error
}

func connect(discd *discoverd.Client, name string, donec chan struct{}) (*serviceConn, error) {
	srv := discd.Service(name)
	eventc := make(chan *discoverd.Event)

	stream, err := srv.Watch(eventc)
	if err != nil {
		return nil, err
	}

	sc := &serviceConn{
		cond:   &sync.Cond{L: &sync.Mutex{}},
		donec:  donec,
		closec: make(chan struct{}),
		errc:   make(chan error),
	}

	if err := sc.connect(srv); err != nil {
		grohl.Log(grohl.Data{"service": name, "status": "connect-error", "err": err.Error()})
	}

	go sc.watch(srv, eventc, stream)
	return sc, nil
}

// Write writes data to the connection.
// Write blocks while the service is unavailable. Errors from the internal
// connection are returned.
func (c *serviceConn) Write(p []byte) (int, error) {
	c.cond.L.Lock()
	defer c.cond.L.Unlock()

	for {
		if c.closed {
			return 0, errors.New("connection closed")
		}

		if c.Conn != nil {
			n, err := c.Conn.Write(p)
			if err == nil {
				return n, nil
			}

			c.errc <- err
		}

		c.cond.Wait()
	}
}

func (c *serviceConn) watch(srv discoverd.Service, eventc <-chan *discoverd.Event, stream stream.Stream) {
	g := grohl.NewContext(grohl.Data{"at": "logmux_service_watch"})

	var (
		resetc                  = make(chan time.Time)
		reconc <-chan time.Time = resetc
	)
	defer close(resetc)

	for {
		select {
		case event, ok := <-eventc:
			if !ok {
				c.hangup()
				return
			}
			g.Log(grohl.Data{"status": "event", "event": event.Kind.String()})

			switch event.Kind {
			case discoverd.EventKindLeader:
				reconc = resetc

				if err := c.reset(); err != nil {
					g.Log(grohl.Data{"status": "error", "err": err.Error()})
				}

				if err := c.connect(srv); err != nil {
					g.Log(grohl.Data{"status": "error", "err": err.Error()})
					reconc = time.After(100 * time.Millisecond)
				}
			default:
			}
		case err := <-c.errc:
			g.Log(grohl.Data{"status": "write-error", "err": err.Error()})
			reconc = resetc

			if err := c.reset(); err != nil {
				g.Log(grohl.Data{"status": "error", "err": err.Error()})
			}

			if err := c.connect(srv); err != nil {
				g.Log(grohl.Data{"status": "error", "err": err.Error()})
				reconc = time.After(100 * time.Millisecond)
			}
		case <-reconc:
			if err := c.connect(srv); err != nil {
				g.Log(grohl.Data{"status": "reconnect-error", "err": err.Error()})
				reconc = time.After(100 * time.Millisecond)
			}
		case <-c.donec:
			if err := stream.Close(); err != nil {
				g.Log(grohl.Data{"status": "error", "err": err.Error()})
			}
			if err := c.reset(); err != nil {
				g.Log(grohl.Data{"status": "error", "err": err.Error()})
			}

			return
		case <-c.closec:
			if err := stream.Close(); err != nil {
				g.Log(grohl.Data{"status": "error", "err": err.Error()})
			}

			c.hangup()
			return
		}
	}
}

func (c *serviceConn) connect(srv discoverd.Service) error {
	ldr, err := srv.Leader()
	if err != nil {
		return err
	}

	conn, err := net.Dial("tcp", ldr.Addr)
	if err != nil {
		return err
	}

	c.cond.L.Lock()
	c.Conn = conn
	c.cond.L.Unlock()
	c.cond.Broadcast()

	return nil
}

func (c *serviceConn) reset() error {
	c.cond.L.Lock()
	conn := c.Conn
	c.Conn = nil
	c.cond.L.Unlock()

	if conn == nil {
		return nil
	}
	return conn.Close()
}

func (c *serviceConn) hangup() {
	c.reset()

	c.cond.L.Lock()
	c.closed = true
	c.cond.L.Unlock()
	c.cond.Broadcast()
}
<|endoftext|>