<commit_before>package verbo

import (
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
	"math"
	"regexp"
	"strings"
	"strconv"
	"unicode"
	"unicode/utf8"
)

func Camelize(str string, decapitalize bool) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`[-_\s]+(.)?`)
	str = re.ReplaceAllStringFunc(str, func(c string) string {
		if c != "" {
			out := re.FindStringSubmatch(c)
			return strings.ToUpper(out[1])
		} else {
			return ""
		}
	})

	if decapitalize {
		return Decapitalize(str)
	} else {
		return str
	}
}

func Capitalize(str string, lowercaseRest bool) string {
	remainingChars := ""
	if !lowercaseRest {
		remainingChars = str[1:]
	} else {
		remainingChars = strings.ToLower(str[1:])
	}
	return strings.ToUpper(string(str[0])) + remainingChars
}

func Chop(str string, step int) []string {
	if str == "" {
		var a []string
		return a
	}

	step = int(math.Floor(float64(step)))
	if step > 0 {
		re := regexp.MustCompile(".{1," + strconv.Itoa(step) + "}")
		return re.FindAllString(str, -1)
	} else {
		a := []string{str}
		return a
	}
}

func Classify(str string) string {
	re := regexp.MustCompile(`[\W_]`)
	str = re.ReplaceAllString(str, " ")
	return Capitalize(Camelize(str, false), false)
}

func Clean(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`\s\s+`)
	return re.ReplaceAllString(str, " ")
}

func CleanDiacritics(str string) string {
	isMn := func(r rune) bool {
		return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
	}
	t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
	result, _, _ := transform.String(t, str)
	return result
}

func Dasherize(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile("([A-Z])")
	str = re.ReplaceAllString(str, "-$1")
	re = regexp.MustCompile(`[-_\s]+`)
	str = re.ReplaceAllString(str, "-")
	return strings.ToLower(str)
}

func Decapitalize(str string) string {
	return strings.ToLower(string(str[0])) + str[1:]
}

func Humanize(str string) string {
	str = Underscored(str)
	re := regexp.MustCompile(`_id$`)
	str = re.ReplaceAllString(str, "")
	re = regexp.MustCompile(`_`)
	str = re.ReplaceAllString(str, " ")
	return Capitalize(strings.TrimSpace(str), false)
}

func IsBlank(str string) bool {
	re := regexp.MustCompile(`^\s*$`)
	return re.MatchString(str)
}

func LeftPad(str string, length int, padStr string) string {
  return Pad(str, length, padStr, "")
}

func Levenshtein(str1, str2 string) int {

	// Short cut cases
	if str1 == str2 {
		return 0
	}
	if str1 == "" || str2 == "" {
		return int(math.Max(float64(utf8.RuneCountInString(str1)), float64(utf8.RuneCountInString(str2))))
	}

	// two rows
	size := utf8.RuneCountInString(str2) + 1
	prevRow := make([]int, size)

	// initialise previous row
	i := 0
	for i < len(prevRow) {
		prevRow[i] = i
		i += 1
	}

	nextCol := 0

	// calculate current row distance from previous row
	i = 0
	for i < utf8.RuneCountInString(str1) {
		nextCol = i + 1
		j := 0
		for j < utf8.RuneCountInString(str2) {
			curCol := nextCol

			// substution
			val := 0
			if str1[i] == str2[j] {
				val = 0
			} else {
				val = 1
			}

			nextCol := prevRow[j] + val

			// insertion
			tmp := curCol + 1
			if nextCol > tmp {
				nextCol = tmp
			}
			// deletion
			tmp = prevRow[j+1] + 1
			if nextCol > tmp {
				nextCol = tmp
			}

			// copy current col value into previous (in preparation for next iteration)
			prevRow[j] = curCol
			j += 1
		}
		// copy last col value into previous (in preparation for next iteration)
		prevRow[j] = nextCol
		i += 1
	}

	return nextCol
}

func Lines(str string) []string {
	if str == "" {
		var a []string
		return a
	}
	re := regexp.MustCompile(`\r\n?|\n`)
	return re.Split(str, -1)
}

func Pad(str string, length int, padStr, padtype string) string {
  length = int(math.Floor(float64(length)))

  var padlen = 0

  if padStr == "" {
    padStr = " "
  } else if utf8.RuneCountInString(padStr) > 1 {
    padStr = string(padStr[0])
	}

  switch padtype {
  case "right":
    padlen = length - utf8.RuneCountInString(str)
    return str + strRepeat(padStr, padlen)
  case "both":
    padlen = length - utf8.RuneCountInString(str)
    return strRepeat(padStr, int(math.Ceil(float64(padlen) / 2))) + str + strRepeat(padStr, int(math.Floor(float64(padlen) / 2)))
  default: // 'left'
    padlen = length - utf8.RuneCountInString(str)
    return strRepeat(padStr, padlen) + str
  }
}

func Pred(str string) string {
	return adjacent(str, -1)
}

func Prune(str string, length int, pruneStr string) string {
	length = int(math.Floor(float64(length)))
	if pruneStr != "" {
		pruneStr = pruneStr
	} else {
		pruneStr = "..."
	}

	if utf8.RuneCountInString(str) <= length {
		return str
	}

	tmpl := func(c string) string {
		if strings.ToUpper(c) != strings.ToLower(c) {
			return "A"
		} else {
			return " "
		}
	}

	re := regexp.MustCompile(`.(?=\W*\w*$)`)
	template := re.ReplaceAllStringFunc(str[0:length+1], tmpl) // 'Hello, world' -> 'HellAA AAAAA'

	re = regexp.MustCompile(`\w\w`)
	if re.MatchString(template[utf8.RuneCountInString(template)-2:]) {
		re = regexp.MustCompile(`\s*\S+$`)
		template = re.ReplaceAllString(template, "")
	} else {
		template = strings.TrimRight(template[0:utf8.RuneCountInString(template)-1], " ")
	}

	if utf8.RuneCountInString(template+pruneStr) > utf8.RuneCountInString(str) {
		return str
	} else {
		return str[0:utf8.RuneCountInString(template)] + pruneStr
	}
}

func Repeat(str string, qty int, separator string) string {
	qty = int(math.Floor(float64(qty)))

	// using faster implementation if separator is not needed;
	if separator == "" {
		return strRepeat(str, qty)
	}

	var repeat []string
	for qty > 0 {
		repeat[qty] = str
	}
	return strings.Join(repeat, separator)
}

func Reverse(str string) string {
	n := utf8.RuneCountInString(str)
	runes := make([]rune, n)
	for _, rune := range str {
		n--
		runes[n] = rune
	}
	return string(runes[n:])
}

func RightPad(str string, length int, padStr string) string {
  return Pad(str, length, padStr, "right");
}

func Slugify(str string) string {
	re := regexp.MustCompile(`[^\w\s-]`)
	str = re.ReplaceAllString(str, "-")
	str = strings.ToLower(str)
	str = CleanDiacritics(str)
	return strings.Trim(Dasherize(str), "-")
}

/*
func Splice(str string, i, howmany int, substr string) (string) {
  arr := chars(str)
  arr.splice(^i, ^howmany, substr);
  return strings.Join(arr, "")
}
*/

func Succ(str string) string {
	return adjacent(str, 1)
}

func SwapCase(str string) string {
	re := regexp.MustCompile("\\S")
	return re.ReplaceAllStringFunc(str, func(c string) string {
		if c == strings.ToUpper(c) {
			return strings.ToLower(c)
		} else {
			return strings.ToUpper(c)
		}
	})
}

func Titleize(str string) string {
	str = strings.ToLower(str)
	re := regexp.MustCompile("(?:^|\\s|-)\\S")
	return re.ReplaceAllStringFunc(str, func(c string) string {
		return strings.ToUpper(c)
	})
}

func Trim(str, characters string) string {
  if characters == "" {
		return strings.TrimSpace(str)
	}

	characters = defaultToWhiteSpace(characters)
	re := regexp.MustCompile("^" + characters + "+|" + characters + "+$")
	return re.ReplaceAllString(str, "")
}

func Truncate(str string, length int, truncateStr string) string {
	if truncateStr == "" {
		truncateStr = "..."
	}
	length = int(math.Floor(float64(length)))
	if utf8.RuneCountInString(str) > length {
		return str[0:length] + truncateStr
	} else {
		return str
	}
}

func Underscored(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`([a-z\d])([A-Z]+)`)
	str = re.ReplaceAllString(str, `$1-$2`)
	re = regexp.MustCompile(`[-\s]+`)
	str = re.ReplaceAllString(str, "_")
	return strings.ToLower(str)
}

func Unquote(str, quoteChar string) string {
	if quoteChar == "" {
		quoteChar = "\""
	}
	if string(str[0]) == quoteChar &&
		string(str[utf8.RuneCountInString(str)-1]) == quoteChar {
		return str[1 : utf8.RuneCountInString(str)-1]
	} else {
		return str
	}
}

func Words(str, delimiter string) []string {
	if IsBlank(str) {
		var a []string
		return a
	}
	if delimiter == "" {
		delimiter = `\s+`
	}
	re := regexp.MustCompile(delimiter)
	return re.Split(Trim(str, delimiter), -1)
}
<commit_msg>Fixing chop implmentation<commit_after>package verbo

import (
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
	"math"
	"regexp"
	"strings"
	"strconv"
	"unicode"
	"unicode/utf8"
)

func Camelize(str string, decapitalize bool) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`[-_\s]+(.)?`)
	str = re.ReplaceAllStringFunc(str, func(c string) string {
		if c != "" {
			out := re.FindStringSubmatch(c)
			return strings.ToUpper(out[1])
		} else {
			return ""
		}
	})

	if decapitalize {
		return Decapitalize(str)
	} else {
		return str
	}
}

func Capitalize(str string, lowercaseRest bool) string {
	remainingChars := ""
	if !lowercaseRest {
		remainingChars = str[1:]
	} else {
		remainingChars = strings.ToLower(str[1:])
	}
	return strings.ToUpper(string(str[0])) + remainingChars
}

func Chop(str string, step int) []string {
	if str == "" {
		var a []string
		return a
	}

	step = int(math.Floor(float64(step)))
	if step > 0 {
		re := regexp.MustCompile(".{1," + strconv.Itoa(step) + "}")
		return re.FindAllString(str, -1)
	} else {
		a := []string{str}
		return a
	}
}

func Classify(str string) string {
	re := regexp.MustCompile(`[\W_]`)
	str = re.ReplaceAllString(str, " ")
	return Capitalize(Camelize(str, false), false)
}

func Clean(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`\s\s+`)
	return re.ReplaceAllString(str, " ")
}

func CleanDiacritics(str string) string {
	isMn := func(r rune) bool {
		return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
	}
	t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
	result, _, _ := transform.String(t, str)
	return result
}

func Dasherize(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile("([A-Z])")
	str = re.ReplaceAllString(str, "-$1")
	re = regexp.MustCompile(`[-_\s]+`)
	str = re.ReplaceAllString(str, "-")
	return strings.ToLower(str)
}

func Decapitalize(str string) string {
	return strings.ToLower(string(str[0])) + str[1:]
}

func Humanize(str string) string {
	str = Underscored(str)
	re := regexp.MustCompile(`_id$`)
	str = re.ReplaceAllString(str, "")
	re = regexp.MustCompile(`_`)
	str = re.ReplaceAllString(str, " ")
	return Capitalize(strings.TrimSpace(str), false)
}

func IsBlank(str string) bool {
	re := regexp.MustCompile(`^\s*$`)
	return re.MatchString(str)
}

func LeftPad(str string, length int, padStr string) string {
  return Pad(str, length, padStr, "")
}

func Levenshtein(str1, str2 string) int {

	// Short cut cases
	if str1 == str2 {
		return 0
	}
	if str1 == "" || str2 == "" {
		return int(math.Max(float64(utf8.RuneCountInString(str1)), float64(utf8.RuneCountInString(str2))))
	}

	// two rows
	size := utf8.RuneCountInString(str2) + 1
	prevRow := make([]int, size)

	// initialise previous row
	i := 0
	for i < len(prevRow) {
		prevRow[i] = i
		i += 1
	}

	nextCol := 0

	// calculate current row distance from previous row
	i = 0
	for i < utf8.RuneCountInString(str1) {
		nextCol = i + 1
		j := 0
		for j < utf8.RuneCountInString(str2) {
			curCol := nextCol

			// substution
			val := 0
			if str1[i] == str2[j] {
				val = 0
			} else {
				val = 1
			}

			nextCol = prevRow[j] + val

			// insertion
			tmp := curCol + 1
			if nextCol > tmp {
				nextCol = tmp
			}
			// deletion
			tmp = prevRow[j+1] + 1
			if nextCol > tmp {
				nextCol = tmp
			}

			// copy current col value into previous (in preparation for next iteration)
			prevRow[j] = curCol
			j += 1
		}
		// copy last col value into previous (in preparation for next iteration)
		prevRow[j] = nextCol
		i += 1
	}

	return nextCol
}

func Lines(str string) []string {
	if str == "" {
		var a []string
		return a
	}
	re := regexp.MustCompile(`\r\n?|\n`)
	return re.Split(str, -1)
}

func Pad(str string, length int, padStr, padtype string) string {
  length = int(math.Floor(float64(length)))

  var padlen = 0

  if padStr == "" {
    padStr = " "
  } else if utf8.RuneCountInString(padStr) > 1 {
    padStr = string(padStr[0])
	}

  switch padtype {
  case "right":
    padlen = length - utf8.RuneCountInString(str)
    return str + strRepeat(padStr, padlen)
  case "both":
    padlen = length - utf8.RuneCountInString(str)
    return strRepeat(padStr, int(math.Ceil(float64(padlen) / 2))) + str + strRepeat(padStr, int(math.Floor(float64(padlen) / 2)))
  default: // 'left'
    padlen = length - utf8.RuneCountInString(str)
    return strRepeat(padStr, padlen) + str
  }
}

func Pred(str string) string {
	return adjacent(str, -1)
}

func Prune(str string, length int, pruneStr string) string {
	length = int(math.Floor(float64(length)))
	if pruneStr != "" {
		pruneStr = pruneStr
	} else {
		pruneStr = "..."
	}

	if utf8.RuneCountInString(str) <= length {
		return str
	}

	tmpl := func(c string) string {
		if strings.ToUpper(c) != strings.ToLower(c) {
			return "A"
		} else {
			return " "
		}
	}

	re := regexp.MustCompile(`.(?=\W*\w*$)`)
	template := re.ReplaceAllStringFunc(str[0:length+1], tmpl) // 'Hello, world' -> 'HellAA AAAAA'

	re = regexp.MustCompile(`\w\w`)
	if re.MatchString(template[utf8.RuneCountInString(template)-2:]) {
		re = regexp.MustCompile(`\s*\S+$`)
		template = re.ReplaceAllString(template, "")
	} else {
		template = strings.TrimRight(template[0:utf8.RuneCountInString(template)-1], " ")
	}

	if utf8.RuneCountInString(template+pruneStr) > utf8.RuneCountInString(str) {
		return str
	} else {
		return str[0:utf8.RuneCountInString(template)] + pruneStr
	}
}

func Repeat(str string, qty int, separator string) string {
	qty = int(math.Floor(float64(qty)))

	// using faster implementation if separator is not needed;
	if separator == "" {
		return strRepeat(str, qty)
	}

	var repeat []string
	for qty > 0 {
		repeat[qty] = str
	}
	return strings.Join(repeat, separator)
}

func Reverse(str string) string {
	n := utf8.RuneCountInString(str)
	runes := make([]rune, n)
	for _, rune := range str {
		n--
		runes[n] = rune
	}
	return string(runes[n:])
}

func RightPad(str string, length int, padStr string) string {
  return Pad(str, length, padStr, "right");
}

func Slugify(str string) string {
	re := regexp.MustCompile(`[^\w\s-]`)
	str = re.ReplaceAllString(str, "-")
	str = strings.ToLower(str)
	str = CleanDiacritics(str)
	return strings.Trim(Dasherize(str), "-")
}

/*
func Splice(str string, i, howmany int, substr string) (string) {
  arr := chars(str)
  arr.splice(^i, ^howmany, substr);
  return strings.Join(arr, "")
}
*/

func Succ(str string) string {
	return adjacent(str, 1)
}

func SwapCase(str string) string {
	re := regexp.MustCompile("\\S")
	return re.ReplaceAllStringFunc(str, func(c string) string {
		if c == strings.ToUpper(c) {
			return strings.ToLower(c)
		} else {
			return strings.ToUpper(c)
		}
	})
}

func Titleize(str string) string {
	str = strings.ToLower(str)
	re := regexp.MustCompile("(?:^|\\s|-)\\S")
	return re.ReplaceAllStringFunc(str, func(c string) string {
		return strings.ToUpper(c)
	})
}

func Trim(str, characters string) string {
  if characters == "" {
		return strings.TrimSpace(str)
	}

	characters = defaultToWhiteSpace(characters)
	re := regexp.MustCompile("^" + characters + "+|" + characters + "+$")
	return re.ReplaceAllString(str, "")
}

func Truncate(str string, length int, truncateStr string) string {
	if truncateStr == "" {
		truncateStr = "..."
	}
	length = int(math.Floor(float64(length)))
	if utf8.RuneCountInString(str) > length {
		return str[0:length] + truncateStr
	} else {
		return str
	}
}

func Underscored(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`([a-z\d])([A-Z]+)`)
	str = re.ReplaceAllString(str, `$1-$2`)
	re = regexp.MustCompile(`[-\s]+`)
	str = re.ReplaceAllString(str, "_")
	return strings.ToLower(str)
}

func Unquote(str, quoteChar string) string {
	if quoteChar == "" {
		quoteChar = "\""
	}
	if string(str[0]) == quoteChar &&
		string(str[utf8.RuneCountInString(str)-1]) == quoteChar {
		return str[1 : utf8.RuneCountInString(str)-1]
	} else {
		return str
	}
}

func Words(str, delimiter string) []string {
	if IsBlank(str) {
		var a []string
		return a
	}
	if delimiter == "" {
		delimiter = `\s+`
	}
	re := regexp.MustCompile(delimiter)
	return re.Split(Trim(str, delimiter), -1)
}
<|endoftext|>


<commit_before>package dht

import (
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	inet "github.com/jbenet/go-ipfs/p2p/net"
)

// netNotifiee defines methods to be used with the IpfsDHT
type netNotifiee IpfsDHT

func (nn *netNotifiee) DHT() *IpfsDHT {
	return (*IpfsDHT)(nn)
}

func (nn *netNotifiee) Connected(n inet.Network, v inet.Conn) {
	dht := nn.DHT()
	select {
	case <-dht.Closing():
		return
	}
	dht.Update(dht.Context(), v.RemotePeer())
}

func (nn *netNotifiee) Disconnected(n inet.Network, v inet.Conn) {
	dht := nn.DHT()
	select {
	case <-dht.Closing():
		return
	}
	dht.routingTable.Remove(v.RemotePeer())
}

func (nn *netNotifiee) OpenedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) ClosedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) Listen(n inet.Network, a ma.Multiaddr)      {}
func (nn *netNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) {}
<commit_msg>dht/notif: bugfix in hanging connects<commit_after>package dht

import (
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	inet "github.com/jbenet/go-ipfs/p2p/net"
)

// netNotifiee defines methods to be used with the IpfsDHT
type netNotifiee IpfsDHT

func (nn *netNotifiee) DHT() *IpfsDHT {
	return (*IpfsDHT)(nn)
}

func (nn *netNotifiee) Connected(n inet.Network, v inet.Conn) {
	dht := nn.DHT()
	select {
	case <-dht.Closing():
		return
	default:
	}
	dht.Update(dht.Context(), v.RemotePeer())
}

func (nn *netNotifiee) Disconnected(n inet.Network, v inet.Conn) {
	dht := nn.DHT()
	select {
	case <-dht.Closing():
		return
	default:
	}
	dht.routingTable.Remove(v.RemotePeer())
}

func (nn *netNotifiee) OpenedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) ClosedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) Listen(n inet.Network, a ma.Multiaddr)      {}
func (nn *netNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) {}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package servicelabel

import (
	"fmt"

	"github.com/ligato/cn-infra/logging/logroot"
	"github.com/namsral/flag"
	"github.com/ligato/cn-infra/flavors/localdeps"
)

// Plugin exposes the service label(i.e. the string used to identify the particular VNF) to the other plugins.
type Plugin struct {
	// MicroserviceLabel identifies particular VNF.
	// Used primarily as a key prefix to ETCD data store.
	MicroserviceLabel string
}

// OfDifferentAgent sets micorserivce label and returns new instance of Plugin.
// It is meant for watching DB by prefix of different agent. You can pass/inject
// instance of this plugin for example to kvdbsync.
func OfDifferentAgent(microserviceLabel string) *Plugin {
	return &Plugin{MicroserviceLabel: microserviceLabel}
}

var microserviceLabelFlag string

func init() {
	flag.StringVar(&microserviceLabelFlag, "microservice-label", "vpp1", fmt.Sprintf("microservice label; also set via '%v' env variable.", MicroserviceLabelEnvVar))
}

// Init is called at plugin initialization.
func (p *Plugin) Init() error {
	if p.MicroserviceLabel == "" {
		p.MicroserviceLabel = microserviceLabelFlag
	}
	logroot.StandardLogger().Debugf("Microservice label is set to %v", p.MicroserviceLabel)
	return nil
}

// Close is called at plugin cleanup phase.
func (p *Plugin) Close() error {
	return nil
}

// GetAgentLabel returns string that is supposed to be used to distinguish
// (ETCD) key prefixes for particular VNF (particular VPP Agent configuration)
func (p *Plugin) GetAgentLabel() string {
	return p.MicroserviceLabel
}

// GetAgentPrefix returns the string that is supposed to be used as the prefix for configuration of current
// MicroserviceLabel "subtree" of the particular VPP Agent instance (e.g. in ETCD).
func (p *Plugin) GetAgentPrefix() string {
	return agentPrefix + p.MicroserviceLabel + "/"
}

// GetDifferentAgentPrefix returns the string that is supposed to be used as the prefix for configuration
// "subtree" of the particular VPP Agent instance (e.g. in ETCD).
func (p *Plugin) GetDifferentAgentPrefix(microserviceLabel string) string {
	return GetDifferentAgentPrefix(microserviceLabel)
}

// GetAllAgentsPrefix returns the string that is supposed to be used as the prefix for configuration
// subtree of the particular VPP Agent instance (e.g. in ETCD).
func (p *Plugin) GetAllAgentsPrefix() string {
	return GetAllAgentsPrefix()
}
<commit_msg>ODPM-554 examples/idx*cache now using OfDifferentAgent() from cn-infra<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package servicelabel

import (
	"fmt"

	"github.com/ligato/cn-infra/logging/logroot"
	"github.com/namsral/flag"
)

// Plugin exposes the service label(i.e. the string used to identify the particular VNF) to the other plugins.
type Plugin struct {
	// MicroserviceLabel identifies particular VNF.
	// Used primarily as a key prefix to ETCD data store.
	MicroserviceLabel string
}

// OfDifferentAgent sets micorserivce label and returns new instance of Plugin.
// It is meant for watching DB by prefix of different agent. You can pass/inject
// instance of this plugin for example to kvdbsync.
func OfDifferentAgent(microserviceLabel string) *Plugin {
	return &Plugin{MicroserviceLabel: microserviceLabel}
}

var microserviceLabelFlag string

func init() {
	flag.StringVar(&microserviceLabelFlag, "microservice-label", "vpp1", fmt.Sprintf("microservice label; also set via '%v' env variable.", MicroserviceLabelEnvVar))
}

// Init is called at plugin initialization.
func (p *Plugin) Init() error {
	if p.MicroserviceLabel == "" {
		p.MicroserviceLabel = microserviceLabelFlag
	}
	logroot.StandardLogger().Debugf("Microservice label is set to %v", p.MicroserviceLabel)
	return nil
}

// Close is called at plugin cleanup phase.
func (p *Plugin) Close() error {
	return nil
}

// GetAgentLabel returns string that is supposed to be used to distinguish
// (ETCD) key prefixes for particular VNF (particular VPP Agent configuration)
func (p *Plugin) GetAgentLabel() string {
	return p.MicroserviceLabel
}

// GetAgentPrefix returns the string that is supposed to be used as the prefix for configuration of current
// MicroserviceLabel "subtree" of the particular VPP Agent instance (e.g. in ETCD).
func (p *Plugin) GetAgentPrefix() string {
	return agentPrefix + p.MicroserviceLabel + "/"
}

// GetDifferentAgentPrefix returns the string that is supposed to be used as the prefix for configuration
// "subtree" of the particular VPP Agent instance (e.g. in ETCD).
func (p *Plugin) GetDifferentAgentPrefix(microserviceLabel string) string {
	return GetDifferentAgentPrefix(microserviceLabel)
}

// GetAllAgentsPrefix returns the string that is supposed to be used as the prefix for configuration
// subtree of the particular VPP Agent instance (e.g. in ETCD).
func (p *Plugin) GetAllAgentsPrefix() string {
	return GetAllAgentsPrefix()
}
<|endoftext|>


<commit_before><commit_msg>fail fix<commit_after><|endoftext|>


<commit_before>package repository

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/runabove/sail/internal"

	"github.com/spf13/cobra"
)

var cmdRepositoryAdd = &cobra.Command{
	// FIXME: only support adding from source via CLI, type external. Rename to 'register' ?
	Use:   "add",
	Short: "Add an external repository: sail repository add [<applicationName>/]<repositoryName> [<registryURL>] [<namespace>/]<externalRepositoryName> ",
	Long: `Add an external repository: sail repository add [<applicationName>/]<repositoryName> [<registryURL>] [<user>/]<externalRepositoryName>[:<tag>]
examples:
	- sail repository add myapp/myrepo docker.private.registry.com privateRegistryAccount/mysql:5.5
	- sail repository add myapp/myrepo dockerHubAccount/mysql:5.5
	- sail repository add myapp/myrepo mysql

[<user>] The account which hold the distant repository. If ommitted it use official docker image.
<externalRepositoryName> the name of the distant repository. Example : tutum/nginx
[<registryURL>] Url of the registry. Default value : the docker hub (e.g. https://hub.docker.com)
Only pubic repositories are supported yet. ADD NOTE FOR PRIVATE REGISTRIES
	`,
	Run: func(cmd *cobra.Command, args []string) {
		var registryURL, externalRepositoryName, repositoryName string
		switch len(args) {
		case 2:
			repositoryName = args[0]
			registryURL = ""
			externalRepositoryName = args[1]
		case 3:
			repositoryName = args[0]
			registryURL = args[1]
			externalRepositoryName = args[2]

		default:
			fmt.Fprintln(os.Stderr, "Invalid usage. sail repository add [<applicationName>/]<repositoryName> [<registryURL>/][<user>/]<externalRepositoryName>[:<tag>]. Please see sail repository add --help")
			return
		}

		n := repositoryAddStruct{
			RegistryURL:            registryURL,
			ExternalRepositoryName: externalRepositoryName,
		}
		repositoryAdd(repositoryName, n)
	},
}

type repositoryAddStruct struct {
	RegistryURL            string `json:"registryURL,omitempty"`
	ExternalRepositoryName string `json:"externalRepositoryName"`
}

func repositoryAdd(repositoryName string, args repositoryAddStruct) {
	// Split namespace and repository
	host, app, repo, tag, err := internal.ParseResourceName(repositoryName)
	internal.Check(err)

	if !internal.CheckHostConsistent(host) {
		fmt.Fprintf(os.Stderr, "Error: Invalid Host %s for endpoint %s\n", host, internal.Host)
		os.Exit(1)
	} else if len(tag) > 0 {
		fmt.Fprintf(os.Stderr, "Error: Invalid repository name. Please see sail repository add --help\n")
		os.Exit(1)
	}

	body, err := json.Marshal(args)
	internal.Check(err)

	path := fmt.Sprintf("/repositories/%s/%s", app, repo)
	internal.FormatOutputDef(internal.PostBodyWantJSON(path, body))

}
<commit_msg>improve doc<commit_after>package repository

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/runabove/sail/internal"

	"github.com/spf13/cobra"
)

var cmdRepositoryAdd = &cobra.Command{
	// FIXME: only support adding from source via CLI, type external. Rename to 'register' ?
	Use:   "add",
	Short: "Add an external repository: sail repository add [<applicationName>/]<repositoryName> [<registryURL>] [<namespace>/]<externalRepositoryName> ",
	Long: `Add an external repository: sail repository add [<applicationName>/]<repositoryName> [<registryURL>] [<user>/]<externalRepositoryName>[:<tag>]
examples:
	- sail repository add myapp/myrepo docker.private.registry.com privateRegistryAccount/mysql:5.5
	- sail repository add myapp/myrepo dockerHubAccount/mysql:5.5
	- sail repository add myapp/myrepo mysql

[<user>] The account which hold the distant repository. If ommitted it use official docker image.
<externalRepositoryName> the name of the distant repository. Example : tutum/nginx
[<registryURL>] Url of the registry. Default value : the docker hub (e.g. https://hub.docker.com)
Only publicly accessible repositories are supported.
	`,
	Run: func(cmd *cobra.Command, args []string) {
		var registryURL, externalRepositoryName, repositoryName string
		switch len(args) {
		case 2:
			repositoryName = args[0]
			registryURL = ""
			externalRepositoryName = args[1]
		case 3:
			repositoryName = args[0]
			registryURL = args[1]
			externalRepositoryName = args[2]

		default:
			fmt.Fprintln(os.Stderr, "Invalid usage. sail repository add [<applicationName>/]<repositoryName> [<registryURL>/][<user>/]<externalRepositoryName>[:<tag>]. Please see sail repository add --help")
			return
		}

		n := repositoryAddStruct{
			RegistryURL:            registryURL,
			ExternalRepositoryName: externalRepositoryName,
		}
		repositoryAdd(repositoryName, n)
	},
}

type repositoryAddStruct struct {
	RegistryURL            string `json:"registryURL,omitempty"`
	ExternalRepositoryName string `json:"externalRepositoryName"`
}

func repositoryAdd(repositoryName string, args repositoryAddStruct) {
	// Split namespace and repository
	host, app, repo, tag, err := internal.ParseResourceName(repositoryName)
	internal.Check(err)

	if !internal.CheckHostConsistent(host) {
		fmt.Fprintf(os.Stderr, "Error: Invalid Host %s for endpoint %s\n", host, internal.Host)
		os.Exit(1)
	} else if len(tag) > 0 {
		fmt.Fprintf(os.Stderr, "Error: Invalid repository name. Please see sail repository add --help\n")
		os.Exit(1)
	}

	body, err := json.Marshal(args)
	internal.Check(err)

	path := fmt.Sprintf("/repositories/%s/%s", app, repo)
	internal.FormatOutputDef(internal.PostBodyWantJSON(path, body))

}
<|endoftext|>


<commit_before>package cwbdata

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"log"
	"net/http"
	"time"
)

const ApiUrl = "http://opendata.cwb.gov.tw/opendataapi"
const CwbTimeFormat = "2006-01-02T15:04:05-07:00"

type CwbDataSource struct {
	APIKey string
}

type CwbDataSet struct {
	RawData []byte
	DataID  string
}

type CwbOpenData struct {
	Identifier string    `xml:"identifier"`
	Sender     string    `xml:"sender"`
	Sent       time.Time `xml:"sent"`
	Status     string    `xml:"status"`
	Scope      string    `xml:"scope"`
	MsgType    string    `xml:"msgType"`
	DataID     string    `xml:"dataid"`
	Source     string    `xml:"source"`
	DataSet    []byte    `xml:"dataset,innerXML"`
}
type rawCwbOpenData struct {
	CwbOpenData
	Sent string `xml:"sent"`
}

func (openData *CwbOpenData) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	origin := new(rawCwbOpenData)
	d.DecodeElement(origin, &start)

	openData.Identifier = origin.Identifier
	openData.Sender = origin.Sender
	openData.Status = origin.Status
	openData.Scope = origin.Scope
	openData.MsgType = origin.MsgType
	openData.DataID = origin.DataID
	openData.Source = origin.Source
	openData.DataSet = origin.DataSet
	timeStamp, err := time.Parse(CwbTimeFormat, origin.Sent)
	if err != nil {
		return err
	}
	openData.Sent = timeStamp
	return nil
}

func InitDataSource(apiKey string) CwbDataSource {
	return CwbDataSource{APIKey: apiKey}
}

func (cwb CwbDataSource) LoadDataSet(dataID string) (result CwbDataSet) {
	result = CwbDataSet{DataID: dataID}
	response, err := http.Get(fmt.Sprintf("%s?dataid=%s&authorizationkey=%s", ApiUrl, dataID, cwb.APIKey))
	if err != nil {
		log.Fatal(err)
		return
	}
	buffer := new(bytes.Buffer)
	defer response.Body.Close()
	buffer.ReadFrom(response.Body)
	result.RawData = buffer.Bytes()
	return
}
<commit_msg>Removed CwbDataSource.<commit_after>package cwbdata

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"net/http"
	"time"
)

const ApiUrl = "http://opendata.cwb.gov.tw/opendataapi"
const CwbTimeFormat = "2006-01-02T15:04:05-07:00"

type OpenDataSource interface {
	GetOpenData(dateID string) (CwbOpenData, error)
}
type CwbDataSource struct {
	APIKey string
}

type CwbDataSet struct {
	RawData []byte
	DataID  string
}

type CwbOpenData struct {
	Identifier string    `xml:"identifier"`
	Sender     string    `xml:"sender"`
	Sent       time.Time `xml:"sent"`
	Status     string    `xml:"status"`
	Scope      string    `xml:"scope"`
	MsgType    string    `xml:"msgType"`
	DataID     string    `xml:"dataid"`
	Source     string    `xml:"source"`
	DataSet    []byte    `xml:"dataset,innerXML"`
}
type rawCwbOpenData struct {
	CwbOpenData
	Sent string `xml:"sent"`
}

func (openData *CwbOpenData) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	origin := new(rawCwbOpenData)
	d.DecodeElement(origin, &start)

	openData.Identifier = origin.Identifier
	openData.Sender = origin.Sender
	openData.Status = origin.Status
	openData.Scope = origin.Scope
	openData.MsgType = origin.MsgType
	openData.DataID = origin.DataID
	openData.Source = origin.Source
	openData.DataSet = origin.DataSet
	timeStamp, err := time.Parse(CwbTimeFormat, origin.Sent)
	if err != nil {
		return err
	}
	openData.Sent = timeStamp
	return nil
}

func GetOpenData(apiKey string, dataID string) (openData CwbOpenData, err error) {
	response, err := http.Get(fmt.Sprintf("%s?dataid=%s&authorizationkey=%s", ApiUrl, dataID, apiKey))
	if err != nil {
		return
	}
	buffer := new(bytes.Buffer)
	defer response.Body.Close()
	buffer.ReadFrom(response.Body)
	openData = CwbOpenData{}
	err = xml.Unmarshal(buffer.Bytes(), &openData)
	return
}
<|endoftext|>


<commit_before>package pi

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
)

var (
	errNoFiles = fmt.Errorf("no files")
)

// RequestContext represents the context of the HTTP request.
// It is shared across interceptors and handler.
type RequestContext struct {
	W        http.ResponseWriter
	R        *http.Request
	RouteURL string
	Data     map[interface{}]interface{}
}

// newRequestContext returns a new RequestContext.
func newRequestContext(w http.ResponseWriter, r *http.Request, routeURL string) *RequestContext {
	return &RequestContext{
		W:        w,
		R:        r,
		RouteURL: routeURL,
		Data:     make(map[interface{}]interface{}),
	}
}

// WriteString writes the specified strings to the ResponseWriter.
func (c *RequestContext) WriteString(strings ...string) error {
	c.W.Header().Set("Content-Type", "text/plain; charset=utf-8")
	for _, s := range strings {
		if _, err := c.W.Write([]byte(s)); err != nil {
			return err
		}
	}
	return nil
}

// WriteJSON marshal the object to JSON and writes it via the ResponseWriter.
func (c *RequestContext) WriteJSON(object interface{}) error {
	c.W.Header().Set("Content-Type", "application/json; charset=utf-8")
	if debugMode {
		output, err := json.MarshalIndent(object, "", "  ")
		if err != nil {
			return err
		}
		writeDebug("WriteJSON", c.R.RemoteAddr, string(output))
		c.W.Write(output)
	} else {
		output, err := json.Marshal(object)
		if err != nil {
			return err
		}
		c.W.Write(output)
	}
	return nil
}

// WriteXML marshal the object to XML and writes it via the ResponseWriter.
func (c *RequestContext) WriteXML(object interface{}) error {
	c.W.Header().Set("Content-Type", "application/xml; charset=utf-8")
	if debugMode {
		output, err := xml.MarshalIndent(object, "", "  ")
		if err != nil {
			return err
		}
		writeDebug("WriteXML", c.R.RemoteAddr, string(output))
		c.W.Write(output)
	} else {
		output, err := xml.Marshal(object)
		if err != nil {
			return err
		}
		c.W.Write(output)
	}
	return nil
}

// SetStatusCode sets the response status code.
func (c *RequestContext) SetStatusCode(statusCode int)  {
	c.W.WriteHeader(statusCode)
}

// GetBody return the body as a ReadCloser. It is the client responsibility to close the body.
func (c *RequestContext) GetBody() io.ReadCloser {
	return c.R.Body
}

// GetRawBody returns the body as a byte array, closing the body reader.
func (c *RequestContext) GetRawBody() ([]byte, error) {
	body := c.GetBody()
	defer body.Close()
	rawBody, err := ioutil.ReadAll(body)
	if err != nil {
		return nil, err
	}
	if debugMode {
		writeDebug("GetRawBody", c.R.RemoteAddr, fmt.Sprintf("got %s", string(rawBody)))
	}
	return ioutil.ReadAll(body)
}

// GetJSONObject call json.Unmarshal by sending the reference of the given object.
// For example:
//		func GetUser(c *pi.RequestContext) error {
// 			user := &User{}
// 			if err := c.GetJSONObject(user); err != nil {
//				return pi.NewError(400, err)
//			}
//			// Do something with the user...
//			return nil
//		}
//
func (c *RequestContext) GetJSONObject(object interface{}) error {
	rawBody, err := c.GetRawBody()
	if err != nil {
		return err
	}
	return json.Unmarshal(rawBody, &object)
}

// GetXMLObject call xml.Unmarshal by sending the reference of the given object.
func (c *RequestContext) GetXMLObject(object interface{}) error {
	rawBody, err := c.GetRawBody()
	if err != nil {
		return err
	}
	return xml.Unmarshal(rawBody, &object)
}

// GetURLParam returns an URL parameter.
// For example, given this URL:
//		/user?id=1234
//
//		fmt.Println(c.GetURLParam("id"))
//		// Outputs 1234
//
func (c *RequestContext) GetURLParam(param string) string {
	return c.R.FormValue(param)
}

// GetRouteVariable returns a route variable.
// For example:
//		getUserByID := func(c *RequestContext) error {
//			id := c.GetRouteVariable("id")
//			// Do something with the ID.
//			return nil
//		}
//
//		p := pi.New()
//		p.Route("/user/{id}").Get(getUserByID)
//		p.ListenAndServe(":8080")
//
func (c *RequestContext) GetRouteVariable(key string) string {
	return c.R.URL.Query().Get(":" + key)
}

// GetFileHeaders returns an array of FileHeader.
// For example:
//		getImages := func(c *RequestContext) error {
//			files, err := c.GetFileHeaders("files")
//			if err != nil {
//				return pi.NewError(400, err)
//			}
//			// Handle files
//			return nil
//		}
//
//		p := pi.New()
//		p.Route("/images").Post(getImages)
//		p.ListenAndServe(":8080")
//
func (c *RequestContext) GetFileHeaders(key string) ([]*multipart.FileHeader, error) {
	if err := c.R.ParseMultipartForm(32 << 20); err != nil {
		return nil, err
	}
	if c.R.MultipartForm != nil && c.R.MultipartForm.File[key] != nil {
		return c.R.MultipartForm.File[key], nil
	}
	return nil, errNoFiles
}
<commit_msg>Added J type<commit_after>package pi

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
)

var (
	errNoFiles = fmt.Errorf("no files")
)

// J is an helper to write JSON.
// Example:
//		c.WriteJSON(pi.J{"status": "OK"})
//
type J map[string]interface{}

// RequestContext represents the context of the HTTP request.
// It is shared across interceptors and handler.
type RequestContext struct {
	W        http.ResponseWriter
	R        *http.Request
	RouteURL string
	Data     map[interface{}]interface{}
}

// newRequestContext returns a new RequestContext.
func newRequestContext(w http.ResponseWriter, r *http.Request, routeURL string) *RequestContext {
	return &RequestContext{
		W:        w,
		R:        r,
		RouteURL: routeURL,
		Data:     make(map[interface{}]interface{}),
	}
}

// WriteString writes the specified strings to the ResponseWriter.
func (c *RequestContext) WriteString(strings ...string) error {
	c.W.Header().Set("Content-Type", "text/plain; charset=utf-8")
	for _, s := range strings {
		if _, err := c.W.Write([]byte(s)); err != nil {
			return err
		}
	}
	return nil
}

// WriteJSON marshal the object to JSON and writes it via the ResponseWriter.
func (c *RequestContext) WriteJSON(object interface{}) error {
	c.W.Header().Set("Content-Type", "application/json; charset=utf-8")
	if debugMode {
		output, err := json.MarshalIndent(object, "", "  ")
		if err != nil {
			return err
		}
		writeDebug("WriteJSON", c.R.RemoteAddr, string(output))
		c.W.Write(output)
	} else {
		output, err := json.Marshal(object)
		if err != nil {
			return err
		}
		c.W.Write(output)
	}
	return nil
}

// WriteXML marshal the object to XML and writes it via the ResponseWriter.
func (c *RequestContext) WriteXML(object interface{}) error {
	c.W.Header().Set("Content-Type", "application/xml; charset=utf-8")
	if debugMode {
		output, err := xml.MarshalIndent(object, "", "  ")
		if err != nil {
			return err
		}
		writeDebug("WriteXML", c.R.RemoteAddr, string(output))
		c.W.Write(output)
	} else {
		output, err := xml.Marshal(object)
		if err != nil {
			return err
		}
		c.W.Write(output)
	}
	return nil
}

// SetStatusCode sets the response status code.
func (c *RequestContext) SetStatusCode(statusCode int)  {
	c.W.WriteHeader(statusCode)
}

// GetBody return the body as a ReadCloser. It is the client responsibility to close the body.
func (c *RequestContext) GetBody() io.ReadCloser {
	return c.R.Body
}

// GetRawBody returns the body as a byte array, closing the body reader.
func (c *RequestContext) GetRawBody() ([]byte, error) {
	body := c.GetBody()
	defer body.Close()
	rawBody, err := ioutil.ReadAll(body)
	if err != nil {
		return nil, err
	}
	if debugMode {
		writeDebug("GetRawBody", c.R.RemoteAddr, fmt.Sprintf("got %s", string(rawBody)))
	}
	return ioutil.ReadAll(body)
}

// GetJSONObject call json.Unmarshal by sending the reference of the given object.
// For example:
//		func GetUser(c *pi.RequestContext) error {
// 			user := &User{}
// 			if err := c.GetJSONObject(user); err != nil {
//				return pi.NewError(400, err)
//			}
//			// Do something with the user...
//			return nil
//		}
//
func (c *RequestContext) GetJSONObject(object interface{}) error {
	rawBody, err := c.GetRawBody()
	if err != nil {
		return err
	}
	return json.Unmarshal(rawBody, &object)
}

// GetXMLObject call xml.Unmarshal by sending the reference of the given object.
func (c *RequestContext) GetXMLObject(object interface{}) error {
	rawBody, err := c.GetRawBody()
	if err != nil {
		return err
	}
	return xml.Unmarshal(rawBody, &object)
}

// GetURLParam returns an URL parameter.
// For example, given this URL:
//		/user?id=1234
//
//		fmt.Println(c.GetURLParam("id"))
//		// Outputs 1234
//
func (c *RequestContext) GetURLParam(param string) string {
	return c.R.FormValue(param)
}

// GetRouteVariable returns a route variable.
// For example:
//		getUserByID := func(c *RequestContext) error {
//			id := c.GetRouteVariable("id")
//			// Do something with the ID.
//			return nil
//		}
//
//		p := pi.New()
//		p.Route("/user/{id}").Get(getUserByID)
//		p.ListenAndServe(":8080")
//
func (c *RequestContext) GetRouteVariable(key string) string {
	return c.R.URL.Query().Get(":" + key)
}

// GetFileHeaders returns an array of FileHeader.
// For example:
//		getImages := func(c *RequestContext) error {
//			files, err := c.GetFileHeaders("files")
//			if err != nil {
//				return pi.NewError(400, err)
//			}
//			// Handle files
//			return nil
//		}
//
//		p := pi.New()
//		p.Route("/images").Post(getImages)
//		p.ListenAndServe(":8080")
//
func (c *RequestContext) GetFileHeaders(key string) ([]*multipart.FileHeader, error) {
	if err := c.R.ParseMultipartForm(32 << 20); err != nil {
		return nil, err
	}
	if c.R.MultipartForm != nil && c.R.MultipartForm.File[key] != nil {
		return c.R.MultipartForm.File[key], nil
	}
	return nil, errNoFiles
}
<|endoftext|>


<commit_before>package megos

// State represents the JSON from the state.json of a mesos node
type State struct {
	// Missing fields...
	// TODO "attributes": {},
	// TODO Better type as string for "2015-05-05 06:16:58" ?
	// TODO: completed_frameworks "completed_frameworks": [],
	// TODO "orphan_tasks": [],
	ActivatedSlaves        int         `json:"activated_slaves"`
	BuildDate              string      `json:"build_date"`
	BuildTime              int         `json:"build_time"`
	BuildUser              string      `json:"build_user"`
	Cluster                string      `json:"cluster"`
	DeactivatedSlaves      int         `json:"deactivated_slaves"`
	ElectedTime            float32     `json:"elected_time"`
	FailedTasks            int         `json:"failed_tasks"`
	FinishedTasks          int         `json:"finished_tasks"`
	Flags                  Flags       `json:"flags"`
	Frameworks             []Framework `json:"frameworks"`
	GitSHA                 string      `json:"git_sha"`
	GitTag                 string      `json:"git_tag"`
	Hostname               string      `json:"hostname"`
	ID                     string      `json:"id"`
	KilledTasks            int         `json:"killed_tasks"`
	Leader                 string      `json:"leader"`
	LogDir                 string      `json:"log_dir"`
	LostTasks              int         `json:"lost_tasks"`
	MasterHostname         string      `json:"master_hostname"`
	PID                    string      `json:"pid"`
	Resources              Resources   `json:"resources"`
	Slaves                 []Slave     `json:"slaves"`
	StagedTasks            int         `json:"staged_tasks"`
	StartTime              float32     `json:"start_time"`
	StartedTasks           int         `json:"started_tasks"`
	UnregisteredFrameworks []Framework `json:"unregistered_frameworks"`
	Version                string      `json:"version"`
}

// Flags represents the flags of a mesos state
type Flags struct {
	AllocationInterval          string `json:"allocation_interval"`
	Authenticate                string `json:"authenticate"`
	Authenticatee               string `json:"authenticatee"`
	AuthenticateSlaves          string `json:"authenticate_slaves"`
	Authenticators              string `json:"authenticators"`
	CgroupsEnableCfs            string `json:"cgroups_enable_cfs"`
	CgroupsHierarchy            string `json:"cgroups_hierarchy"`
	CgroupsLimitSwap            string `json:"cgroups_limit_swap"`
	CgroupsRoot                 string `json:"cgroups_root"`
	Cluster                     string `json:"cluster"`
	ContainerDiskWatchInterval  string `json:"container_disk_watch_interval"`
	Containerizers              string `json:"containerizers"`
	DefaultRole                 string `json:"default_role"`
	DiskWatchInterval           string `json:"disk_watch_interval"`
	Docker                      string `json:"docker"`
	DockerRemoveDelay           string `json:"docker_remove_delay"`
	DockerSandboxDirectory      string `json:"docker_sandbox_directory"`
	DockerStopTimeout           string `json:"docker_stop_timeout"`
	EnforceContainerDiskQuota   string `json:"enforce_container_disk_quota"`
	ExecutorRegistrationTimeout string `json:"executor_registration_timeout"`
	ExecutorShutdownGracePeriod string `json:"executor_shutdown_grace_period"`
	FrameworksHome              string `json:"frameworks_home"`
	FrameworkSorter             string `json:"framework_sorter"`
	GCDelay                     string `json:"gc_delay"`
	GCDiskHeadroom              string `json:"gc_disk_headroom"`
	HadoopHome                  string `json:"hadoop_home"`
	Help                        string `json:"help"`
	Hostname                    string `json:"hostname"`
	InitializeDriverLogging     string `json:"initialize_driver_logging"`
	IP                          string `json:"ip"`
	Isolation                   string `json:"isolation"`
	LauncherDir                 string `json:"launcher_dir"`
	LogAutoInitialize           string `json:"log_auto_initialize"`
	LogDir                      string `json:"log_dir"`
	Logbufsecs                  string `json:"logbufsecs"`
	LoggingLevel                string `json:"logging_level"`
	Master                      string `json:"master"`
	PerfDuration                string `json:"perf_duration"`
	PerfInterval                string `json:"perf_interval"`
	Port                        string `json:"port"`
	Quiet                       string `json:"quiet"`
	Quorum                      string `json:"quorum"`
	Recover                     string `json:"recover"`
	RecoverySlaveRemovalLimit   string `json:"recovery_slave_removal_limit"`
	RecoveryTimeout             string `json:"recovery_timeout"`
	RegistrationBackoffFactor   string `json:"registration_backoff_factor"`
	Registry                    string `json:"registry"`
	RegistryFetchTimeout        string `json:"registry_fetch_timeout"`
	RegistryStoreTimeout        string `json:"registry_store_timeout"`
	RegistryStrict              string `json:"registry_strict"`
	ResourceMonitoringInterval  string `json:"resource_monitoring_interval"`
	RootSubmissions             string `json:"root_submissions"`
	SlaveReregisterTimeout      string `json:"slave_reregister_timeout"`
	Strict                      string `json:"strict"`
	SwitchUser                  string `json:"switch_user"`
	UserSorter                  string `json:"user_sorter"`
	Version                     string `json:"version"`
	WebuiDir                    string `json:"webui_dir"`
	WorkDir                     string `json:"work_dir"`
	ZK                          string `json:"zk"`
	ZKSessionTimeout            string `json:"zk_session_timeout"`
}

// Framework represent a single framework of a mesos node
type Framework struct {
	// Missing fields
	// TODO: "offers": [],
	Active             bool       `json:"active"`
	Checkpoint         bool       `json:"checkpoint"`
	CompletedExecutors []Executor `json:"completed_executors"`
	CompletedTasks     []Task     `json:"completed_tasks"`
	FailoverTimeout    int        `json:"failover_timeout"`
	Hostname           string     `json:"hostname"`
	ID                 string     `json:"id"`
	Name               string     `json:"name"`
	OfferedResources   Resources  `json:"offered_resources"`
	RegisteredTime     float32    `json:"registered_time"`
	ReregisteredTime   float32    `json:"reregistered_time"`
	Resources          Resources  `json:"resources"`
	Role               string     `json:"role"`
	Tasks              []Task     `json:"tasks"`
	UnregisteredTime   float32    `json:"unregistered_time"`
	UsedResources      Resources  `json:"used_resources"`
	User               string     `json:"user"`
	WebuiURL           string     `json:"webui_url"`
}

// Task represent a single Mesos task
type Task struct {
	// Missing fields
	// TODO: "labels": [],
	ExecutorID  string       `json:"executor_id"`
	FrameworkID string       `json:"framework_id"`
	ID          string       `json:"id"`
	Name        string       `json:"name"`
	Resources   Resources    `json:"resources"`
	SlaveID     string       `json:"slave_id"`
	State       string       `json:"state"`
	Statuses    []TaskStatus `json:"statuses"`
}

// Resources represents a resource type for a task
type Resources struct {
	CPUs  float32 `json:"cpus"`
	Disk  int     `json:"disk"`
	Mem   float64 `json:"mem"`
	Ports string  `json:"ports"`
}

// TaskStatus represents the status of a single task
type TaskStatus struct {
	State     string  `json:"state"`
	Timestamp float32 `json:"timestamp"`
}

// Slave represents a single mesos slave node
type Slave struct {
	Active         bool              `json:"active"`
	Hostname       string            `json:"hostname"`
	ID             string            `json:"id"`
	PID            string            `json:"pid"`
	RegisteredTime float32           `json:"registered_time"`
	Resources      Resources         `json:"resources"`
	Attributes     map[string]string `json:"attributes"`
}

// Executor represents a single executor of a framework
type Executor struct {
	// Missing fields
	// TODO "queued_tasks": [],
	// TODO "tasks": []
	CompletedTasks []Task    `json:"completed_tasks"`
	Container      string    `json:"container"`
	Directory      string    `json:"directory"`
	ID             string    `json:"id"`
	Name           string    `json:"name"`
	Resources      Resources `json:"resources"`
	Source         string    `json:"source"`
}
<commit_msg>Update version to 0.24.1<commit_after>package megos

// State represents the JSON from the state.json of a mesos node
type State struct {
	ActivatedSlaves        int         `json:"activated_slaves"`
	BuildDate              string      `json:"build_date"`
	BuildTime              int         `json:"build_time"`
	BuildUser              string      `json:"build_user"`
	Cluster                string      `json:"cluster"`
	CompletedFrameworks    []Framework `json:"completed_frameworks"`
	DeactivatedSlaves      int         `json:"deactivated_slaves"`
	ElectedTime            float32     `json:"elected_time"`
	Flags                  Flags       `json:"flags"`
	Frameworks             []Framework `json:"frameworks"`
	GitSHA                 string      `json:"git_sha"`
	GitBranch              string      `json:"git_branch"`
	GitTag                 string      `json:"git_tag"`
	Hostname               string      `json:"hostname"`
	ID                     string      `json:"id"`
	Leader                 string      `json:"leader"`
	LogDir                 string      `json:"log_dir"`
	ExternalLogFile        string      `json:"external_log_file"`
	OrphanTasks            []Task      `json:"orphan_tasks"`
	PID                    string      `json:"pid"`
	Slaves                 []Slave     `json:"slaves"`
	StartTime              float32     `json:"start_time"`
	UnregisteredFrameworks []string    `json:"unregistered_frameworks"`
	Version                string      `json:"version"`
}

// Flags represents the flags of a mesos state
type Flags struct {
	AllocationInterval          string `json:"allocation_interval"`
	Allocator                   string `json:"allocator"`
	Authenticate                string `json:"authenticate"`
	Authenticatee               string `json:"authenticatee"`
	AuthenticateSlaves          string `json:"authenticate_slaves"`
	Authenticators              string `json:"authenticators"`
	Authorizers                 string `json:"authorizers"`
	CgroupsEnableCfs            string `json:"cgroups_enable_cfs"`
	CgroupsHierarchy            string `json:"cgroups_hierarchy"`
	CgroupsLimitSwap            string `json:"cgroups_limit_swap"`
	CgroupsRoot                 string `json:"cgroups_root"`
	Cluster                     string `json:"cluster"`
	ContainerDiskWatchInterval  string `json:"container_disk_watch_interval"`
	Containerizers              string `json:"containerizers"`
	DefaultRole                 string `json:"default_role"`
	DiskWatchInterval           string `json:"disk_watch_interval"`
	Docker                      string `json:"docker"`
	DockerRemoveDelay           string `json:"docker_remove_delay"`
	DockerSandboxDirectory      string `json:"docker_sandbox_directory"`
	DockerStopTimeout           string `json:"docker_stop_timeout"`
	EnforceContainerDiskQuota   string `json:"enforce_container_disk_quota"`
	ExecutorRegistrationTimeout string `json:"executor_registration_timeout"`
	ExecutorShutdownGracePeriod string `json:"executor_shutdown_grace_period"`
	FrameworksHome              string `json:"frameworks_home"`
	FrameworkSorter             string `json:"framework_sorter"`
	GCDelay                     string `json:"gc_delay"`
	GCDiskHeadroom              string `json:"gc_disk_headroom"`
	HadoopHome                  string `json:"hadoop_home"`
	Help                        string `json:"help"`
	Hostname                    string `json:"hostname"`
	InitializeDriverLogging     string `json:"initialize_driver_logging"`
	IP                          string `json:"ip"`
	Isolation                   string `json:"isolation"`
	LauncherDir                 string `json:"launcher_dir"`
	LogAutoInitialize           string `json:"log_auto_initialize"`
	LogDir                      string `json:"log_dir"`
	Logbufsecs                  string `json:"logbufsecs"`
	LoggingLevel                string `json:"logging_level"`
	MaxSlavePingTimeouts        string `json:"max_slave_ping_timeouts"`
	Master                      string `json:"master"`
	PerfDuration                string `json:"perf_duration"`
	PerfInterval                string `json:"perf_interval"`
	Port                        string `json:"port"`
	Quiet                       string `json:"quiet"`
	Quorum                      string `json:"quorum"`
	Recover                     string `json:"recover"`
	RecoverySlaveRemovalLimit   string `json:"recovery_slave_removal_limit"`
	RecoveryTimeout             string `json:"recovery_timeout"`
	RegistrationBackoffFactor   string `json:"registration_backoff_factor"`
	Registry                    string `json:"registry"`
	RegistryFetchTimeout        string `json:"registry_fetch_timeout"`
	RegistryStoreTimeout        string `json:"registry_store_timeout"`
	RegistryStrict              string `json:"registry_strict"`
	ResourceMonitoringInterval  string `json:"resource_monitoring_interval"`
	RootSubmissions             string `json:"root_submissions"`
	SlavePingTimeout            string `json:"slave_ping_timeout"`
	SlaveReregisterTimeout      string `json:"slave_reregister_timeout"`
	Strict                      string `json:"strict"`
	SwitchUser                  string `json:"switch_user"`
	UserSorter                  string `json:"user_sorter"`
	Version                     string `json:"version"`
	WebuiDir                    string `json:"webui_dir"`
	WorkDir                     string `json:"work_dir"`
	ZK                          string `json:"zk"`
	ZKSessionTimeout            string `json:"zk_session_timeout"`
}

// Framework represent a single framework of a mesos node
type Framework struct {
	Active           bool       `json:"active"`
	Checkpoint       bool       `json:"checkpoint"`
	CompletedTasks   []Task     `json:"completed_tasks"`
	Executors        []Executor `json:"executors"`
	FailoverTimeout  int        `json:"failover_timeout"`
	Hostname         string     `json:"hostname"`
	ID               string     `json:"id"`
	Name             string     `json:"name"`
	OfferedResources Resources  `json:"offered_resources"`
	Offers           []Offer    `json:"offers"`
	RegisteredTime   float32    `json:"registered_time"`
	ReregisteredTime float32    `json:"reregistered_time"`
	Resources        Resources  `json:"resources"`
	Role             string     `json:"role"`
	Tasks            []Task     `json:"tasks"`
	UnregisteredTime float32    `json:"unregistered_time"`
	UsedResources    Resources  `json:"used_resources"`
	User             string     `json:"user"`
	WebuiURL         string     `json:"webui_url"`
	Labels           []Label    `json:"label"`
}

type Offer struct {
	ID          string            `json:"id"`
	FrameworkID string            `json:"framework_id"`
	SlaveID     string            `json:"slave_id"`
	Hostname    string            `json:"hostname"`
	URL         URL               `json:"url"`
	Resources   Resources         `json:"resources"`
	Attributes  map[string]string `json:"attributes"`
}

type URL struct {
	Scheme     string      `json:"scheme"`
	Address    Address     `json:"address"`
	Path       string      `json:"path"`
	Parameters []Parameter `json:"parameters"`
}

type Address struct {
	Hostname string `json:"hostname"`
	IP       string `json:"ip"`
	Port     int    `json:"port"`
}

type Parameter struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type Label struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

// Task represent a single Mesos task
type Task struct {
	// Missing fields
	// TODO: "labels": [],
	ExecutorID  string       `json:"executor_id"`
	FrameworkID string       `json:"framework_id"`
	ID          string       `json:"id"`
	Name        string       `json:"name"`
	Resources   Resources    `json:"resources"`
	SlaveID     string       `json:"slave_id"`
	State       string       `json:"state"`
	Statuses    []TaskStatus `json:"statuses"`
}

// Resources represents a resource type for a task
type Resources struct {
	CPUs  float32 `json:"cpus"`
	Disk  int     `json:"disk"`
	Mem   float64 `json:"mem"`
	Ports string  `json:"ports"`
}

// TaskStatus represents the status of a single task
type TaskStatus struct {
	State     string  `json:"state"`
	Timestamp float32 `json:"timestamp"`
}

// Slave represents a single mesos slave node
type Slave struct {
	Active         bool              `json:"active"`
	Hostname       string            `json:"hostname"`
	ID             string            `json:"id"`
	PID            string            `json:"pid"`
	RegisteredTime float32           `json:"registered_time"`
	Resources      Resources         `json:"resources"`
	Attributes     map[string]string `json:"attributes"`
}

// Executor represents a single executor of a framework
type Executor struct {
	// Missing fields
	// TODO "queued_tasks": [],
	// TODO "tasks": []
	CompletedTasks []Task    `json:"completed_tasks"`
	Container      string    `json:"container"`
	Directory      string    `json:"directory"`
	ID             string    `json:"id"`
	Name           string    `json:"name"`
	Resources      Resources `json:"resources"`
	Source         string    `json:"source"`
}
<|endoftext|>


<commit_before>package tenordb

type AbsNote struct {
	Id int	`db:id`
	Name string `db:name`
}

type RelNote struct {
	Id int	`db:id`
	Name string `db:name`
}

type Chord struct {
	Id int	`db:id`
	Cp_id int `db:cp_id`
	Root_an_id int `db:root_an_id`
}

type Scale struct {
	Id int	`db:id`
	Sp_id int `db:sp_id`
	Root_an_id int `db:root_an_id`
}

type ChordPattern struct {
	Id int	`db:id`
	Name string `db:name`
}

type ScalePattern struct {
	Id int	`db:id`
	Name string `db:name`
}

type ChordNote struct {
	An_id int `db:an_id`
	Rn_id int `db:rn_id`
	C_id int `db:c_id`
}

type ScaleNote struct {
	An_id int `db:an_id`
	Rn_id int `db:rn_id`
	S_id int `db:s_id`
}

type ChordPatternNote struct {
	Cp_id int `db:cp_id`
	Rn_id int `db:rn_id`
}

type ScalePatternNote struct {
	Sp_id int `db:sp_id`
	Rn_id int `db:rn_id`
}
<commit_msg>Go fmt<commit_after>package tenordb

type AbsNote struct {
	Id   int    `db:id`
	Name string `db:name`
}

type RelNote struct {
	Id   int    `db:id`
	Name string `db:name`
}

type Chord struct {
	Id         int `db:id`
	Cp_id      int `db:cp_id`
	Root_an_id int `db:root_an_id`
}

type Scale struct {
	Id         int `db:id`
	Sp_id      int `db:sp_id`
	Root_an_id int `db:root_an_id`
}

type ChordPattern struct {
	Id   int    `db:id`
	Name string `db:name`
}

type ScalePattern struct {
	Id   int    `db:id`
	Name string `db:name`
}

type ChordNote struct {
	An_id int `db:an_id`
	Rn_id int `db:rn_id`
	C_id  int `db:c_id`
}

type ScaleNote struct {
	An_id int `db:an_id`
	Rn_id int `db:rn_id`
	S_id  int `db:s_id`
}

type ChordPatternNote struct {
	Cp_id int `db:cp_id`
	Rn_id int `db:rn_id`
}

type ScalePatternNote struct {
	Sp_id int `db:sp_id`
	Rn_id int `db:rn_id`
}
<|endoftext|>


<commit_before>package	gearman // import "github.com/nathanaelle/gearman"

import	(
	"errors"
)

type	(
	MarshalerGearman interface {
		MarshalGearman() ([]byte,error)
		Len()	int
	}

	UnmarshalerGearman interface {
		UnmarshalGearman([]byte) error
	}

	Opaque		interface {
		MarshalerGearman
		UnmarshalerGearman
		Cast(UnmarshalerGearman) error
		Bytes()	[]byte
	}

	TaskID		[64]byte

	Function	string
	ClientId	[]byte

	opaque		[]byte

	opaque0size	struct {}
)

var empty_opaque	*opaque0size = &opaque0size{}

func Opacify(b []byte) Opaque {
	if len(b) == 0 {
		return empty_opaque
	}

	o := opaque(b)

	return &o
}

func (o *opaque)UnmarshalGearman(d []byte) error {
	*o = d
	return	nil
}

func (o *opaque)MarshalGearman() ([]byte,error) {
	return o.Bytes(), nil
}

func (o *opaque)Bytes()	[]byte {
	return	[]byte(*o)
}

func (o *opaque)Len() int {
	return	len(o.Bytes())
}

func (o *opaque)Cast(um UnmarshalerGearman) error {
	return	um.UnmarshalGearman([]byte(*o))
}

func (_ *opaque0size)UnmarshalGearman(d []byte) error {
	if len(d) > 0 {
		return errors.New("empty_opaque can't unmarshal data")
	}
	return	nil
}

func (_ *opaque0size)MarshalGearman() ([]byte,error) {
	return []byte{}, nil
}

func (_ *opaque0size)Bytes() []byte {
	return []byte{}
}

func (_ *opaque0size)Len() int {
	return	0
}

func (_ *opaque0size)Cast(um UnmarshalerGearman) error {
	return	um.UnmarshalGearman([]byte{})
}

func (tid TaskID)MarshalGearman() ([]byte,error) {
	return	tid[0:tid.Len()],nil
}

func (tid TaskID)Len() int {
	end := 63
	for tid[end] == 0 {
		end--
	}

	return end+1
}

func (tid *TaskID)UnmarshalGearman(d []byte) error {
	if len(d) > 64 {
		return	errors.New("tid too long")
	}

	for _,v := range d {
		if v == 0 {
			return errors.New("invalid TaskID")
		}
	}

	copy(tid[0:len(d)], d[:])
	return	nil
}

func (fn *Function)UnmarshalGearman(d []byte) error {
	*fn = Function(string(d))
	return	nil
}

func (fn Function)MarshalGearman() ([]byte,error) {
	return []byte(fn), nil
}

func (fn Function)Len() int {
	return	len([]byte(string(fn)))
}

func (clid *ClientId)UnmarshalGearman(d []byte) error {
	*clid = d
	return	nil
}

func (clid ClientId)MarshalGearman() ([]byte,error) {
	return clid, nil
}

func (clid ClientId)Len() int {
	return	len([]byte(clid))
}
<commit_msg>add Stringer interface to Function vanity type<commit_after>package	gearman // import "github.com/nathanaelle/gearman"

import	(
	"errors"
	"encoding/base64"
)

type	(
	MarshalerGearman interface {
		MarshalGearman() ([]byte,error)
		Len()	int
	}

	UnmarshalerGearman interface {
		UnmarshalGearman([]byte) error
	}

	Opaque		interface {
		MarshalerGearman
		UnmarshalerGearman
		Cast(UnmarshalerGearman) error
		Bytes()	[]byte
	}

	TaskID		[64]byte

	Function	[]byte
	ClientId	[]byte

	opaque		[]byte

	opaque0size	struct {}
)

var empty_opaque	*opaque0size = &opaque0size{}

func Opacify(b []byte) Opaque {
	if len(b) == 0 {
		return empty_opaque
	}

	o := opaque(b)

	return &o
}

func (o *opaque)UnmarshalGearman(d []byte) error {
	*o = d
	return	nil
}

func (o *opaque)MarshalGearman() ([]byte,error) {
	return o.Bytes(), nil
}

func (o *opaque)Bytes()	[]byte {
	return	[]byte(*o)
}

func (o *opaque)Len() int {
	return	len(o.Bytes())
}

func (o *opaque)Cast(um UnmarshalerGearman) error {
	return	um.UnmarshalGearman([]byte(*o))
}

func (_ *opaque0size)UnmarshalGearman(d []byte) error {
	if len(d) > 0 {
		return errors.New("empty_opaque can't unmarshal data")
	}
	return	nil
}

func (_ *opaque0size)MarshalGearman() ([]byte,error) {
	return []byte{}, nil
}

func (_ *opaque0size)Bytes() []byte {
	return []byte{}
}

func (_ *opaque0size)Len() int {
	return	0
}

func (_ *opaque0size)Cast(um UnmarshalerGearman) error {
	return	um.UnmarshalGearman([]byte{})
}

func (tid TaskID)MarshalGearman() ([]byte,error) {
	return	tid[0:tid.Len()],nil
}

func (tid TaskID)Len() int {
	end := 63
	for tid[end] == 0 {
		end--
	}

	return end+1
}

func (tid *TaskID)UnmarshalGearman(d []byte) error {
	if len(d) > 64 {
		return	errors.New("tid too long")
	}

	for _,v := range d {
		if v == 0 {
			return errors.New("invalid TaskID")
		}
	}

	copy(tid[0:len(d)], d[:])
	return	nil
}

func (fn *Function)UnmarshalGearman(d []byte) error {
	*fn = Function(d)
	return	nil
}

func (fn Function)MarshalGearman() ([]byte,error) {
	return fn, nil
}

func (fn Function)Len() int {
	return	len(fn)
}

func (fn Function)String() string {
	return	base64.RawURLEncoding.EncodeToString([]byte(fn))
}

func (clid *ClientId)UnmarshalGearman(d []byte) error {
	*clid = d
	return	nil
}

func (clid ClientId)MarshalGearman() ([]byte,error) {
	return clid, nil
}

func (clid ClientId)Len() int {
	return	len([]byte(clid))
}
<|endoftext|>


<commit_before>package file

import (
	"encoding/csv"
	"fmt"
	"io"
	"os"

	"github.com/vladimirvivien/automi/api"
)

// CsvRead implements an Source process that reads the content of a
// specified file and emits its record via its Output Channel
// and serializes each row as a slice []string.
type CsvRead struct {
	Name          string   // string identifer for the Csv emitter
	FilePath      string   // path for the file
	DelimiterChar rune     // Delimiter charater, defaults to comma
	CommentChar   rune     // Charater indicating line is a comment
	Headers       []string // Column header names (specified here or read from file)
	HasHeaderRow  bool     // indicates first row is for headers (default false). Overrides the Headers attribute.
	FieldCount    int      // if greater than zero is used to validate field count

	file   *os.File
	reader *csv.Reader
	logs   chan interface{}
	output chan interface{}
}

func (c *CsvRead) Init() error {
	// validation
	if c.Name == "" {
		return fmt.Errorf("CsvRead  missing an identifying name.")
	}
	if c.FilePath == "" {
		return fmt.Errorf("CsvRead [%s] - Missing required FilePath attribute.")
	}

	// establish defaults
	if c.DelimiterChar == 0 {
		c.DelimiterChar = ','
	}

	if c.CommentChar == 0 {
		c.CommentChar = '#'
	}

	// open file
	file, err := os.Open(c.FilePath)
	if err != nil {
		return fmt.Errorf("CsvRead [%s] - Failed to create file: %s ", c.Name, err)
	}

	c.file = file
	c.reader = csv.NewReader(file)
	c.reader.Comment = c.CommentChar
	c.reader.Comma = c.DelimiterChar

	// resolve header and field count
	if c.HasHeaderRow {
		if headers, err := c.reader.Read(); err == nil {
			c.FieldCount = len(headers)
			c.Headers = headers
		} else {
			return fmt.Errorf("CsvRead [%s] - Failed to read header row: %s", c.Name, err)
		}
	} else {
		if c.Headers != nil {
			c.FieldCount = len(c.Headers)
		}
	}

	// init channel
	c.output = make(chan interface{})
	c.logs = make(chan interface{})
	return nil
}

func (c *CsvRead) Uninit() error {
	return nil
}

func (c *CsvRead) GetName() string {
	return c.Name
}

func (c *CsvRead) GetOutput() <-chan interface{} {
	return c.output
}

func (c *CsvRead) GetLogs() <-chan interface{} {
	return c.logs
}

func (c *CsvRead) Exec() (err error) {
	go func() {
		defer func() {
			close(c.output)
			close(c.logs)
			err = c.file.Close()
		}()

		for {
			row, err := c.reader.Read()

			if err != nil {
				if err == io.EOF {
					return
				}
				c.logs <- api.ProcError{
					Err:      fmt.Errorf("CsvRead [%s] Error reading row: %s", err),
					ProcName: c.GetName(),
				}
				continue
			}

			c.output <- row
		}
	}()

	return nil
}
<commit_msg>Update error type in csvread.go<commit_after>package file

import (
	"encoding/csv"
	"fmt"
	"io"
	"os"

	"github.com/vladimirvivien/automi/api"
)

// CsvRead implements an Source process that reads the content of a
// specified file and emits its record via its Output Channel
// and serializes each row as a slice []string.
type CsvRead struct {
	Name          string   // string identifer for the Csv emitter
	FilePath      string   // path for the file
	DelimiterChar rune     // Delimiter charater, defaults to comma
	CommentChar   rune     // Charater indicating line is a comment
	Headers       []string // Column header names (specified here or read from file)
	HasHeaderRow  bool     // indicates first row is for headers (default false). Overrides the Headers attribute.
	FieldCount    int      // if greater than zero is used to validate field count

	file   *os.File
	reader *csv.Reader
	logs   chan interface{}
	output chan interface{}
}

func (c *CsvRead) Init() error {
	// validation
	if c.Name == "" {
		return api.ProcError{
			Err: fmt.Errorf("CsvRead missing an identifying name"),
		}
	}
	if c.FilePath == "" {
		return api.ProcError{
			ProcName: c.Name,
			Err:      fmt.Errorf("Missing required FilePath attribute"),
		}
	}

	// establish defaults
	if c.DelimiterChar == 0 {
		c.DelimiterChar = ','
	}

	if c.CommentChar == 0 {
		c.CommentChar = '#'
	}

	// open file
	file, err := os.Open(c.FilePath)
	if err != nil {
		return api.ProcError{
			ProcName: c.Name,
			Err:      fmt.Errorf("Failed to create file: %s ", err),
		}
	}

	c.file = file
	c.reader = csv.NewReader(file)
	c.reader.Comment = c.CommentChar
	c.reader.Comma = c.DelimiterChar

	// resolve header and field count
	if c.HasHeaderRow {
		if headers, err := c.reader.Read(); err == nil {
			c.FieldCount = len(headers)
			c.Headers = headers
		} else {
			return api.ProcError{
				ProcName: c.Name,
				Err:      fmt.Errorf("Unable to read header row: %s", err),
			}
		}
	} else {
		if c.Headers != nil {
			c.FieldCount = len(c.Headers)
		}
	}

	// init channel
	c.output = make(chan interface{})
	c.logs = make(chan interface{})
	return nil
}

func (c *CsvRead) Uninit() error {
	return nil
}

func (c *CsvRead) GetName() string {
	return c.Name
}

func (c *CsvRead) GetOutput() <-chan interface{} {
	return c.output
}

func (c *CsvRead) GetLogs() <-chan interface{} {
	return c.logs
}

func (c *CsvRead) Exec() (err error) {
	go func() {
		defer func() {
			close(c.output)
			close(c.logs)
			err = c.file.Close()
		}()

		for {
			row, err := c.reader.Read()

			if err != nil {
				if err == io.EOF {
					return
				}
				c.logs <- api.ProcError{
					Err:      fmt.Errorf("CsvRead [%s] Error reading row: %s", err),
					ProcName: c.GetName(),
				}
				continue
			}

			c.output <- row
		}
	}()

	return nil
}
<|endoftext|>


<commit_before><commit_msg>test case for “eq” comparing boolean values #547<commit_after><|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package mtail_test

import (
	"context"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"testing"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/metrics"
	"github.com/google/mtail/internal/metrics/datum"
	"github.com/google/mtail/internal/mtail"
	"github.com/google/mtail/internal/mtail/golden"
	"github.com/google/mtail/internal/testutil"
	"github.com/google/mtail/internal/waker"
	"golang.org/x/sys/unix"
)

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	goldenfile  string // Expected metrics after processing.
}{
	{
		"examples/rsyncd.mtail",
		"testdata/rsyncd.log",
		"testdata/rsyncd.golden",
	},
	{
		"examples/sftp.mtail",
		"testdata/sftp_chroot.log",
		"testdata/sftp_chroot.golden",
	},
	{
		"examples/dhcpd.mtail",
		"testdata/anonymised_dhcpd_log",
		"testdata/anonymised_dhcpd_log.golden",
	},
	{
		"examples/ntpd.mtail",
		"testdata/ntp4",
		"testdata/ntp4.golden",
	},
	{
		"examples/ntpd_peerstats.mtail",
		"testdata/xntp3_peerstats",
		"testdata/xntp3_peerstats.golden",
	},
	{
		"examples/otherwise.mtail",
		"testdata/otherwise.log",
		"testdata/otherwise.golden",
	},
	{
		"examples/else.mtail",
		"testdata/else.log",
		"testdata/else.golden",
	},
	{
		"examples/types.mtail",
		"testdata/types.log",
		"testdata/types.golden",
	},
	// TODO(jaq): getfilename() is broken by poll log patterns in tailer constructor.
	// {
	// 	"examples/filename.mtail",
	// 	"testdata/else.log",
	// 	"testdata/filename.golden",
	// },
	{
		"examples/logical.mtail",
		"testdata/logical.log",
		"testdata/logical.golden",
	},
	{
		"examples/strcat.mtail",
		"testdata/strcat.log",
		"testdata/strcat.golden",
	},
	{
		"examples/add_assign_float.mtail",
		"testdata/add_assign_float.log",
		"testdata/add_assign_float.golden",
	},
	{
		"examples/typed-comparison.mtail",
		"testdata/typed-comparison.log",
		"testdata/typed-comparison.golden",
	},
	{
		"examples/match-expression.mtail",
		"testdata/match-expression.log",
		"testdata/match-expression.golden",
	},
	{
		"examples/apache_combined.mtail",
		"testdata/apache-combined.log",
		"testdata/apache-combined.golden",
	},
	{
		"examples/apache_common.mtail",
		"testdata/apache-common.log",
		"testdata/apache-common.golden",
	},
	{
		"examples/metric-as-rvalue.mtail",
		"testdata/metric-as-rvalue.log",
		"testdata/metric-as-rvalue.golden",
	},
	{
		"examples/decorator.mtail",
		"testdata/decorator.log",
		"testdata/decorator.golden",
	},
	{
		"examples/stringy.mtail",
		"testdata/stringy.log",
		"testdata/stringy.golden",
	},
	{
		"examples/ip-addr.mtail",
		"testdata/ip-addr.log",
		"testdata/ip-addr.golden",
	},
	{
		"examples/vsftpd.mtail",
		"testdata/vsftpd_log",
		"testdata/vsftpd_log.golden",
	},
	{
		"examples/vsftpd.mtail",
		"testdata/vsftpd_xferlog",
		"testdata/vsftpd_xferlog.golden",
	},
	{
		"examples/lighttpd.mtail",
		"testdata/lighttpd_access.log",
		"testdata/lighttpd_accesslog.golden",
	},
	{
		"examples/mysql_slowqueries.mtail",
		"testdata/mysql_slowqueries.log",
		"testdata/mysql_slowqueries.golden",
	},
}

func TestExamplePrograms(t *testing.T) {
	testutil.SkipIfShort(t)
	for _, tc := range exampleProgramTests {
		t.Run(fmt.Sprintf("%s on %s", tc.programfile, tc.logfile), func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			waker, _ := waker.NewTest(0) // oneshot means we should never need to wake the stream
			store := metrics.NewStore()
			programFile := path.Join("../..", tc.programfile)
			mtail, err := mtail.New(ctx, store, mtail.ProgramPath(programFile), mtail.LogPathPatterns(tc.logfile), mtail.OneShot, mtail.OmitMetricSource, mtail.DumpAstTypes, mtail.DumpBytecode, mtail.OmitDumpMetricStore, mtail.LogPatternPollWaker(waker), mtail.LogstreamPollWaker(waker))
			testutil.FatalIfErr(t, err)

			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()
				testutil.FatalIfErr(t, mtail.Run())
			}()
			// Oneshot mode means we can wait for shutdown before cancelling.
			wg.Wait()
			cancel()

			g, err := os.Open(tc.goldenfile)
			testutil.FatalIfErr(t, err)
			defer g.Close()

			goldenStore := metrics.NewStore()
			golden.ReadTestData(g, tc.programfile, goldenStore)

			testutil.ExpectNoDiff(t, goldenStore, store, testutil.IgnoreUnexported(sync.RWMutex{}, datum.String{}))
		})
	}
}

// This test only compiles examples, but has coverage over all examples
// provided.  This ensures we ship at least syntactically correct examples.
func TestCompileExamplePrograms(t *testing.T) {
	testutil.SkipIfShort(t)
	matches, err := filepath.Glob("../../examples/*.mtail")
	testutil.FatalIfErr(t, err)
	for _, tc := range matches {
		name := filepath.Base(tc)
		t.Run(name, func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			s := metrics.NewStore()
			mtail, err := mtail.New(ctx, s, mtail.ProgramPath(tc), mtail.CompileOnly, mtail.OmitMetricSource, mtail.DumpAstTypes, mtail.DumpBytecode, mtail.OmitDumpMetricStore)
			testutil.FatalIfErr(t, err)
			// Ensure that run shuts down for CompileOnly
			testutil.FatalIfErr(t, mtail.Run())
			cancel()
		})
	}
}

func BenchmarkProgram(b *testing.B) {
	for _, bm := range exampleProgramTests {
		bm := bm
		b.Run(fmt.Sprintf("%s on %s", bm.programfile, bm.logfile), func(b *testing.B) {
			b.ReportAllocs()
			logDir, rmLogDir := testutil.TestTempDir(b)
			defer rmLogDir()
			logFile := path.Join(logDir, "test.log")
			log := testutil.TestOpenFile(b, logFile)
			waker, awaken := waker.NewTest(1)
			store := metrics.NewStore()
			programFile := path.Join("../..", bm.programfile)
			ctx, cancel := context.WithCancel(context.Background())
			mtail, err := mtail.New(ctx, store, mtail.ProgramPath(programFile), mtail.LogPathPatterns(log.Name()), mtail.LogstreamPollWaker(waker))
			testutil.FatalIfErr(b, err)

			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()
				testutil.FatalIfErr(b, mtail.Run())
			}()

			var total int64
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				l, err := os.Open(bm.logfile)
				testutil.FatalIfErr(b, err)
				count, err := io.Copy(log, l)
				testutil.FatalIfErr(b, err)
				total += count
				awaken(1)
			}
			cancel()
			wg.Wait()
			b.StopTimer()
			b.SetBytes(total)
		})
	}
}

// Two mtails both alike in dignity.
func TestFilePipeStreamComparison(t *testing.T) {
	testutil.SkipIfShort(t)

	for _, tc := range exampleProgramTests {
		t.Run(fmt.Sprintf("%s on %s", tc.programfile, tc.logfile), func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			waker, _ := waker.NewTest(0) // oneshot means we should never need to wake the stream
			fileStore, pipeStore := metrics.NewStore(), metrics.NewStore()
			programFile := path.Join("../..", tc.programfile)

			var wg sync.WaitGroup
			wg.Add(2)
			go func() {
				defer wg.Done()
				mtail, err := mtail.New(ctx, fileStore, mtail.ProgramPath(programFile), mtail.LogPathPatterns(tc.logfile), mtail.OneShot, mtail.OmitMetricSource, mtail.OmitDumpMetricStore, mtail.LogPatternPollWaker(waker), mtail.LogstreamPollWaker(waker))
				testutil.FatalIfErr(t, err)
				testutil.FatalIfErr(t, mtail.Run())
			}()
			go func() {
				defer wg.Done()
				tmpDir, rmTmpDir := testutil.TestTempDir(t)
				defer rmTmpDir()
				// Some of our examples use getfilename() which means we need to retain the filename!
				pipeName := filepath.Join(tmpDir, filepath.Base(tc.logfile))
				testutil.FatalIfErr(t, unix.Mkfifo(pipeName, 0600))
				go func() {
					pipe, err := os.OpenFile(pipeName, os.O_WRONLY, os.ModeNamedPipe)
					testutil.FatalIfErr(t, err)
					f, err := os.OpenFile(tc.logfile, os.O_RDONLY, 0)
					testutil.FatalIfErr(t, err)
					defer f.Close()
					n, err := io.Copy(pipe, f)
					testutil.FatalIfErr(t, err)
					glog.Infof("Copied %d bytes into pipe", n)
					pipe.Close()
				}()
				mtail, err := mtail.New(ctx, pipeStore, mtail.ProgramPath(programFile), mtail.LogPathPatterns(pipeName), mtail.OneShot, mtail.OmitMetricSource, mtail.OmitDumpMetricStore, mtail.LogPatternPollWaker(waker), mtail.LogstreamPollWaker(waker))
				testutil.FatalIfErr(t, err)
				testutil.FatalIfErr(t, mtail.Run())
			}()
			// Oneshot mode means we can wait for shutdown before cancelling.
			wg.Wait()
			cancel()

			// Ignore the usual field and the datum.Time field as well, as the results will be unstable otherwise.
			testutil.ExpectNoDiff(t, fileStore, pipeStore, testutil.IgnoreUnexported(sync.RWMutex{}, datum.String{}), testutil.IgnoreFields(datum.BaseDatum{}, "Time"))
		})
	}
}
<commit_msg>Avoid another race by only calling testing.Fatal in the main test goroutine.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package mtail_test

import (
	"context"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"testing"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/metrics"
	"github.com/google/mtail/internal/metrics/datum"
	"github.com/google/mtail/internal/mtail"
	"github.com/google/mtail/internal/mtail/golden"
	"github.com/google/mtail/internal/testutil"
	"github.com/google/mtail/internal/waker"
	"golang.org/x/sys/unix"
)

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	goldenfile  string // Expected metrics after processing.
}{
	{
		"examples/rsyncd.mtail",
		"testdata/rsyncd.log",
		"testdata/rsyncd.golden",
	},
	{
		"examples/sftp.mtail",
		"testdata/sftp_chroot.log",
		"testdata/sftp_chroot.golden",
	},
	{
		"examples/dhcpd.mtail",
		"testdata/anonymised_dhcpd_log",
		"testdata/anonymised_dhcpd_log.golden",
	},
	{
		"examples/ntpd.mtail",
		"testdata/ntp4",
		"testdata/ntp4.golden",
	},
	{
		"examples/ntpd_peerstats.mtail",
		"testdata/xntp3_peerstats",
		"testdata/xntp3_peerstats.golden",
	},
	{
		"examples/otherwise.mtail",
		"testdata/otherwise.log",
		"testdata/otherwise.golden",
	},
	{
		"examples/else.mtail",
		"testdata/else.log",
		"testdata/else.golden",
	},
	{
		"examples/types.mtail",
		"testdata/types.log",
		"testdata/types.golden",
	},
	// TODO(jaq): getfilename() is broken by poll log patterns in tailer constructor.
	// {
	// 	"examples/filename.mtail",
	// 	"testdata/else.log",
	// 	"testdata/filename.golden",
	// },
	{
		"examples/logical.mtail",
		"testdata/logical.log",
		"testdata/logical.golden",
	},
	{
		"examples/strcat.mtail",
		"testdata/strcat.log",
		"testdata/strcat.golden",
	},
	{
		"examples/add_assign_float.mtail",
		"testdata/add_assign_float.log",
		"testdata/add_assign_float.golden",
	},
	{
		"examples/typed-comparison.mtail",
		"testdata/typed-comparison.log",
		"testdata/typed-comparison.golden",
	},
	{
		"examples/match-expression.mtail",
		"testdata/match-expression.log",
		"testdata/match-expression.golden",
	},
	{
		"examples/apache_combined.mtail",
		"testdata/apache-combined.log",
		"testdata/apache-combined.golden",
	},
	{
		"examples/apache_common.mtail",
		"testdata/apache-common.log",
		"testdata/apache-common.golden",
	},
	{
		"examples/metric-as-rvalue.mtail",
		"testdata/metric-as-rvalue.log",
		"testdata/metric-as-rvalue.golden",
	},
	{
		"examples/decorator.mtail",
		"testdata/decorator.log",
		"testdata/decorator.golden",
	},
	{
		"examples/stringy.mtail",
		"testdata/stringy.log",
		"testdata/stringy.golden",
	},
	{
		"examples/ip-addr.mtail",
		"testdata/ip-addr.log",
		"testdata/ip-addr.golden",
	},
	{
		"examples/vsftpd.mtail",
		"testdata/vsftpd_log",
		"testdata/vsftpd_log.golden",
	},
	{
		"examples/vsftpd.mtail",
		"testdata/vsftpd_xferlog",
		"testdata/vsftpd_xferlog.golden",
	},
	{
		"examples/lighttpd.mtail",
		"testdata/lighttpd_access.log",
		"testdata/lighttpd_accesslog.golden",
	},
	{
		"examples/mysql_slowqueries.mtail",
		"testdata/mysql_slowqueries.log",
		"testdata/mysql_slowqueries.golden",
	},
}

func TestExamplePrograms(t *testing.T) {
	testutil.SkipIfShort(t)
	for _, tc := range exampleProgramTests {
		t.Run(fmt.Sprintf("%s on %s", tc.programfile, tc.logfile), func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			waker, _ := waker.NewTest(0) // oneshot means we should never need to wake the stream
			store := metrics.NewStore()
			programFile := path.Join("../..", tc.programfile)
			mtail, err := mtail.New(ctx, store, mtail.ProgramPath(programFile), mtail.LogPathPatterns(tc.logfile), mtail.OneShot, mtail.OmitMetricSource, mtail.DumpAstTypes, mtail.DumpBytecode, mtail.OmitDumpMetricStore, mtail.LogPatternPollWaker(waker), mtail.LogstreamPollWaker(waker))
			testutil.FatalIfErr(t, err)

			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()
				testutil.FatalIfErr(t, mtail.Run())
			}()
			// Oneshot mode means we can wait for shutdown before cancelling.
			wg.Wait()
			cancel()

			g, err := os.Open(tc.goldenfile)
			testutil.FatalIfErr(t, err)
			defer g.Close()

			goldenStore := metrics.NewStore()
			golden.ReadTestData(g, tc.programfile, goldenStore)

			testutil.ExpectNoDiff(t, goldenStore, store, testutil.IgnoreUnexported(sync.RWMutex{}, datum.String{}))
		})
	}
}

// This test only compiles examples, but has coverage over all examples
// provided.  This ensures we ship at least syntactically correct examples.
func TestCompileExamplePrograms(t *testing.T) {
	testutil.SkipIfShort(t)
	matches, err := filepath.Glob("../../examples/*.mtail")
	testutil.FatalIfErr(t, err)
	for _, tc := range matches {
		name := filepath.Base(tc)
		t.Run(name, func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			s := metrics.NewStore()
			mtail, err := mtail.New(ctx, s, mtail.ProgramPath(tc), mtail.CompileOnly, mtail.OmitMetricSource, mtail.DumpAstTypes, mtail.DumpBytecode, mtail.OmitDumpMetricStore)
			testutil.FatalIfErr(t, err)
			// Ensure that run shuts down for CompileOnly
			testutil.FatalIfErr(t, mtail.Run())
			cancel()
		})
	}
}

func BenchmarkProgram(b *testing.B) {
	for _, bm := range exampleProgramTests {
		bm := bm
		b.Run(fmt.Sprintf("%s on %s", bm.programfile, bm.logfile), func(b *testing.B) {
			b.ReportAllocs()
			logDir, rmLogDir := testutil.TestTempDir(b)
			defer rmLogDir()
			logFile := path.Join(logDir, "test.log")
			log := testutil.TestOpenFile(b, logFile)
			waker, awaken := waker.NewTest(1)
			store := metrics.NewStore()
			programFile := path.Join("../..", bm.programfile)
			ctx, cancel := context.WithCancel(context.Background())
			mtail, err := mtail.New(ctx, store, mtail.ProgramPath(programFile), mtail.LogPathPatterns(log.Name()), mtail.LogstreamPollWaker(waker))
			testutil.FatalIfErr(b, err)

			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()
				testutil.FatalIfErr(b, mtail.Run())
			}()

			var total int64
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				l, err := os.Open(bm.logfile)
				testutil.FatalIfErr(b, err)
				count, err := io.Copy(log, l)
				testutil.FatalIfErr(b, err)
				total += count
				awaken(1)
			}
			cancel()
			wg.Wait()
			b.StopTimer()
			b.SetBytes(total)
		})
	}
}

// Two mtails both alike in dignity.
func TestFilePipeStreamComparison(t *testing.T) {
	testutil.SkipIfShort(t)

	for _, tc := range exampleProgramTests {
		t.Run(fmt.Sprintf("%s on %s", tc.programfile, tc.logfile), func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			waker, _ := waker.NewTest(0) // oneshot means we should never need to wake the stream
			fileStore, pipeStore := metrics.NewStore(), metrics.NewStore()
			programFile := path.Join("../..", tc.programfile)

			errc := make(chan error)
			var wg sync.WaitGroup
			wg.Add(2)
			go func() {
				defer wg.Done()
				mtail, err := mtail.New(ctx, fileStore, mtail.ProgramPath(programFile), mtail.LogPathPatterns(tc.logfile), mtail.OneShot, mtail.OmitMetricSource, mtail.OmitDumpMetricStore, mtail.LogPatternPollWaker(waker), mtail.LogstreamPollWaker(waker))
				if err != nil {
					errc <- err
				}
				if err = mtail.Run(); err != nil {
					errc <- err
				}
			}()
			go func() {
				defer wg.Done()
				tmpDir, rmTmpDir := testutil.TestTempDir(t)
				defer rmTmpDir()
				// Some of our examples use getfilename() which means we need to retain the filename!
				pipeName := filepath.Join(tmpDir, filepath.Base(tc.logfile))
				err := unix.Mkfifo(pipeName, 0600)
				if err != nil {
					errc <- err
				}
				go func() {
					pipe, err := os.OpenFile(pipeName, os.O_WRONLY, os.ModeNamedPipe)
					if err != nil {
						errc <- err
					}
					f, err := os.OpenFile(tc.logfile, os.O_RDONLY, 0)
					if err != nil {
						errc <- err
					}
					defer f.Close()
					n, err := io.Copy(pipe, f)
					if err != nil {

						errc <- err
					}
					glog.Infof("Copied %d bytes into pipe", n)
					pipe.Close()
				}()
				mtail, err := mtail.New(ctx, pipeStore, mtail.ProgramPath(programFile), mtail.LogPathPatterns(pipeName), mtail.OneShot, mtail.OmitMetricSource, mtail.OmitDumpMetricStore, mtail.LogPatternPollWaker(waker), mtail.LogstreamPollWaker(waker))
				if err != nil {
					errc <- err
				}
				if err = mtail.Run(); err != nil {
					errc <- err
				}
			}()
			go func() {
				// Oneshot mode means we can wait for shutdown before cancelling.
				wg.Wait()
				cancel()
				errc <- nil
			}()
			testutil.FatalIfErr(t, <-errc)

			// Ignore the usual field and the datum.Time field as well, as the results will be unstable otherwise.
			testutil.ExpectNoDiff(t, fileStore, pipeStore, testutil.IgnoreUnexported(sync.RWMutex{}, datum.String{}), testutil.IgnoreFields(datum.BaseDatum{}, "Time"))
		})
	}
}
<|endoftext|>


<commit_before>package broadcastwriter

import (
	"bytes"
	"io"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/jsonlog"
	"github.com/docker/docker/pkg/timeutils"
)

// BroadcastWriter accumulate multiple io.WriteCloser by stream.
type BroadcastWriter struct {
	sync.Mutex
	buf      *bytes.Buffer
	jsLogBuf *bytes.Buffer
	streams  map[string](map[io.WriteCloser]struct{})
}

// AddWriter adds new io.WriteCloser for stream.
// If stream is "", then all writes proceed as is. Otherwise every line from
// input will be packed to serialized jsonlog.JSONLog.
func (w *BroadcastWriter) AddWriter(writer io.WriteCloser, stream string) {
	w.Lock()
	if _, ok := w.streams[stream]; !ok {
		w.streams[stream] = make(map[io.WriteCloser]struct{})
	}
	w.streams[stream][writer] = struct{}{}
	w.Unlock()
}

// Write writes bytes to all writers. Failed writers will be evicted during
// this call.
func (w *BroadcastWriter) Write(p []byte) (n int, err error) {
	w.Lock()
	if writers, ok := w.streams[""]; ok {
		for sw := range writers {
			if n, err := sw.Write(p); err != nil || n != len(p) {
				// On error, evict the writer
				delete(writers, sw)
			}
		}
		// exit if there is no more writers
		if len(w.streams) == 1 {
			w.buf.Reset()
			w.Unlock()
			return len(p), nil
		}
	}
	if w.jsLogBuf == nil {
		w.jsLogBuf = new(bytes.Buffer)
		w.jsLogBuf.Grow(1024)
	}
	var timestamp string
	created := time.Now().UTC()
	w.buf.Write(p)
	for {
		if n := w.buf.Len(); n == 0 {
			break
		}
		i := bytes.IndexByte(w.buf.Bytes(), '\n')
		if i < 0 {
			break
		}
		lineBytes := w.buf.Next(i + 1)
		if timestamp == "" {
			timestamp, err = timeutils.FastMarshalJSON(created)
			if err != nil {
				continue
			}
		}

		for stream, writers := range w.streams {
			if stream == "" {
				continue
			}
			jsonLog := jsonlog.JSONLogBytes{Log: lineBytes, Stream: stream, Created: timestamp}
			err = jsonLog.MarshalJSONBuf(w.jsLogBuf)
			if err != nil {
				log.Errorf("Error making JSON log line: %s", err)
				continue
			}
			w.jsLogBuf.WriteByte('\n')
			b := w.jsLogBuf.Bytes()
			for sw := range writers {
				if _, err := sw.Write(b); err != nil {
					delete(writers, sw)
				}
			}
		}
		w.jsLogBuf.Reset()
	}
	w.jsLogBuf.Reset()
	w.Unlock()
	return len(p), nil
}

// Clean closes and removes all writers. Last non-eol-terminated part of data
// will be saved.
func (w *BroadcastWriter) Clean() error {
	w.Lock()
	for _, writers := range w.streams {
		for w := range writers {
			w.Close()
		}
	}
	w.streams = make(map[string](map[io.WriteCloser]struct{}))
	w.Unlock()
	return nil
}

func New() *BroadcastWriter {
	return &BroadcastWriter{
		streams: make(map[string](map[io.WriteCloser]struct{})),
		buf:     bytes.NewBuffer(nil),
	}
}
<commit_msg>pkg/broadcastwriter: reset after 4 KB w/o stream<commit_after>package broadcastwriter

import (
	"bytes"
	"io"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/jsonlog"
	"github.com/docker/docker/pkg/timeutils"
)

// BroadcastWriter accumulate multiple io.WriteCloser by stream.
type BroadcastWriter struct {
	sync.Mutex
	buf      *bytes.Buffer
	jsLogBuf *bytes.Buffer
	streams  map[string](map[io.WriteCloser]struct{})
}

// AddWriter adds new io.WriteCloser for stream.
// If stream is "", then all writes proceed as is. Otherwise every line from
// input will be packed to serialized jsonlog.JSONLog.
func (w *BroadcastWriter) AddWriter(writer io.WriteCloser, stream string) {
	w.Lock()
	if _, ok := w.streams[stream]; !ok {
		w.streams[stream] = make(map[io.WriteCloser]struct{})
	}
	w.streams[stream][writer] = struct{}{}
	w.Unlock()
}

// Write writes bytes to all writers. Failed writers will be evicted during
// this call.
func (w *BroadcastWriter) Write(p []byte) (n int, err error) {
	w.Lock()
	if writers, ok := w.streams[""]; ok {
		for sw := range writers {
			if n, err := sw.Write(p); err != nil || n != len(p) {
				// On error, evict the writer
				delete(writers, sw)
			}
		}
		if len(w.streams) == 1 {
			if w.buf.Len() >= 4096 {
				w.buf.Reset()
			} else {
				w.buf.Write(p)
			}
			w.Unlock()
			return len(p), nil
		}
	}
	if w.jsLogBuf == nil {
		w.jsLogBuf = new(bytes.Buffer)
		w.jsLogBuf.Grow(1024)
	}
	var timestamp string
	created := time.Now().UTC()
	w.buf.Write(p)
	for {
		if n := w.buf.Len(); n == 0 {
			break
		}
		i := bytes.IndexByte(w.buf.Bytes(), '\n')
		if i < 0 {
			break
		}
		lineBytes := w.buf.Next(i + 1)
		if timestamp == "" {
			timestamp, err = timeutils.FastMarshalJSON(created)
			if err != nil {
				continue
			}
		}

		for stream, writers := range w.streams {
			if stream == "" {
				continue
			}
			jsonLog := jsonlog.JSONLogBytes{Log: lineBytes, Stream: stream, Created: timestamp}
			err = jsonLog.MarshalJSONBuf(w.jsLogBuf)
			if err != nil {
				log.Errorf("Error making JSON log line: %s", err)
				continue
			}
			w.jsLogBuf.WriteByte('\n')
			b := w.jsLogBuf.Bytes()
			for sw := range writers {
				if _, err := sw.Write(b); err != nil {
					delete(writers, sw)
				}
			}
		}
		w.jsLogBuf.Reset()
	}
	w.jsLogBuf.Reset()
	w.Unlock()
	return len(p), nil
}

// Clean closes and removes all writers. Last non-eol-terminated part of data
// will be saved.
func (w *BroadcastWriter) Clean() error {
	w.Lock()
	for _, writers := range w.streams {
		for w := range writers {
			w.Close()
		}
	}
	w.streams = make(map[string](map[io.WriteCloser]struct{}))
	w.Unlock()
	return nil
}

func New() *BroadcastWriter {
	return &BroadcastWriter{
		streams: make(map[string](map[io.WriteCloser]struct{})),
		buf:     bytes.NewBuffer(nil),
	}
}
<|endoftext|>