<commit_before>//
// Copyright: 2016, mgIT GmbH <office@mgit.at>
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"flag"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	etcd "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

const (
	LOCK_FILE_BASE = "/singleton.mgit.at"
)

var (
// Machines contains a list of all etcd machines (http address)
)

// command line flags
var (
	flagNameTemplate   = flag.String("name-template", "", "template for the lockfile name (will get expanded using environment variables)")
	flagInstTemplate   = flag.String("instance-template", "", "template for the instance name (will get expanded using environment variables)")
	flagRequestTimeout = flag.Uint("request-timeout", 5, "timeout in seconds to wait for responses from etcd")
	flagUpdateInterval = flag.Uint("update-interval", 30, "interval in seconds between lock file update requests")
	flagGracePeriod    = flag.Uint("grace-period", 30, "time in seconds to wait for a normal shutdown of the child")
	flagKillBackoff    = flag.Uint("kill-delay", 5, "")
	flagEtcd           = flag.String("etcd", "http://127.0.0.1:2379", "etcd machines (comma separated list)")
	flagCA             = flag.String("ca", "", "CA certificate")
	flagCert           = flag.String("cert", "", "client certificate")
	flagKey            = flag.String("key", "", "client key")
)

func runChild(cmd string, args []string, signals <-chan os.Signal) (exited chan bool, err error) {
	child := exec.Cmd{}
	child.Path = cmd
	child.Args = args
	child.Stdin = os.Stdin
	child.Stdout = os.Stdout
	child.Stderr = os.Stderr

	if err = child.Start(); err != nil {
		return
	}

	exited = make(chan bool, 1)
	go func() {
		defer func() {
			exited <- true
		}()
		err := child.Wait()
		if err != nil {
			log.Printf("singleton-runner: child exited with: %v", err)
		} else {
			log.Printf("singleton-runner: child exited normally")
		}
	}()

	go func() {
		for sig := range signals {
			child.Process.Signal(sig)
		}
	}()
	return
}

func initETCdClient(machines []string, CA, cert, key string, timeout time.Duration) (etcd.KeysAPI, error) {
	var tlsConfig *tls.Config
	if cert != "" && key != "" && CA != "" {
		cert, err := tls.LoadX509KeyPair(cert, key)
		if err != nil {
			log.Fatal("failed to load client key pair:", err)
		}
		caCert, err := ioutil.ReadFile(CA)
		if err != nil {
			log.Fatal("failed to load CA:", err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		tlsConfig = &tls.Config{
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
		tlsConfig.BuildNameToCertificate()
	}

	cfg := etcd.Config{
		Endpoints: machines,
		Transport: &http.Transport{
			Proxy:           http.ProxyFromEnvironment,
			TLSClientConfig: tlsConfig,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
		},
		HeaderTimeoutPerRequest: 5 * time.Second,
	}

	c, err := etcd.New(cfg)
	if err != nil {
		return nil, err
	}
	kapi := etcd.NewKeysAPI(c)
	return kapi, err
}

func IsKeyExists(err error) bool {
	if cErr, ok := err.(etcd.Error); ok {
		return cErr.Code == etcd.ErrorCodeNodeExist
	}
	return false
}

func acquireLock(kapi etcd.KeysAPI, lockfile, instanceID string, ttl, timeout time.Duration) (err error) {
	for {
		log.Printf("singleton-runner: trying to acquire lock: %s (TTL: %v)", lockfile, ttl)

		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		opts := &etcd.SetOptions{PrevExist: etcd.PrevNoExist, TTL: ttl, Refresh: false}
		_, err = kapi.Set(ctx, lockfile, instanceID, opts)
		cancel()
		if err == nil {
			return // we got the lock!
		}
		if !IsKeyExists(err) {
			return // this seems to be a severe problem.. better give up
		}
		log.Printf("singleton-runner: lock is already acquired - watching for changes")

		ctx, cancel = context.WithTimeout(context.Background(), ttl)
		watcher := kapi.Watcher(lockfile, nil)
	WatchLock:
		for {
			resp, err := watcher.Next(ctx)
			if err != nil {
				log.Printf("singleton-runner: watching for events failed: %v", err)
				return err
			}
			switch resp.Action {
			case "expire":
				fallthrough
			case "compareAndDelete":
				fallthrough
			case "delete":
				log.Printf("singleton-runner: delete or expire event - try again!")
				break WatchLock
			case "compareAndSwap":
				return errors.New("locked instance seems to be alive - giving up")
			default:
				log.Printf("singleton-runner: ignoring unknown event '%s'", resp.Action)
			}
		}
	}
	return
}

func releaseLock(kapi etcd.KeysAPI, lockfile, instanceID string) {
	log.Printf("singleton-runner: trying to release lock: %s", lockfile)
	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
	opts := &etcd.DeleteOptions{PrevValue: instanceID}
	_, err := kapi.Delete(ctx, lockfile, opts)
	cancel()
	if err != nil {
		log.Printf("singleton-runner: releasing lock failed: %s", err)
	}
	return
}

func updateLock(kapi etcd.KeysAPI, lockfile, instanceID string, ttl, timeout time.Duration) (err error) {
	log.Printf("singleton-runner: trying to update lock: %s (TTL: %v, Timeout: %v)", lockfile, ttl, timeout)

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	opts := &etcd.SetOptions{PrevValue: instanceID, PrevExist: etcd.PrevExist, TTL: ttl, Refresh: false}
	_, err = kapi.Set(ctx, lockfile, instanceID, opts)
	cancel()
	return
}

func main() {
	//
	// **** parse command line
	//
	flag.Parse()

	if *flagNameTemplate == "" {
		log.Fatal("singleton-runner: '-name-template' is empty")
	}
	name := os.ExpandEnv(*flagNameTemplate)
	lockfilePath := filepath.Join(LOCK_FILE_BASE, path.Clean("/"+name))

	if *flagInstTemplate == "" {
		log.Fatal("singleton-runner: '-instance-template' is empty")
	}
	instanceID := os.ExpandEnv(*flagInstTemplate)

	requestTimeout := time.Duration(*flagRequestTimeout) * time.Second
	updateInterval := time.Duration(*flagUpdateInterval) * time.Second
	gracePeriod := time.Duration(*flagGracePeriod) * time.Second
	killBackoff := time.Duration(*flagKillBackoff) * time.Second
	ttl := updateInterval + requestTimeout + gracePeriod + killBackoff

	machines := strings.Split(*flagEtcd, ",")
	CA := *flagCA
	cert := *flagCert
	key := *flagKey

	var cmd string
	args := flag.Args()
	if args := flag.Args(); len(args) > 0 {
		cmd = args[0]
	} else {
		log.Fatal("singleton-runner: please specify a command to run")
	}

	//
	// **** initialization
	//

	kapi, err := initETCdClient(machines, CA, cert, key, requestTimeout)
	if err != nil {
		log.Fatal("error connecting to etcd:", err)
	}
	// defer releaseLock(kapi, lockfilePath, instanceID) // should this be done in any case?

	//
	// **** try to get the lock
	//
	if err := acquireLock(kapi, lockfilePath, instanceID, ttl, requestTimeout); err != nil {
		log.Fatal("singleton-runner: unable to acquire lock: ", err)
	}
	log.Printf("singleton-runner: lock acquired successfully! .. starting %q", cmd)

	//
	// **** run the child
	//
	signals := make(chan os.Signal)
	signal.Notify(signals, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
	exited, err := runChild(cmd, args, signals)
	if err != nil {
		releaseLock(kapi, lockfilePath, instanceID) // remove here if we do this using the defer above
		log.Fatal("singleton-runner: calling child failed:", err)
	}

	//
	// **** update the lock - release lock if child exits
	//
	t := time.NewTicker(updateInterval)
Loop:
	for {
		select {
		case <-t.C:
			if err := updateLock(kapi, lockfilePath, instanceID, ttl, requestTimeout); err != nil {
				log.Println("singleton-runner: updateting lock failed:", err)
				log.Println("singleton-runner: sending child the TERM signal")
				signals <- syscall.SIGTERM
				break Loop
			}
		case <-exited:
			log.Println("singleton-runner: closing...")
			releaseLock(kapi, lockfilePath, instanceID) // remove here if we do this using the defer above
			return
		}
	}

	//
	// **** wait for child to exit - kill it after grace period
	//
	k := time.NewTimer(gracePeriod)
	select {
	case <-k.C:
		log.Println("singleton-runner: grace period elapsed sending child the KILL signal")
		signals <- syscall.SIGKILL
	case <-exited:
		log.Println("singleton-runner: exiting after child stopped")
		releaseLock(kapi, lockfilePath, instanceID) // remove here if we do this using the defer above
		return
	}
}
<commit_msg>renamed kill-delay argument to kill-backoff<commit_after>//
// Copyright: 2016, mgIT GmbH <office@mgit.at>
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"flag"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	etcd "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

const (
	LOCK_FILE_BASE = "/singleton.mgit.at"
)

var (
// Machines contains a list of all etcd machines (http address)
)

// command line flags
var (
	flagNameTemplate   = flag.String("name-template", "", "template for the lockfile name (will get expanded using environment variables)")
	flagInstTemplate   = flag.String("instance-template", "", "template for the instance name (will get expanded using environment variables)")
	flagRequestTimeout = flag.Uint("request-timeout", 5, "timeout in seconds to wait for responses from etcd")
	flagUpdateInterval = flag.Uint("update-interval", 30, "interval in seconds between lock file update requests")
	flagGracePeriod    = flag.Uint("grace-period", 30, "time in seconds to wait for a normal shutdown of the child")
	flagKillBackoff    = flag.Uint("kill-backoff", 5, "")
	flagEtcd           = flag.String("etcd", "http://127.0.0.1:2379", "etcd machines (comma separated list)")
	flagCA             = flag.String("ca", "", "CA certificate")
	flagCert           = flag.String("cert", "", "client certificate")
	flagKey            = flag.String("key", "", "client key")
)

func runChild(cmd string, args []string, signals <-chan os.Signal) (exited chan bool, err error) {
	child := exec.Cmd{}
	child.Path = cmd
	child.Args = args
	child.Stdin = os.Stdin
	child.Stdout = os.Stdout
	child.Stderr = os.Stderr

	if err = child.Start(); err != nil {
		return
	}

	exited = make(chan bool, 1)
	go func() {
		defer func() {
			exited <- true
		}()
		err := child.Wait()
		if err != nil {
			log.Printf("singleton-runner: child exited with: %v", err)
		} else {
			log.Printf("singleton-runner: child exited normally")
		}
	}()

	go func() {
		for sig := range signals {
			child.Process.Signal(sig)
		}
	}()
	return
}

func initETCdClient(machines []string, CA, cert, key string, timeout time.Duration) (etcd.KeysAPI, error) {
	var tlsConfig *tls.Config
	if cert != "" && key != "" && CA != "" {
		cert, err := tls.LoadX509KeyPair(cert, key)
		if err != nil {
			log.Fatal("failed to load client key pair:", err)
		}
		caCert, err := ioutil.ReadFile(CA)
		if err != nil {
			log.Fatal("failed to load CA:", err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		tlsConfig = &tls.Config{
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
		tlsConfig.BuildNameToCertificate()
	}

	cfg := etcd.Config{
		Endpoints: machines,
		Transport: &http.Transport{
			Proxy:           http.ProxyFromEnvironment,
			TLSClientConfig: tlsConfig,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
		},
		HeaderTimeoutPerRequest: 5 * time.Second,
	}

	c, err := etcd.New(cfg)
	if err != nil {
		return nil, err
	}
	kapi := etcd.NewKeysAPI(c)
	return kapi, err
}

func IsKeyExists(err error) bool {
	if cErr, ok := err.(etcd.Error); ok {
		return cErr.Code == etcd.ErrorCodeNodeExist
	}
	return false
}

func acquireLock(kapi etcd.KeysAPI, lockfile, instanceID string, ttl, timeout time.Duration) (err error) {
	for {
		log.Printf("singleton-runner: trying to acquire lock: %s (TTL: %v)", lockfile, ttl)

		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		opts := &etcd.SetOptions{PrevExist: etcd.PrevNoExist, TTL: ttl, Refresh: false}
		_, err = kapi.Set(ctx, lockfile, instanceID, opts)
		cancel()
		if err == nil {
			return // we got the lock!
		}
		if !IsKeyExists(err) {
			return // this seems to be a severe problem.. better give up
		}
		log.Printf("singleton-runner: lock is already acquired - watching for changes")

		ctx, cancel = context.WithTimeout(context.Background(), ttl)
		watcher := kapi.Watcher(lockfile, nil)
	WatchLock:
		for {
			resp, err := watcher.Next(ctx)
			if err != nil {
				log.Printf("singleton-runner: watching for events failed: %v", err)
				return err
			}
			switch resp.Action {
			case "expire":
				fallthrough
			case "compareAndDelete":
				fallthrough
			case "delete":
				log.Printf("singleton-runner: delete or expire event - try again!")
				break WatchLock
			case "compareAndSwap":
				return errors.New("locked instance seems to be alive - giving up")
			default:
				log.Printf("singleton-runner: ignoring unknown event '%s'", resp.Action)
			}
		}
	}
	return
}

func releaseLock(kapi etcd.KeysAPI, lockfile, instanceID string) {
	log.Printf("singleton-runner: trying to release lock: %s", lockfile)
	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
	opts := &etcd.DeleteOptions{PrevValue: instanceID}
	_, err := kapi.Delete(ctx, lockfile, opts)
	cancel()
	if err != nil {
		log.Printf("singleton-runner: releasing lock failed: %s", err)
	}
	return
}

func updateLock(kapi etcd.KeysAPI, lockfile, instanceID string, ttl, timeout time.Duration) (err error) {
	log.Printf("singleton-runner: trying to update lock: %s (TTL: %v, Timeout: %v)", lockfile, ttl, timeout)

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	opts := &etcd.SetOptions{PrevValue: instanceID, PrevExist: etcd.PrevExist, TTL: ttl, Refresh: false}
	_, err = kapi.Set(ctx, lockfile, instanceID, opts)
	cancel()
	return
}

func main() {
	//
	// **** parse command line
	//
	flag.Parse()

	if *flagNameTemplate == "" {
		log.Fatal("singleton-runner: '-name-template' is empty")
	}
	name := os.ExpandEnv(*flagNameTemplate)
	lockfilePath := filepath.Join(LOCK_FILE_BASE, path.Clean("/"+name))

	if *flagInstTemplate == "" {
		log.Fatal("singleton-runner: '-instance-template' is empty")
	}
	instanceID := os.ExpandEnv(*flagInstTemplate)

	requestTimeout := time.Duration(*flagRequestTimeout) * time.Second
	updateInterval := time.Duration(*flagUpdateInterval) * time.Second
	gracePeriod := time.Duration(*flagGracePeriod) * time.Second
	killBackoff := time.Duration(*flagKillBackoff) * time.Second
	ttl := updateInterval + requestTimeout + gracePeriod + killBackoff

	machines := strings.Split(*flagEtcd, ",")
	CA := *flagCA
	cert := *flagCert
	key := *flagKey

	var cmd string
	args := flag.Args()
	if args := flag.Args(); len(args) > 0 {
		cmd = args[0]
	} else {
		log.Fatal("singleton-runner: please specify a command to run")
	}

	//
	// **** initialization
	//

	kapi, err := initETCdClient(machines, CA, cert, key, requestTimeout)
	if err != nil {
		log.Fatal("error connecting to etcd:", err)
	}
	// defer releaseLock(kapi, lockfilePath, instanceID) // should this be done in any case?

	//
	// **** try to get the lock
	//
	if err := acquireLock(kapi, lockfilePath, instanceID, ttl, requestTimeout); err != nil {
		log.Fatal("singleton-runner: unable to acquire lock: ", err)
	}
	log.Printf("singleton-runner: lock acquired successfully! .. starting %q", cmd)

	//
	// **** run the child
	//
	signals := make(chan os.Signal)
	signal.Notify(signals, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
	exited, err := runChild(cmd, args, signals)
	if err != nil {
		releaseLock(kapi, lockfilePath, instanceID) // remove here if we do this using the defer above
		log.Fatal("singleton-runner: calling child failed:", err)
	}

	//
	// **** update the lock - release lock if child exits
	//
	t := time.NewTicker(updateInterval)
Loop:
	for {
		select {
		case <-t.C:
			if err := updateLock(kapi, lockfilePath, instanceID, ttl, requestTimeout); err != nil {
				log.Println("singleton-runner: updateting lock failed:", err)
				log.Println("singleton-runner: sending child the TERM signal")
				signals <- syscall.SIGTERM
				break Loop
			}
		case <-exited:
			log.Println("singleton-runner: closing...")
			releaseLock(kapi, lockfilePath, instanceID) // remove here if we do this using the defer above
			return
		}
	}

	//
	// **** wait for child to exit - kill it after grace period
	//
	k := time.NewTimer(gracePeriod)
	select {
	case <-k.C:
		log.Println("singleton-runner: grace period elapsed sending child the KILL signal")
		signals <- syscall.SIGKILL
	case <-exited:
		log.Println("singleton-runner: exiting after child stopped")
		releaseLock(kapi, lockfilePath, instanceID) // remove here if we do this using the defer above
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"sync"

	"gopkg.in/yaml.v2"

	mtr "github.com/Shinzu/go-mtr"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
)

type Exporter struct {
	mutex    sync.Mutex
	sent     *prometheus.CounterVec
	received *prometheus.CounterVec
	dropped  *prometheus.CounterVec
	lost     *prometheus.CounterVec
	latency  *prometheus.SummaryVec
	failed   *prometheus.CounterVec
}

type Config struct {
	Arguments []string `yaml:"args"`
	Hosts     []Host   `yaml:"hosts"`
}

type Host struct {
	Name  string `yaml:"name"`
	Alias string `yaml:"alias"`
}

type TargetFeedback struct {
	Target string
	Alias  string
	Hosts  []*mtr.Host
}

var config Config

const (
	Namespace = "mtr"
)

func NewExporter() *Exporter {
	var (
		alias  = "alias"
		server = "server"
		hop_id = "hop_id"
		hop_ip = "hop_ip"
	)

	return &Exporter{
		sent: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "sent",
				Help:      "packets sent",
			},
			[]string{alias, server, hop_id, hop_ip},
		),
		received: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "received",
				Help:      "packets received",
			},
			[]string{alias, server, hop_id, hop_ip},
		),
		dropped: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "dropped",
				Help:      "packets dropped",
			},
			[]string{alias, server, hop_id, hop_ip},
		),
		lost: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "lost",
				Help:      "packets lost",
			},
			[]string{alias, server, hop_id, hop_ip},
		),
		latency: prometheus.NewSummaryVec(
			prometheus.SummaryOpts{
				Namespace: Namespace,
				Name:      "latency",
				Help:      "packet latency in microseconds",
				Objectives: map[float64]float64{
					0.5:  0.05,
					0.9:  0.01,
					0.99: 0.001,
				},
				MaxAge:     prometheus.DefMaxAge,
				AgeBuckets: prometheus.DefAgeBuckets,
				BufCap:     prometheus.DefBufCap,
			},
			[]string{alias, server, hop_id, hop_ip},
		),
		failed: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "failed",
				Help:      "MTR runs failed",
			},
			[]string{alias, server},
		),
	}
}

func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.sent.Describe(ch)
	e.received.Describe(ch)
	e.dropped.Describe(ch)
	e.lost.Describe(ch)
	e.latency.Describe(ch)
	e.failed.Describe(ch)
}

func (e *Exporter) collect() error {
	for {
		results := make(chan *TargetFeedback)
		wg := &sync.WaitGroup{}
		wg.Add(len(config.Hosts))

		for w, host := range config.Hosts {
			go func(w int, host Host) {
				log.Infoln("worker", w, "processing job", host.Name, "aliased as", host.Alias)
				err := worker(w, host, results, wg)
				if err != nil {
					log.Errorf("worker %d failed job %v aliased as %v: %v\n", w, host.Name, host.Alias, err)
					e.failed.WithLabelValues(host.Alias, host.Name).Inc()
				} else {
					log.Infoln("worker", w, "finished job", host.Name, "aliased as", host.Alias)
				}
			}(w, host)
		}

		go func() {
			wg.Wait()
			close(results)
		}()

		for tf := range results {
			for _, host := range tf.Hosts {
				e.sent.WithLabelValues(tf.Alias, tf.Target, strconv.Itoa(host.Hop), host.IP.String()).Add(float64(host.Sent))
				e.received.WithLabelValues(tf.Alias, tf.Target, strconv.Itoa(host.Hop), host.IP.String()).Add(float64(host.Received))
				e.dropped.WithLabelValues(tf.Alias, tf.Target, strconv.Itoa(host.Hop), host.IP.String()).Add(float64(host.Dropped))
				e.lost.WithLabelValues(tf.Alias, tf.Target, strconv.Itoa(host.Hop), host.IP.String()).Add(host.LostPercent * float64(host.Sent))
				e.latency.WithLabelValues(tf.Alias, tf.Target, strconv.Itoa(host.Hop), host.IP.String()).Observe(host.Mean)
			}
		}
	}
}

func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	e.sent.Collect(ch)
	e.received.Collect(ch)
	e.dropped.Collect(ch)
	e.lost.Collect(ch)
	e.latency.Collect(ch)
	e.failed.Collect(ch)
	return
}

func trace(host Host, results chan<- *TargetFeedback) error {
	// run MTR and wait for it to complete
	a := mtr.New(1, host.Name, config.Arguments...)
	<-a.Done

	// output result
	if a.Error == nil {
		results <- &TargetFeedback{
			Target: host.Name,
			Alias:  host.Alias,
			Hosts:  a.Hosts,
		}
	}
	return a.Error
}

func worker(id int, host Host, results chan<- *TargetFeedback, wg *sync.WaitGroup) error {
	defer wg.Done()
	return trace(host, results)
}

func main() {
	var (
		configFile    = flag.String("config.file", "mtr.yaml", "MTR exporter configuration file.")
		listenAddress = flag.String("web.listen-address", ":9116", "The address to listen on for HTTP requests.")
		showVersion   = flag.Bool("version", false, "Print version information.")
	)

	flag.Parse()

	if *showVersion {
		fmt.Fprintln(os.Stdout, version.Print("mtr_exporter"))
		os.Exit(0)
	}

	log.Infoln("Starting mtr_exporter", version.Info())
	log.Infoln("Build context", version.BuildContext())

	yamlFile, err := ioutil.ReadFile(*configFile)

	if err != nil {
		log.Fatalf("Error reading config file: %s", err)
	}

	err = yaml.Unmarshal(yamlFile, &config)
	if err != nil {
		log.Fatalf("Error parsing config file: %s", err)
	}

	prometheus.MustRegister(version.NewCollector("mtr_exporter"))
	exporter := NewExporter()
	prometheus.MustRegister(exporter)

	go exporter.collect()

	http.Handle("/metrics", prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
            <title>MTR Exporter</title>

MTR Exporter

Metrics

            `))
	})

	log.Infoln("Listening on", *listenAddress)
	if err := http.ListenAndServe(*listenAddress, nil); err != nil {
		log.Fatalf("Error starting HTTP server: %s", err)
	}
}
<commit_msg>Also count route and destination changes<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"reflect"
	"strconv"
	"sync"

	"gopkg.in/yaml.v2"

	mtr "github.com/Shinzu/go-mtr"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
)

type Exporter struct {
	mutex              sync.Mutex
	sent               *prometheus.CounterVec
	received           *prometheus.CounterVec
	dropped            *prometheus.CounterVec
	lost               *prometheus.CounterVec
	latency            *prometheus.SummaryVec
	routeChanges       *prometheus.CounterVec
	destinationChanges *prometheus.CounterVec
	failed             *prometheus.CounterVec
	lastDest           map[string]net.IP
	lastRoute          map[string][]net.IP
}

type Config struct {
	Arguments []string `yaml:"args"`
	Hosts     []Host   `yaml:"hosts"`
}

type Host struct {
	Name  string `yaml:"name"`
	Alias string `yaml:"alias"`
}

type TargetFeedback struct {
	Target string
	Alias  string
	Hosts  []*mtr.Host
}

var config Config

const (
	Namespace = "mtr"
)

func NewExporter() *Exporter {
	var (
		alias  = "alias"
		server = "server"
		hop_id = "hop_id"
		hop_ip = "hop_ip"
	)

	return &Exporter{
		sent: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "sent",
				Help:      "packets sent",
			},
			[]string{alias, server, hop_id, hop_ip},
		),
		received: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "received",
				Help:      "packets received",
			},
			[]string{alias, server, hop_id, hop_ip},
		),
		dropped: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "dropped",
				Help:      "packets dropped",
			},
			[]string{alias, server, hop_id, hop_ip},
		),
		lost: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "lost",
				Help:      "packets lost",
			},
			[]string{alias, server, hop_id, hop_ip},
		),
		latency: prometheus.NewSummaryVec(
			prometheus.SummaryOpts{
				Namespace: Namespace,
				Name:      "latency",
				Help:      "packet latency in microseconds",
				Objectives: map[float64]float64{
					0.5:  0.05,
					0.9:  0.01,
					0.99: 0.001,
				},
				MaxAge:     prometheus.DefMaxAge,
				AgeBuckets: prometheus.DefAgeBuckets,
				BufCap:     prometheus.DefBufCap,
			},
			[]string{alias, server, hop_id, hop_ip},
		),
		routeChanges: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "route_changes",
				Help:      "route changes",
			},
			[]string{alias, server},
		),
		destinationChanges: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "destination_changes",
				Help:      "Number of times the destination IP has changed",
			},
			[]string{alias, server},
		),
		failed: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: Namespace,
				Name:      "failed",
				Help:      "MTR runs failed",
			},
			[]string{alias, server},
		),
		lastDest:  make(map[string]net.IP, len(config.Hosts)),
		lastRoute: make(map[string][]net.IP, len(config.Hosts)),
	}
}

func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.sent.Describe(ch)
	e.received.Describe(ch)
	e.dropped.Describe(ch)
	e.lost.Describe(ch)
	e.latency.Describe(ch)
	e.routeChanges.Describe(ch)
	e.destinationChanges.Describe(ch)
	e.failed.Describe(ch)
}

func (e *Exporter) collect() error {
	for {
		results := make(chan *TargetFeedback)
		wg := &sync.WaitGroup{}
		wg.Add(len(config.Hosts))

		for w, host := range config.Hosts {
			go func(w int, host Host) {
				log.Infoln("worker", w, "processing job", host.Name, "aliased as", host.Alias)
				err := worker(w, host, results, wg)
				if err != nil {
					log.Errorf("worker %d failed job %v aliased as %v: %v\n", w, host.Name, host.Alias, err)
					e.failed.WithLabelValues(host.Alias, host.Name).Inc()
				} else {
					log.Infoln("worker", w, "finished job", host.Name, "aliased as", host.Alias)
				}
			}(w, host)
		}

		go func() {
			wg.Wait()
			close(results)
		}()

		for tf := range results {
			route := make([]net.IP, len(tf.Hosts))
			destination := tf.Hosts[len(tf.Hosts)-1].IP
			for i, host := range tf.Hosts {
				route[i] = host.IP
				e.sent.WithLabelValues(tf.Alias, tf.Target, strconv.Itoa(host.Hop), host.IP.String()).Add(float64(host.Sent))
				e.received.WithLabelValues(tf.Alias, tf.Target, strconv.Itoa(host.Hop), host.IP.String()).Add(float64(host.Received))
				e.dropped.WithLabelValues(tf.Alias, tf.Target, strconv.Itoa(host.Hop), host.IP.String()).Add(float64(host.Dropped))
				e.lost.WithLabelValues(tf.Alias, tf.Target, strconv.Itoa(host.Hop), host.IP.String()).Add(host.LostPercent * float64(host.Sent))
				e.latency.WithLabelValues(tf.Alias, tf.Target, strconv.Itoa(host.Hop), host.IP.String()).Observe(host.Mean)
			}
			if !reflect.DeepEqual(route, e.lastRoute[tf.Alias]) {
				e.routeChanges.WithLabelValues(tf.Alias, tf.Target).Inc()
				e.lastRoute[tf.Alias] = route
			}
			if !reflect.DeepEqual(destination, e.lastDest[tf.Alias]) {
				e.destinationChanges.WithLabelValues(tf.Alias, tf.Target).Inc()
				e.lastDest[tf.Alias] = destination
			}
		}
	}
}

func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	e.sent.Collect(ch)
	e.received.Collect(ch)
	e.dropped.Collect(ch)
	e.lost.Collect(ch)
	e.latency.Collect(ch)
	e.routeChanges.Collect(ch)
	e.destinationChanges.Collect(ch)
	e.failed.Collect(ch)
	return
}

func trace(host Host, results chan<- *TargetFeedback) error {
	// run MTR and wait for it to complete
	a := mtr.New(1, host.Name, config.Arguments...)
	<-a.Done

	// output result
	if a.Error == nil {
		results <- &TargetFeedback{
			Target: host.Name,
			Alias:  host.Alias,
			Hosts:  a.Hosts,
		}
	}
	return a.Error
}

func worker(id int, host Host, results chan<- *TargetFeedback, wg *sync.WaitGroup) error {
	defer wg.Done()
	return trace(host, results)
}

func main() {
	var (
		configFile    = flag.String("config.file", "mtr.yaml", "MTR exporter configuration file.")
		listenAddress = flag.String("web.listen-address", ":9116", "The address to listen on for HTTP requests.")
		showVersion   = flag.Bool("version", false, "Print version information.")
	)

	flag.Parse()

	if *showVersion {
		fmt.Fprintln(os.Stdout, version.Print("mtr_exporter"))
		os.Exit(0)
	}

	log.Infoln("Starting mtr_exporter", version.Info())
	log.Infoln("Build context", version.BuildContext())

	yamlFile, err := ioutil.ReadFile(*configFile)

	if err != nil {
		log.Fatalf("Error reading config file: %s", err)
	}

	err = yaml.Unmarshal(yamlFile, &config)
	if err != nil {
		log.Fatalf("Error parsing config file: %s", err)
	}

	prometheus.MustRegister(version.NewCollector("mtr_exporter"))
	exporter := NewExporter()
	prometheus.MustRegister(exporter)

	go exporter.collect()

	http.Handle("/metrics", prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
            <title>MTR Exporter</title>

MTR Exporter

Metrics

            `))
	})

	log.Infoln("Listening on", *listenAddress)
	if err := http.ListenAndServe(*listenAddress, nil); err != nil {
		log.Fatalf("Error starting HTTP server: %s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/photoshelf/photoshelf-storage/application"
	"github.com/photoshelf/photoshelf-storage/presentation/router"
	"log"
	"os"
)

func main() {
	conf, err := application.Configure(os.Args[1:]...)
	if err != nil {
		log.Fatal(err)
		os.Exit(-1)
	}
	e, err := router.Load()
	if err != nil {
		log.Fatal(err)
		os.Exit(-1)
	}

	address := fmt.Sprintf(":%d", conf.Server.Port)
	e.Logger.Debug(e.Start(address))
}
<commit_msg>[WIP] add grpc<commit_after>package main

import (
	"log"
	"os"
	"net"
	"google.golang.org/grpc"
	"context"
	"github.com/photoshelf/photoshelf-storage/presentation"
	"github.com/syndtr/goleveldb/leveldb"
)

func main() {
	//conf, err := application.Configure(os.Args[1:]...)
	//if err != nil {
	//	log.Fatal(err)
	//	os.Exit(-1)
	//}
	//e, err := router.Load()
	//if err != nil {
	//	log.Fatal(err)
	//	os.Exit(-1)
	//}
	//
	//address := fmt.Sprintf(":%d", conf.Server.Port)
	//e.Logger.Debug(e.Start(address))

	lis, err := net.Listen("tcp", ":1323")
	if err != nil {
		log.Fatal(err)
		os.Exit(-1)
	}

	db, err := leveldb.OpenFile("leveldb", nil)
	if err != nil {
		log.Fatalf("%v", err)
	}

	s := grpc.NewServer()
	presentation.RegisterPhotoServiceServer(s, &server{db: db})
	go func() {
		s.Serve(lis)
	}()

	conn, err := grpc.Dial(":1323", grpc.WithInsecure())
	if err != nil {
		log.Fatalf("did not connect: %v", err)
	}
	defer conn.Close()
	c := presentation.NewPhotoServiceClient(conn)

	//id := "hoge"
	//_, err = c.Find(context.Background(), &presentation.Id{Value: &id})
	//if err != nil {
	//	log.Fatal(err)
	//}

	image := []byte("Hello World.")
	identifier, err := c.Save(context.Background(), &presentation.Photo{Image: image})
	log.Print(*identifier.Value)

	newid := "newid"
	photo, err := c.Find(context.Background(), &presentation.Id{Value: &newid})
	log.Print(photo)
}

type server struct {
	db *leveldb.DB
}

func (s *server) Save(ctx context.Context, in *presentation.Photo) (*presentation.Id, error) {
	var id *string
	if in.Id == nil {
		newId := "newid"
		id = &newId
	} else {
		id = in.Id.Value
	}

	s.db.Put([]byte(*id), in.Image, nil)
	return &presentation.Id{Value: id}, nil
}

func (s *server) Find(ctx context.Context, in *presentation.Id) (*presentation.Photo, error) {
	image, err := s.db.Get([]byte(*in.Value), nil)
	if err != nil {
		return nil, err
	}
	return &presentation.Photo{Id: in, Image: image}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/julienschmidt/httprouter"
	"net/http"
	"os"
	"strings"
)

func (db *db) handler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	switch r.Method {
	case "GET":
		key := ps.ByName("key")
		value := db.dataMap[key]
		fmt.Fprintln(w, value)
	case "POST":
		m := make(map[string]string)
		m["key"] = ps.ByName("key")
		m["value"] = r.FormValue("value")
		db.writeChan <- m
		fmt.Fprintln(w, r.FormValue("value"))
	}
}

func NewHandler(db *db) func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	return db.handler
}

func main() {
	dbFilename := "db"
	logFilename := "log"
	db := initDB(dbFilename, logFilename)
	handler := NewHandler(db)

	router := httprouter.New()
	router.GET("/get/:key", handler)
	router.POST("/set/:key", handler)

	port := strings.TrimSpace(os.Getenv("PORT"))
	if port == "" {
		port = "3001"
	}

	http.ListenAndServe(":"+port, router)
}
<commit_msg>return value alone (Fprint instead of Fprintln)<commit_after>package main

import (
	"fmt"
	"github.com/julienschmidt/httprouter"
	"net/http"
	"os"
	"strings"
)

func (db *db) handler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	switch r.Method {
	case "GET":
		key := ps.ByName("key")
		value := db.dataMap[key]
		fmt.Fprintln(w, value)
	case "POST":
		m := make(map[string]string)
		m["key"] = ps.ByName("key")
		m["value"] = r.FormValue("value")
		db.writeChan <- m
		fmt.Fprint(w, r.FormValue("value"))
	}
}

func NewHandler(db *db) func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	return db.handler
}

func main() {
	dbFilename := "db"
	logFilename := "log"
	db := initDB(dbFilename, logFilename)
	handler := NewHandler(db)

	router := httprouter.New()
	router.GET("/get/:key", handler)
	router.POST("/set/:key", handler)

	port := strings.TrimSpace(os.Getenv("PORT"))
	if port == "" {
		port = "3001"
	}

	http.ListenAndServe(":"+port, router)
}
<|endoftext|>


<commit_before>package main

// TODO Fix installpath

import (
	"github.com/yannh/r10k-go/git"
	"log"
	"os"
	"path"
	"strconv"
	"sync"
	"time"
)

// ForgeModule, GitModule, GithubTarballModule, ....
type PuppetModule interface {
	IsUpToDate() bool
	Name() string
	Download(string, *Cache) *DownloadError
	Folder() string
	SetModulesFolder(to string)
	ModulesFolder() string
	Hash() string
	Processed()
}

// Can be a PuppetFile or a metadata.json file
type moduleFile interface {
	Filename() string
	Process(modules chan<- PuppetModule) error
	Close()
}

type DownloadError struct {
	error
	retryable bool
}

type DownloadResult struct {
	err     *DownloadError
	skipped bool
}

func downloadModule(m PuppetModule, cache *Cache) DownloadResult {
	if m.IsUpToDate() {
		return DownloadResult{err: nil, skipped: true}
	}

	if err := os.RemoveAll(m.Folder()); err != nil {
		log.Fatalf("Error removing folder: %s", m.Folder())
	}

	if derr := m.Download(m.Folder(), cache); derr != nil {
		return DownloadResult{err: derr, skipped: false}
	}

	return DownloadResult{err: nil, skipped: false}
}

func downloadModules(modules chan PuppetModule, cache *Cache, downloadDeps bool, wg *sync.WaitGroup, errorsCount chan<- int) {
	maxTries := 3
	retryDelay := 5 * time.Second
	errors := 0

	for m := range modules {
		cache.LockModule(m.Hash())

		dres := downloadModule(m, cache)
		for i := 1; dres.err != nil && dres.err.retryable && i < maxTries; i++ {
			log.Printf("failed downloading %s: %v... Retrying\n", m.Name(), dres.err)
			time.Sleep(retryDelay)
			dres = downloadModule(m, cache)
		}

		if dres.err == nil {
			if downloadDeps {
				if mf := NewMetadataFile(path.Join(m.Folder(), "metadata.json"), m.ModulesFolder()); mf != nil {
					wg.Add(1)
					go func() {
						processModuleFile(mf, modules)
						mf.Close()
						wg.Done()
					}()
				}
			}

			if !dres.skipped {
				log.Println("Downloaded " + m.Name() + " to " + m.ModulesFolder())
			}
		} else {
			log.Printf("failed downloading %s: %v. Giving up!\n", m.Name(), dres.err)
			errors++
		}

		m.Processed()
		cache.UnlockModule(m.Hash())
	}

	errorsCount <- errors
}

func processModuleFile(mf moduleFile, modules chan PuppetModule) {
	if err := mf.Process(modules); err != nil {
		if serr, ok := err.(ErrMalformedPuppetfile); ok {
			log.Fatal(serr)
		} else {
			log.Printf("failed parsing %s: %v\n", mf.Filename(), err)
		}
	}
}

func main() {
	var err error
	var numWorkers int
	var cache *Cache

	cliOpts := cli()

	if cliOpts["--workers"] == nil {
		numWorkers = 4
	} else {
		numWorkers, err = strconv.Atoi(cliOpts["--workers"].(string))
		if err != nil {
			log.Fatalf("Parameter --workers should be an integer")
		}
	}

	cacheDir := ".cache"
	var puppetFiles []*PuppetFile

	if cache, err = NewCache(cacheDir); err != nil {
		log.Fatal(err)
	}

	if cliOpts["deploy"] == true {
		r10kFile := "r10k.yml"
		r10kConfig, err := NewR10kConfig(r10kFile)
		if err != nil {
			log.Fatalf("Error parsing r10k configuration file %s: %v", r10kFile, err)
		}

		if r10kConfig.Cachedir != "" {
			cacheDir = r10kConfig.Cachedir
		}

		// Find in which source the environment is
		// TODO: render deterministic
		for _, envName := range cliOpts["<env>"].([]string) {
			environmentSource := ""

			for sourceName, sourceOpts := range r10kConfig.Sources {
				if git.RepoHasBranch(sourceOpts.Remote, envName) {
					environmentSource = sourceName
					break
				}
			}

			sourceCacheFolder := path.Join(cacheDir, environmentSource)

			// Clone if environment doesnt exist, fetch otherwise
			if err := git.RevParse(sourceCacheFolder); err != nil {
				if err := git.Clone(r10kConfig.Sources[environmentSource].Remote, git.Ref{Branch: envName}, sourceCacheFolder); err != nil {
					log.Fatalf("failed downloading environment: %v", err)
				}
			} else {
				git.Fetch(sourceCacheFolder)
			}

			git.WorktreeAdd(sourceCacheFolder, git.Ref{Branch: envName}, path.Join(r10kConfig.Sources[environmentSource].Basedir, envName))
			puppetfile := path.Join(r10kConfig.Sources[environmentSource].Basedir, envName, "Puppetfile")

			pf := NewPuppetFile(puppetfile, path.Join(path.Dir(puppetfile), "modules"))
			if pf == nil {
				log.Fatalf("no such file or directory %s", puppetfile)
			}
			puppetFiles = append(puppetFiles, pf)
		}
	}

	if cliOpts["install"] == true {
		puppetfile := ""
		if cliOpts["--puppetfile"] == nil {
			wd, _ := os.Getwd()
			puppetfile = path.Join(wd, "Puppetfile")
		} else {
			puppetfile = cliOpts["--puppetfile"].(string)
		}

		pf := NewPuppetFile(puppetfile, path.Join(path.Dir(puppetfile), "modules"))
		if pf == nil {
			log.Fatalf("no such file or directory %s", puppetfile)
		}

		puppetFiles = append(puppetFiles, pf)
	}

	if cliOpts["install"] == true || cliOpts["deploy"] == true {
		modules := make(chan PuppetModule)

		var wg sync.WaitGroup
		errorCount := make(chan int)

		for w := 1; w <= numWorkers; w++ {
			go downloadModules(modules, cache, !cliOpts["--no-deps"].(bool), &wg, errorCount)
		}

		for _, pf := range puppetFiles {
			wg.Add(1)
			go func(pf moduleFile, modules chan PuppetModule) {
				processModuleFile(pf, modules)
				pf.Close()
				wg.Done()
			}(pf, modules)
		}

		wg.Wait()
		close(modules)

		nErr := 0
		for w := 1; w <= numWorkers; w++ {
			nErr += <-errorCount
		}

		close(errorCount)

		os.Exit(nErr)
	}
}
<commit_msg>refactoring module file processing<commit_after>package main

// TODO Fix installpath

import (
	"github.com/yannh/r10k-go/git"
	"log"
	"os"
	"path"
	"strconv"
	"sync"
	"time"
)

// ForgeModule, GitModule, GithubTarballModule, ....
type PuppetModule interface {
	IsUpToDate() bool
	Name() string
	Download(string, *Cache) *DownloadError
	Folder() string
	SetModulesFolder(to string)
	ModulesFolder() string
	Hash() string
	Processed()
}

// Can be a PuppetFile or a metadata.json file
type moduleFile interface {
	Filename() string
	Process(modules chan<- PuppetModule) error
}

type DownloadError struct {
	error
	retryable bool
}

type DownloadResult struct {
	err     *DownloadError
	skipped bool
}

func downloadModule(m PuppetModule, cache *Cache) DownloadResult {
	if m.IsUpToDate() {
		return DownloadResult{err: nil, skipped: true}
	}

	if err := os.RemoveAll(m.Folder()); err != nil {
		log.Fatalf("Error removing folder: %s", m.Folder())
	}

	if derr := m.Download(m.Folder(), cache); derr != nil {
		return DownloadResult{err: derr, skipped: false}
	}

	return DownloadResult{err: nil, skipped: false}
}

func downloadModules(modules chan PuppetModule, cache *Cache, downloadDeps bool, wg *sync.WaitGroup, errorsCount chan<- int) {
	maxTries := 3
	retryDelay := 5 * time.Second
	errors := 0

	for m := range modules {
		cache.LockModule(m.Hash())

		dres := downloadModule(m, cache)
		for i := 1; dres.err != nil && dres.err.retryable && i < maxTries; i++ {
			log.Printf("failed downloading %s: %v... Retrying\n", m.Name(), dres.err)
			time.Sleep(retryDelay)
			dres = downloadModule(m, cache)
		}

		if dres.err == nil {
			if downloadDeps {
				if mf := NewMetadataFile(path.Join(m.Folder(), "metadata.json"), m.ModulesFolder()); mf != nil {
					wg.Add(1)
					go func() {
						if err := mf.Process(modules); err != nil {
							log.Printf("failed parsing %s: %v\n", mf.Filename(), err)
						}

						mf.Close()
						wg.Done()
					}()
				}
			}

			if !dres.skipped {
				log.Println("Downloaded " + m.Name() + " to " + m.ModulesFolder())
			}
		} else {
			log.Printf("failed downloading %s: %v. Giving up!\n", m.Name(), dres.err)
			errors++
		}

		m.Processed()
		cache.UnlockModule(m.Hash())
	}

	errorsCount <- errors
}

func main() {
	var err error
	var numWorkers int
	var cache *Cache

	cliOpts := cli()

	if cliOpts["--workers"] == nil {
		numWorkers = 4
	} else {
		numWorkers, err = strconv.Atoi(cliOpts["--workers"].(string))
		if err != nil {
			log.Fatalf("Parameter --workers should be an integer")
		}
	}

	cacheDir := ".cache"
	var puppetFiles []*PuppetFile

	if cache, err = NewCache(cacheDir); err != nil {
		log.Fatal(err)
	}

	if cliOpts["deploy"] == true {
		r10kFile := "r10k.yml"
		r10kConfig, err := NewR10kConfig(r10kFile)
		if err != nil {
			log.Fatalf("Error parsing r10k configuration file %s: %v", r10kFile, err)
		}

		if r10kConfig.Cachedir != "" {
			cacheDir = r10kConfig.Cachedir
		}

		// Find in which source the environment is
		// TODO: render deterministic
		for _, envName := range cliOpts["<env>"].([]string) {
			environmentSource := ""

			for sourceName, sourceOpts := range r10kConfig.Sources {
				if git.RepoHasBranch(sourceOpts.Remote, envName) {
					environmentSource = sourceName
					break
				}
			}

			sourceCacheFolder := path.Join(cacheDir, environmentSource)

			// Clone if environment doesnt exist, fetch otherwise
			if err := git.RevParse(sourceCacheFolder); err != nil {
				if err := git.Clone(r10kConfig.Sources[environmentSource].Remote, git.Ref{Branch: envName}, sourceCacheFolder); err != nil {
					log.Fatalf("failed downloading environment: %v", err)
				}
			} else {
				git.Fetch(sourceCacheFolder)
			}

			git.WorktreeAdd(sourceCacheFolder, git.Ref{Branch: envName}, path.Join(r10kConfig.Sources[environmentSource].Basedir, envName))
			puppetfile := path.Join(r10kConfig.Sources[environmentSource].Basedir, envName, "Puppetfile")

			pf := NewPuppetFile(puppetfile, path.Join(path.Dir(puppetfile), "modules"))
			if pf == nil {
				log.Fatalf("no such file or directory %s", puppetfile)
			}
			puppetFiles = append(puppetFiles, pf)
		}
	}

	if cliOpts["install"] == true {
		puppetfile := ""
		if cliOpts["--puppetfile"] == nil {
			wd, _ := os.Getwd()
			puppetfile = path.Join(wd, "Puppetfile")
		} else {
			puppetfile = cliOpts["--puppetfile"].(string)
		}

		pf := NewPuppetFile(puppetfile, path.Join(path.Dir(puppetfile), "modules"))
		if pf == nil {
			log.Fatalf("no such file or directory %s", puppetfile)
		}

		puppetFiles = append(puppetFiles, pf)
	}

	if cliOpts["install"] == true || cliOpts["deploy"] == true {
		modules := make(chan PuppetModule)

		var wg sync.WaitGroup
		errorCount := make(chan int)

		for w := 1; w <= numWorkers; w++ {
			go downloadModules(modules, cache, !cliOpts["--no-deps"].(bool), &wg, errorCount)
		}

		for _, pf := range puppetFiles {
			wg.Add(1)
			go func(pf *PuppetFile, modules chan PuppetModule) {
				if err := pf.Process(modules); err != nil {
					if serr, ok := err.(ErrMalformedPuppetfile); ok {
						log.Fatal(serr)
					} else {
						log.Printf("failed parsing %s: %v\n", pf.Filename(), err)
					}
				}

				pf.Close()
				wg.Done()
			}(pf, modules)
		}

		wg.Wait()
		close(modules)

		nErr := 0
		for w := 1; w <= numWorkers; w++ {
			nErr += <-errorCount
		}

		close(errorCount)

		os.Exit(nErr)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"sync"

	"github.com/jroimartin/orujo"
	"github.com/jroimartin/orujo-handlers/basic"
	olog "github.com/jroimartin/orujo-handlers/log"
)

var (
	addr       = flag.String("addr", ":8001", "HTTP service address")
	quotesFile = flag.String("quotesfile", "quotes.txt", "file where quotes will be stored")
	user       = flag.String("user", "user", "username used for basic authentication")
	pass       = flag.String("pass", "s3cr3t", "password used for basic authentication")
	re         = regexp.MustCompile(`[\r\n]+`)
)

var mutex sync.RWMutex

func main() {
	flag.Parse()

	s := orujo.NewServer(*addr)

	logger := log.New(os.Stdout, "[quotesrv] ", log.LstdFlags)
	logHandler := olog.NewLogHandler(logger, logLine)
	basicHandler := basic.NewBasicHandler("Quotes System", *user, *pass)

	s.RouteDefault(http.NotFoundHandler(), orujo.M(logHandler))

	s.Route(`^/$`,
		basicHandler,
		http.HandlerFunc(listQuotes),
		orujo.M(logHandler),
	).Methods("GET")

	s.Route(`^/$`,
		basicHandler,
		http.HandlerFunc(addQuote),
		orujo.M(logHandler),
	).Methods("POST")

	logger.Fatalln(s.ListenAndServe())
}

func listQuotes(w http.ResponseWriter, r *http.Request) {
	mutex.RLock()
	defer mutex.RUnlock()

	f, err := os.Open(*quotesFile)
	if err != nil {
		errorResponse(w, err)
		return
	}
	defer f.Close()

	if _, err := io.Copy(w, f); err != nil {
		errorResponse(w, err)
		return
	}
}

func addQuote(w http.ResponseWriter, r *http.Request) {
	mutex.Lock()
	defer mutex.Unlock()

	f, err := os.OpenFile(*quotesFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
	if err != nil {
		errorResponse(w, err)
		return
	}
	defer f.Close()

	bQuote, err := ioutil.ReadAll(r.Body)
	if err != nil {
		errorResponse(w, err)
		return
	}
	quote := re.ReplaceAllString(string(bQuote), " ") + "\n"

	if _, err := f.WriteString(quote); err != nil {
		errorResponse(w, err)
		return
	}
}

func errorResponse(w http.ResponseWriter, err error) {
	orujo.RegisterError(w, err)
	w.WriteHeader(http.StatusInternalServerError)
}

const logLine = `{{.Req.RemoteAddr}} - {{.Req.Method}} {{.Req.RequestURI}}
{{range  $err := .Errors}}  Err: {{$err}}
{{end}}`
<commit_msg>Minor change<commit_after>package main

import (
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"sync"

	"github.com/jroimartin/orujo"
	"github.com/jroimartin/orujo-handlers/basic"
	olog "github.com/jroimartin/orujo-handlers/log"
)

var (
	addr       = flag.String("addr", ":8001", "HTTP service address")
	quotesFile = flag.String("quotesfile", "quotes.txt", "file where quotes will be stored")
	user       = flag.String("user", "user", "username used for basic authentication")
	pass       = flag.String("pass", "s3cr3t", "password used for basic authentication")
	re         = regexp.MustCompile(`[\r\n]+`)
)

var mutex sync.RWMutex

func main() {
	flag.Parse()

	s := orujo.NewServer(*addr)

	logger := log.New(os.Stdout, "[quotesrv] ", log.LstdFlags)
	logHandler := olog.NewLogHandler(logger, logLine)
	basicHandler := basic.NewBasicHandler("Quotes System", *user, *pass)

	s.RouteDefault(http.NotFoundHandler(), orujo.M(logHandler))

	s.Route(`^/$`,
		basicHandler,
		http.HandlerFunc(listQuotes),
		orujo.M(logHandler),
	).Methods("GET")

	s.Route(`^/$`,
		basicHandler,
		http.HandlerFunc(addQuote),
		orujo.M(logHandler),
	).Methods("POST")

	logger.Fatalln(s.ListenAndServe())
}

func listQuotes(w http.ResponseWriter, r *http.Request) {
	mutex.RLock()
	defer mutex.RUnlock()

	f, err := os.Open(*quotesFile)
	if err != nil {
		errorResponse(w, err)
		return
	}
	defer f.Close()

	if _, err = io.Copy(w, f); err != nil {
		errorResponse(w, err)
		return
	}
}

func addQuote(w http.ResponseWriter, r *http.Request) {
	mutex.Lock()
	defer mutex.Unlock()

	f, err := os.OpenFile(*quotesFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
	if err != nil {
		errorResponse(w, err)
		return
	}
	defer f.Close()

	bQuote, err := ioutil.ReadAll(r.Body)
	if err != nil {
		errorResponse(w, err)
		return
	}
	quote := re.ReplaceAllString(string(bQuote), " ") + "\n"

	if _, err = f.WriteString(quote); err != nil {
		errorResponse(w, err)
		return
	}
}

func errorResponse(w http.ResponseWriter, err error) {
	orujo.RegisterError(w, err)
	w.WriteHeader(http.StatusInternalServerError)
}

const logLine = `{{.Req.RemoteAddr}} - {{.Req.Method}} {{.Req.RequestURI}}
{{range  $err := .Errors}}  Err: {{$err}}
{{end}}`
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io/ioutil"
	"os"
	"runtime"

	"github.com/mithrandie/csvq/lib/action"
	"github.com/mithrandie/csvq/lib/cmd"
	"github.com/mithrandie/csvq/lib/query"

	"github.com/urfave/cli"
)

var version = "v0.8.7"

func main() {
	cli.AppHelpTemplate = appHHelpTemplate
	cli.CommandHelpTemplate = commandHelpTemplate

	app := cli.NewApp()

	app.Name = "csvq"
	app.Usage = "SQL like query language for csv"
	app.ArgsUsage = "[\"query\"|\"statements\"|argument]"
	app.Version = version

	defaultCPU := runtime.NumCPU() / 2
	if defaultCPU < 1 {
		defaultCPU = 1
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "delimiter, d",
			Usage: "field delimiter. Default is \",\" for csv files, \"\\t\" for tsv files.",
		},
		cli.StringFlag{
			Name:  "encoding, e",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "line-break, l",
			Value: "LF",
			Usage: "line break. one of: CRLF|LF|CR",
		},
		cli.StringFlag{
			Name:  "timezone, z",
			Value: "Local",
			Usage: "default timezone. \"Local\", \"UTC\" or a timezone name(e.g. \"America/Los_Angeles\")",
		},
		cli.StringFlag{
			Name:  "repository, r",
			Usage: "directory path where files are located",
		},
		cli.StringFlag{
			Name:  "source, s",
			Usage: "load query from `FILE`",
		},
		cli.StringFlag{
			Name:  "datetime-format, t",
			Usage: "set datetime format to parse strings",
		},
		cli.Float64Flag{
			Name:  "wait-timeout, w",
			Value: 10,
			Usage: "limit of the waiting time in seconds to wait for locked files to be released",
		},
		cli.BoolFlag{
			Name:  "no-header, n",
			Usage: "import the first line as a record",
		},
		cli.BoolFlag{
			Name:  "without-null, a",
			Usage: "parse empty fields as empty strings",
		},
		cli.StringFlag{
			Name:  "write-encoding, E",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "out, o",
			Usage: "write output to `FILE`",
		},
		cli.StringFlag{
			Name:  "format, f",
			Usage: "output format. one of: CSV|TSV|JSON|TEXT",
		},
		cli.StringFlag{
			Name:  "write-delimiter, D",
			Usage: "field delimiter for CSV",
		},
		cli.BoolFlag{
			Name:  "without-header, N",
			Usage: "when the file format is specified as CSV or TSV, write without the header line",
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "suppress operation log output",
		},
		cli.IntFlag{
			Name:  "cpu, p",
			Value: defaultCPU,
			Usage: "hint for the number of cpu cores to be used. 1 - number of cpu cores",
		},
		cli.BoolFlag{
			Name:  "stats, x",
			Usage: "show execution time and memory statistics",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:      "fields",
			Usage:     "Show fields in a file",
			ArgsUsage: "CSV_FILE_PATH",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "fields")
					return cli.NewExitError("table is not specified", 1)
				}

				table := c.Args().First()

				err := action.ShowFields(table)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
		{
			Name:      "calc",
			Usage:     "Calculate a value from stdin",
			ArgsUsage: "\"expression\"",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "calc")
					return cli.NewExitError("expression is empty", 1)
				}

				expr := c.Args().First()
				err := action.Calc(expr)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
	}

	app.Before = func(c *cli.Context) error {
		err := setFlags(c)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		return nil
	}

	app.Action = func(c *cli.Context) error {
		queryString, err := readQuery(c)
		if err != nil {
			cli.ShowAppHelp(c)
			return cli.NewExitError(err.Error(), 1)
		}

		if len(queryString) < 1 {
			err = action.LaunchInteractiveShell()
		} else {
			err = action.Run(queryString, cmd.GetFlags().Source)
		}

		if err != nil {
			code := 1
			if apperr, ok := err.(query.AppError); ok {
				code = apperr.GetCode()
			} else if ex, ok := err.(*query.Exit); ok {
				code = ex.GetCode()
			}
			return cli.NewExitError(err.Error(), code)
		}

		return nil
	}

	app.Run(os.Args)
}

func readQuery(c *cli.Context) (string, error) {
	var queryString string

	flags := cmd.GetFlags()
	if 0 < len(flags.Source) {
		fp, err := os.Open(flags.Source)
		if err != nil {
			return queryString, err
		}
		defer fp.Close()

		buf, err := ioutil.ReadAll(fp)
		if err != nil {
			return queryString, err
		}
		queryString = string(buf)

	} else {
		if 1 < c.NArg() {
			return queryString, errors.New("multiple queries or statements were passed")
		}
		queryString = c.Args().First()
	}

	return queryString, nil
}

func setFlags(c *cli.Context) error {
	if err := cmd.SetDelimiter(c.GlobalString("delimiter")); err != nil {
		return err
	}
	if err := cmd.SetEncoding(c.GlobalString("encoding")); err != nil {
		return err
	}
	if err := cmd.SetLineBreak(c.String("line-break")); err != nil {
		return err
	}
	if err := cmd.SetLocation(c.String("timezone")); err != nil {
		return err
	}
	if err := cmd.SetRepository(c.GlobalString("repository")); err != nil {
		return err
	}
	if err := cmd.SetSource(c.GlobalString("source")); err != nil {
		return err
	}
	cmd.SetDatetimeFormat(c.GlobalString("datetime-format"))
	cmd.SetWaitTimeout(c.GlobalFloat64("wait-timeout"))
	cmd.SetNoHeader(c.GlobalBool("no-header"))
	cmd.SetWithoutNull(c.GlobalBool("without-null"))

	if err := cmd.SetWriteEncoding(c.GlobalString("write-encoding")); err != nil {
		return err
	}
	if err := cmd.SetOut(c.GlobalString("out")); err != nil {
		return err
	}
	if err := cmd.SetFormat(c.GlobalString("format")); err != nil {
		return err
	}
	if err := cmd.SetWriteDelimiter(c.GlobalString("write-delimiter")); err != nil {
		return err
	}
	cmd.SetWithoutHeader(c.GlobalBool("without-header"))

	cmd.SetQuiet(c.GlobalBool("quiet"))
	cmd.SetCPU(c.GlobalInt("cpu"))
	cmd.SetStats(c.GlobalBool("stats"))

	return nil
}
<commit_msg>Update version for Release v1.0.0<commit_after>package main

import (
	"errors"
	"io/ioutil"
	"os"
	"runtime"

	"github.com/mithrandie/csvq/lib/action"
	"github.com/mithrandie/csvq/lib/cmd"
	"github.com/mithrandie/csvq/lib/query"

	"github.com/urfave/cli"
)

var version = "v1.0.0"

func main() {
	cli.AppHelpTemplate = appHHelpTemplate
	cli.CommandHelpTemplate = commandHelpTemplate

	app := cli.NewApp()

	app.Name = "csvq"
	app.Usage = "SQL like query language for csv"
	app.ArgsUsage = "[\"query\"|\"statements\"|argument]"
	app.Version = version

	defaultCPU := runtime.NumCPU() / 2
	if defaultCPU < 1 {
		defaultCPU = 1
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "delimiter, d",
			Usage: "field delimiter. Default is \",\" for csv files, \"\\t\" for tsv files.",
		},
		cli.StringFlag{
			Name:  "encoding, e",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "line-break, l",
			Value: "LF",
			Usage: "line break. one of: CRLF|LF|CR",
		},
		cli.StringFlag{
			Name:  "timezone, z",
			Value: "Local",
			Usage: "default timezone. \"Local\", \"UTC\" or a timezone name(e.g. \"America/Los_Angeles\")",
		},
		cli.StringFlag{
			Name:  "repository, r",
			Usage: "directory path where files are located",
		},
		cli.StringFlag{
			Name:  "source, s",
			Usage: "load query from `FILE`",
		},
		cli.StringFlag{
			Name:  "datetime-format, t",
			Usage: "set datetime format to parse strings",
		},
		cli.Float64Flag{
			Name:  "wait-timeout, w",
			Value: 10,
			Usage: "limit of the waiting time in seconds to wait for locked files to be released",
		},
		cli.BoolFlag{
			Name:  "no-header, n",
			Usage: "import the first line as a record",
		},
		cli.BoolFlag{
			Name:  "without-null, a",
			Usage: "parse empty fields as empty strings",
		},
		cli.StringFlag{
			Name:  "write-encoding, E",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "out, o",
			Usage: "write output to `FILE`",
		},
		cli.StringFlag{
			Name:  "format, f",
			Usage: "output format. one of: CSV|TSV|JSON|TEXT",
		},
		cli.StringFlag{
			Name:  "write-delimiter, D",
			Usage: "field delimiter for CSV",
		},
		cli.BoolFlag{
			Name:  "without-header, N",
			Usage: "when the file format is specified as CSV or TSV, write without the header line",
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "suppress operation log output",
		},
		cli.IntFlag{
			Name:  "cpu, p",
			Value: defaultCPU,
			Usage: "hint for the number of cpu cores to be used. 1 - number of cpu cores",
		},
		cli.BoolFlag{
			Name:  "stats, x",
			Usage: "show execution time and memory statistics",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:      "fields",
			Usage:     "Show fields in a file",
			ArgsUsage: "CSV_FILE_PATH",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "fields")
					return cli.NewExitError("table is not specified", 1)
				}

				table := c.Args().First()

				err := action.ShowFields(table)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
		{
			Name:      "calc",
			Usage:     "Calculate a value from stdin",
			ArgsUsage: "\"expression\"",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "calc")
					return cli.NewExitError("expression is empty", 1)
				}

				expr := c.Args().First()
				err := action.Calc(expr)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
	}

	app.Before = func(c *cli.Context) error {
		err := setFlags(c)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		return nil
	}

	app.Action = func(c *cli.Context) error {
		queryString, err := readQuery(c)
		if err != nil {
			cli.ShowAppHelp(c)
			return cli.NewExitError(err.Error(), 1)
		}

		if len(queryString) < 1 {
			err = action.LaunchInteractiveShell()
		} else {
			err = action.Run(queryString, cmd.GetFlags().Source)
		}

		if err != nil {
			code := 1
			if apperr, ok := err.(query.AppError); ok {
				code = apperr.GetCode()
			} else if ex, ok := err.(*query.Exit); ok {
				code = ex.GetCode()
			}
			return cli.NewExitError(err.Error(), code)
		}

		return nil
	}

	app.Run(os.Args)
}

func readQuery(c *cli.Context) (string, error) {
	var queryString string

	flags := cmd.GetFlags()
	if 0 < len(flags.Source) {
		fp, err := os.Open(flags.Source)
		if err != nil {
			return queryString, err
		}
		defer fp.Close()

		buf, err := ioutil.ReadAll(fp)
		if err != nil {
			return queryString, err
		}
		queryString = string(buf)

	} else {
		if 1 < c.NArg() {
			return queryString, errors.New("multiple queries or statements were passed")
		}
		queryString = c.Args().First()
	}

	return queryString, nil
}

func setFlags(c *cli.Context) error {
	if err := cmd.SetDelimiter(c.GlobalString("delimiter")); err != nil {
		return err
	}
	if err := cmd.SetEncoding(c.GlobalString("encoding")); err != nil {
		return err
	}
	if err := cmd.SetLineBreak(c.String("line-break")); err != nil {
		return err
	}
	if err := cmd.SetLocation(c.String("timezone")); err != nil {
		return err
	}
	if err := cmd.SetRepository(c.GlobalString("repository")); err != nil {
		return err
	}
	if err := cmd.SetSource(c.GlobalString("source")); err != nil {
		return err
	}
	cmd.SetDatetimeFormat(c.GlobalString("datetime-format"))
	cmd.SetWaitTimeout(c.GlobalFloat64("wait-timeout"))
	cmd.SetNoHeader(c.GlobalBool("no-header"))
	cmd.SetWithoutNull(c.GlobalBool("without-null"))

	if err := cmd.SetWriteEncoding(c.GlobalString("write-encoding")); err != nil {
		return err
	}
	if err := cmd.SetOut(c.GlobalString("out")); err != nil {
		return err
	}
	if err := cmd.SetFormat(c.GlobalString("format")); err != nil {
		return err
	}
	if err := cmd.SetWriteDelimiter(c.GlobalString("write-delimiter")); err != nil {
		return err
	}
	cmd.SetWithoutHeader(c.GlobalBool("without-header"))

	cmd.SetQuiet(c.GlobalBool("quiet"))
	cmd.SetCPU(c.GlobalInt("cpu"))
	cmd.SetStats(c.GlobalBool("stats"))

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

type testRunner struct {
	wg sync.WaitGroup
	w  io.Writer
	r  io.Reader

	sync.Mutex
	errors          []error
	stepsInLine     int
	scenarios       int
	scenariosPassed int
	steps           int
	stepsPassed     int
}

func NewTestRunner() *testRunner {
	var e []error
	reader, writer := io.Pipe()
	var wg sync.WaitGroup
	return &testRunner{
		wg:              wg,
		w:               writer,
		r:               reader,
		stepsInLine:     0,
		scenarios:       0,
		scenariosPassed: 0,
		steps:           0,
		stepsPassed:     0,
		errors:          e,
	}
}

func main() {
	t := NewTestRunner()
	start := time.Now()

	features := features()
	pos := 0
	for n := len(features); n > 0; {
		size := 2
		if n < size {
			size = n
		}
		n -= size
		chunk := features[pos : size+pos]
		pos += size
		for _, feature := range chunk {
			t.wg.Add(1)
			go t.executeTest(feature)
		}
		t.wg.Wait()
	}
	//for _, test := range features {
	//t.wg.Add(1)
	//go t.executeTest(test)
	//}
	//t.wg.Wait()
	fmt.Println()
	for _, e := range t.errors {
		fmt.Println(e)
	}
	fmt.Printf("%d scenarios (\033[32m%d passed\033[0m)\n", t.scenarios, t.scenariosPassed)
	fmt.Printf("%d steps (\033[32m%d passed\033[0m)\n", t.steps, t.stepsPassed)
	fmt.Printf("Tests ran in: %s\n", time.Since(start))
}

func (t *testRunner) executeTest(test string) {
	behat := exec.Command("./bin/behat", "-f", "progress", test)
	stdout, err := behat.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	go t.proccessOutput(stdout)
	err = behat.Start()
	if err != nil {
		log.Fatal(err)
	}
	err = behat.Wait()
	if err != nil {
		t.errors = append(t.errors, fmt.Errorf("TODO: handle std err output from behat: %s", err))
	}
	t.wg.Done()
}

func (t *testRunner) proccessOutput(out io.Reader) {
	reader := bufio.NewReader(out)
	for {
		c, err := reader.ReadByte()
		switch {
		case c == '\n':
			// if we encounted two new lines in a row - steps have finished
			// and we try to parse information about runned scenarios and steps
			nextByte, err := reader.Peek(1)
			if err != nil {
				break
			}
			if nextByte[0] == '\n' {
				_, err = reader.ReadByte()
				for {
					line, err := reader.ReadBytes('\n')
					if err != nil {
						break
					}
					// TODO:
					// parse scenarios and steps info and store somewhere.
					// parse failed, skipped and undefined scenarios/steps
					scenariosRe := regexp.MustCompile("([0-9]+) scenario")
					match := scenariosRe.FindString(string(line))
					if match != "" {
						scenarios := strings.Split(match, " ")
						t.Lock()
						i, _ := strconv.Atoi(scenarios[0])
						t.scenarios += i
						t.Unlock()

						//scenarios passed
						scenariosPassedRe := regexp.MustCompile("([0-9]+) passed")
						match = scenariosPassedRe.FindString(string(line))
						if match != "" {
							passed := strings.Split(match, " ")
							t.Lock()
							i, _ := strconv.Atoi(passed[0])
							t.scenariosPassed += i
							t.Unlock()
						}
					}

					stepsRe := regexp.MustCompile("([0-9]+) step")
					match = stepsRe.FindString(string(line))
					if match != "" {
						steps := strings.Split(match, " ")
						t.Lock()
						i, _ := strconv.Atoi(steps[0])
						t.steps += i
						t.Unlock()

						//steps passed
						stepsPassedRe := regexp.MustCompile("([0-9]+) passed")
						match = stepsPassedRe.FindString(string(line))
						if match != "" {
							passed := strings.Split(match, " ")
							t.Lock()
							i, _ := strconv.Atoi(passed[0])
							t.stepsPassed += i
							t.Unlock()
						}
					}
				}
			}
			break
		case c == '.':
			if t.stepsInLine > 0 && t.stepsInLine%70 == 0 {
				fmt.Printf(" %d\n", t.stepsInLine)
			}
			fmt.Printf("\033[32m%s\033[0m", string(c))
			t.Lock()
			t.stepsInLine += 1
			t.Unlock()
			break
		case c == 'F':
			if t.stepsInLine > 0 && t.stepsInLine%70 == 0 {
				fmt.Printf(" %d\n", t.stepsInLine)
			}
			fmt.Printf("\033[31m%s\033[0m", string(c))
			t.Lock()
			t.stepsInLine += 1
			t.Unlock()
			break
		case c == '-':
			if t.stepsInLine > 0 && t.stepsInLine%70 == 0 {
				fmt.Printf(" %d\n", t.stepsInLine)
			}
			fmt.Printf("\033[36m%s\033[0m", string(c))
			t.Lock()
			t.stepsInLine += 1
			t.Unlock()
			break
		case c == 'U':
			if t.stepsInLine > 0 && t.stepsInLine%70 == 0 {
				fmt.Printf(" %d\n", t.stepsInLine)
			}
			fmt.Printf("\033[33m%s\033[0m", string(c))
			t.Lock()
			t.stepsInLine += 1
			t.Unlock()
			break
		}
		if err == io.EOF {
			break
		}
		if err != nil {
			break
			log.Printf("Unknown error while proccessing output: %s", err)
		}
	}
}

func features() []string {
	var features []string
	err := filepath.Walk("features", func(path string, file os.FileInfo, err error) error {
		if err == nil && !file.IsDir() {
			features = append(features, path)
		}
		return err
	})
	if err != nil {
		panic("failed to walk directory: " + err.Error())
	}
	return features
}
<commit_msg>Change concurent test execution handling<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

type testRunner struct {
	wg        sync.WaitGroup
	w         io.Writer
	r         io.Reader
	semaphore chan int

	sync.Mutex
	errors          []error
	stepsInLine     int
	scenarios       int
	scenariosPassed int
	steps           int
	stepsPassed     int
}

func NewTestRunner() *testRunner {
	reader, writer := io.Pipe()
	var wg sync.WaitGroup
	var e []error
	var sem = make(chan int, 2)
	return &testRunner{
		wg:              wg,
		w:               writer,
		r:               reader,
		stepsInLine:     0,
		scenarios:       0,
		scenariosPassed: 0,
		steps:           0,
		stepsPassed:     0,
		errors:          e,
		semaphore:       sem,
	}
}

func (t *testRunner) run() {
	features := features()
	for _, feature := range features {
		t.wg.Add(1)
		go t.executeTest(feature)
	}
	t.wg.Wait()
}

func main() {
	t := NewTestRunner()
	start := time.Now()

	t.run()

	fmt.Println()
	for _, e := range t.errors {
		fmt.Println(e)
	}
	fmt.Printf("%d scenarios (\033[32m%d passed\033[0m)\n", t.scenarios, t.scenariosPassed)
	fmt.Printf("%d steps (\033[32m%d passed\033[0m)\n", t.steps, t.stepsPassed)
	fmt.Printf("Tests ran in: %s\n", time.Since(start))
}

func (t *testRunner) executeTest(test string) {
	t.semaphore <- 1
	behat := exec.Command("./bin/behat", "-f", "progress", test)
	stdout, err := behat.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	go t.proccessOutput(stdout)
	err = behat.Start()
	if err != nil {
		log.Fatal(err)
	}
	err = behat.Wait()
	if err != nil {
		t.errors = append(t.errors, fmt.Errorf("TODO: handle std err output from behat: %s", err))
	}
	<-t.semaphore
	t.wg.Done()
}

func (t *testRunner) proccessOutput(out io.Reader) {
	reader := bufio.NewReader(out)
	for {
		c, err := reader.ReadByte()
		switch {
		case c == '\n':
			// if we encounted two new lines in a row - steps have finished
			// and we try to parse information about runned scenarios and steps
			nextByte, err := reader.Peek(1)
			if err != nil {
				break
			}
			if nextByte[0] == '\n' {
				_, err = reader.ReadByte()
				for {
					line, err := reader.ReadBytes('\n')
					if err != nil {
						break
					}
					// TODO:
					// parse scenarios and steps info and store somewhere.
					// parse failed, skipped and undefined scenarios/steps
					scenariosRe := regexp.MustCompile("([0-9]+) scenario")
					match := scenariosRe.FindString(string(line))
					if match != "" {
						scenarios := strings.Split(match, " ")
						t.Lock()
						i, _ := strconv.Atoi(scenarios[0])
						t.scenarios += i
						t.Unlock()

						//scenarios passed
						scenariosPassedRe := regexp.MustCompile("([0-9]+) passed")
						match = scenariosPassedRe.FindString(string(line))
						if match != "" {
							passed := strings.Split(match, " ")
							t.Lock()
							i, _ := strconv.Atoi(passed[0])
							t.scenariosPassed += i
							t.Unlock()
						}
					}

					stepsRe := regexp.MustCompile("([0-9]+) step")
					match = stepsRe.FindString(string(line))
					if match != "" {
						steps := strings.Split(match, " ")
						t.Lock()
						i, _ := strconv.Atoi(steps[0])
						t.steps += i
						t.Unlock()

						//steps passed
						stepsPassedRe := regexp.MustCompile("([0-9]+) passed")
						match = stepsPassedRe.FindString(string(line))
						if match != "" {
							passed := strings.Split(match, " ")
							t.Lock()
							i, _ := strconv.Atoi(passed[0])
							t.stepsPassed += i
							t.Unlock()
						}
					}
				}
			}
			break
		case c == '.':
			if t.stepsInLine > 0 && t.stepsInLine%70 == 0 {
				fmt.Printf(" %d\n", t.stepsInLine)
			}
			fmt.Printf("\033[32m%s\033[0m", string(c))
			t.Lock()
			t.stepsInLine += 1
			t.Unlock()
			break
		case c == 'F':
			if t.stepsInLine > 0 && t.stepsInLine%70 == 0 {
				fmt.Printf(" %d\n", t.stepsInLine)
			}
			fmt.Printf("\033[31m%s\033[0m", string(c))
			t.Lock()
			t.stepsInLine += 1
			t.Unlock()
			break
		case c == '-':
			if t.stepsInLine > 0 && t.stepsInLine%70 == 0 {
				fmt.Printf(" %d\n", t.stepsInLine)
			}
			fmt.Printf("\033[36m%s\033[0m", string(c))
			t.Lock()
			t.stepsInLine += 1
			t.Unlock()
			break
		case c == 'U':
			if t.stepsInLine > 0 && t.stepsInLine%70 == 0 {
				fmt.Printf(" %d\n", t.stepsInLine)
			}
			fmt.Printf("\033[33m%s\033[0m", string(c))
			t.Lock()
			t.stepsInLine += 1
			t.Unlock()
			break
		}
		if err == io.EOF {
			break
		}
		if err != nil {
			break
			log.Printf("Unknown error while proccessing output: %s", err)
		}
	}
}

func features() []string {
	var features []string
	err := filepath.Walk("features", func(path string, file os.FileInfo, err error) error {
		if err == nil && !file.IsDir() {
			features = append(features, path)
		}
		return err
	})
	if err != nil {
		panic("failed to walk directory: " + err.Error())
	}
	return features
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"time"
	"flag"
	"syscall"
	"os/signal"
)

const (
	HAProxyPath = "/opt/vamp/"
	Timeout = 5 * time.Second
)

var (
	logstashHost = flag.String("logstashHost", "127.0.0.1", "Address of the Logstash instance")
	logstashPort = flag.Int("logstashPort", 10001, "The UDP input port of the Logstash instance")

	storeType = flag.String("storeType", "", "zookeeper, consul or etcd.")
	storeConnection = flag.String("storeConnection", "", "Key-value store connection string.")
	configurationPath = flag.String("configurationPath", "/vamp/gateways/haproxy/1.6", "HAProxy configuration path.")

	logo = flag.Bool("logo", true, "Show logo.")

	debug = flag.Bool("debug", false, "Switches on extra log statements.")

	logger = CreateLogger()
)

func Logo(version string) string {
	return `
██╗   ██╗ █████╗ ███╗   ███╗██████╗
██║   ██║██╔══██╗████╗ ████║██╔══██╗
██║   ██║███████║██╔████╔██║██████╔╝
╚██╗ ██╔╝██╔══██║██║╚██╔╝██║██╔═══╝
 ╚████╔╝ ██║  ██║██║ ╚═╝ ██║██║
  ╚═══╝  ╚═╝  ╚═╝╚═╝     ╚═╝╚═╝
                       gateway agent
                       version ` + version + `
                       by magnetic.io
                                      `
}

type Watcher interface {
	Watch(onChange func([]byte) error)
}

func main() {

	flag.Parse()

	if *logo {
		logger.Notice(Logo("0.8.2"))
	}

	if len(*storeType) == 0 {
		logger.Panic("Key-value store type not speciffed.")
		return
	}

	if len(*storeConnection) == 0 {
		logger.Panic("Key-value store servers not speciffed.")
		return
	}

	logger.Notice("Starting Vamp Gateway Agent")

	haProxy := HAProxy{
		Binary:     "haproxy",
		ConfigFile: HAProxyPath + "haproxy.cfg",
		PidFile:    HAProxyPath + "haproxy.pid",
		LogSocket:  HAProxyPath + "haproxy.log.sock",
	}

	// Waiter keeps the program from exiting instantly.
	waiter := make(chan bool)

	cleanup := func() { os.Remove(haProxy.LogSocket) }

	// Catch a CTR+C exits so the cleanup routine is called.
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	go func() {
		<-c
		cleanup()
		os.Exit(1)
	}()

	defer cleanup()

	haProxy.Init()
	haProxy.Run()

	keyValueWatcher := keyValueWatcher()

	if keyValueWatcher == nil {
		return
	}

	go keyValueWatcher.Watch(haProxy.Reload)

	waiter <- true
}

func keyValueWatcher() Watcher {
	if *storeType == "etcd" {
		return &Etcd{
			ConnectionString: *storeConnection,
			Path: *configurationPath,
		}
	} else if *storeType == "consul" {
		return &Consul{
			ConnectionString: *storeConnection,
			Path: *configurationPath,
		}
	} else if *storeType == "zookeeper" {
		return &ZooKeeper{
			ConnectionString: *storeConnection,
			Path: *configurationPath,
		}
	} else {
		logger.Panic("Key-value store type not supported: ", *storeType)
		return nil
	}
}
<commit_msg>don't exit if 'help' arg is specified<commit_after>package main

import (
	"os"
	"time"
	"flag"
	"syscall"
	"os/signal"
)

const (
	HAProxyPath = "/opt/vamp/"
	Timeout = 5 * time.Second
)

var (
	logstashHost = flag.String("logstashHost", "127.0.0.1", "Address of the Logstash instance")
	logstashPort = flag.Int("logstashPort", 10001, "The UDP input port of the Logstash instance")

	storeType = flag.String("storeType", "", "zookeeper, consul or etcd.")
	storeConnection = flag.String("storeConnection", "", "Key-value store connection string.")
	configurationPath = flag.String("configurationPath", "/vamp/gateways/haproxy/1.6", "HAProxy configuration path.")

	logo = flag.Bool("logo", true, "Show logo.")
	debug = flag.Bool("debug", false, "Switches on extra log statements.")

	logger = CreateLogger()
)

func Logo(version string) string {
	return `
██╗   ██╗ █████╗ ███╗   ███╗██████╗
██║   ██║██╔══██╗████╗ ████║██╔══██╗
██║   ██║███████║██╔████╔██║██████╔╝
╚██╗ ██╔╝██╔══██║██║╚██╔╝██║██╔═══╝
 ╚████╔╝ ██║  ██║██║ ╚═╝ ██║██║
  ╚═══╝  ╚═╝  ╚═╝╚═╝     ╚═╝╚═╝
                       gateway agent
                       version ` + version + `
                       by magnetic.io
                                      `
}

type Watcher interface {
	Watch(onChange func([]byte) error)
}

func main() {

	flag.Bool("help", false, "Print usage.")

	flag.Parse()

	if *logo {
		logger.Notice(Logo("0.8.2"))
	}

	if len(*storeType) == 0 {
		logger.Panic("Key-value store type not speciffed.")
		return
	}

	if len(*storeConnection) == 0 {
		logger.Panic("Key-value store servers not speciffed.")
		return
	}

	logger.Notice("Starting Vamp Gateway Agent")

	haProxy := HAProxy{
		Binary:     "haproxy",
		ConfigFile: HAProxyPath + "haproxy.cfg",
		PidFile:    HAProxyPath + "haproxy.pid",
		LogSocket:  HAProxyPath + "haproxy.log.sock",
	}

	// Waiter keeps the program from exiting instantly.
	waiter := make(chan bool)

	cleanup := func() { os.Remove(haProxy.LogSocket) }

	// Catch a CTR+C exits so the cleanup routine is called.
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	go func() {
		<-c
		cleanup()
		os.Exit(1)
	}()

	defer cleanup()

	haProxy.Init()
	haProxy.Run()

	keyValueWatcher := keyValueWatcher()

	if keyValueWatcher == nil {
		return
	}

	go keyValueWatcher.Watch(haProxy.Reload)

	waiter <- true
}

func keyValueWatcher() Watcher {
	if *storeType == "etcd" {
		return &Etcd{
			ConnectionString: *storeConnection,
			Path: *configurationPath,
		}
	} else if *storeType == "consul" {
		return &Consul{
			ConnectionString: *storeConnection,
			Path: *configurationPath,
		}
	} else if *storeType == "zookeeper" {
		return &ZooKeeper{
			ConnectionString: *storeConnection,
			Path: *configurationPath,
		}
	} else {
		logger.Panic("Key-value store type not supported: ", *storeType)
		return nil
	}
}
<|endoftext|>


<commit_before>// http://qiita.com/suin/items/82ecb6f63ff4104d4f5d

package main

import (
    "log"
    "time"
)

func main() {
    a := 1
    log.Print("started.")

    // チャネル
    sleep1_finished := make(chan bool)
    sleep2_finished := make(chan bool)
    sleep3_finished := make(chan bool)

    go func() {
        // 1秒かかるコマンド
        log.Print("sleep1 started.")
        time.Sleep(1 * time.Second)
        log.Print("sleep1 finished.")
        a++
        log.Printf("a : %v", a)
        sleep1_finished <- true
    }()

    go func() {
        // 2秒かかるコマンド
        log.Print("sleep2 started.")
        time.Sleep(2 * time.Second)
        log.Print("sleep2 finished.")
        a++
        log.Printf("a : %v", a)
        sleep2_finished <- true
    }()

    go func() {
        // 3秒かかるコマンド
        log.Print("sleep3 started.")
        time.Sleep(3 * time.Second)
        log.Print("sleep3 finished.")
        a++
        log.Printf("a : %v", a)
        sleep3_finished <- true
    }()

    // 終わるまで待つ
    <- sleep3_finished
    <- sleep2_finished
    <- sleep1_finished

    log.Print("all finished.")
}
<commit_msg>Update<commit_after>package main

import (
    "log"
    "time"
)

func main() {
    a := 1
    log.Print("started.")

    // チャネル
    sleep1_finished := make(chan bool)

    go func() {
        // 1秒かかるコマンド
        log.Print("sleep1 started.")
        time.Sleep(1 * time.Second)
        log.Print("sleep1 finished.")
        a++
        log.Printf("a : %v", a)
        sleep1_finished <- true
    }()

    // 終わるまで待つ
    <- sleep1_finished

    log.Print("all finished.")
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	"collectd.org/api"
	"collectd.org/network"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"
)

// timeout specifies the number of iterations after which a metric times out,
// i.e. becomes stale and is removed from collectdCollector.valueLists. It is
// modeled and named after the top-level "Timeout" setting of collectd.
const timeout = 2

var (
	webAddress       = flag.String("web.listen-address", ":9103", "Address on which to expose metrics and web interface.")
	collectdAddress  = flag.String("collectd.listen-address", "", "Network address on which to accept collectd binary network packets, e.g. \":25826\".")
	collectdAuth     = flag.String("collectd.auth-file", "", "File mapping user names to pre-shared keys (passwords).")
	collectdSecurity = flag.String("collectd.security-level", "None", "Minimum required security level for accepted packets. Must be one of \"None\", \"Sign\" and \"Encrypt\".")
	metricsPath      = flag.String("web.telemetry-path", "/metrics", "Path under which to expose Prometheus metrics.")
	collectdPostPath = flag.String("web.collectd-push-path", "/collectd-post", "Path under which to accept POST requests from collectd.")
	lastPush         = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "collectd_last_push_timestamp_seconds",
			Help: "Unix timestamp of the last received collectd metrics push in seconds.",
		},
	)
)

// newName converts one data source of a value list to a string representation.
func newName(vl api.ValueList, index int) string {
	var name string
	if vl.Plugin == vl.Type || strings.HasPrefix(vl.Type, vl.Plugin+"_") {
		name = "collectd_" + vl.Type
	} else {
		name = "collectd_" + vl.Plugin + "_" + vl.Type
	}
	if dsname := vl.DSName(index); dsname != "value" {
		name += "_" + dsname
	}
	switch vl.Values[index].(type) {
	case api.Counter:
		name += "_total"
	}

	return name
}

// newLabels converts the plugin and type instance of vl to a set of prometheus.Labels.
func newLabels(vl api.ValueList) prometheus.Labels {
	labels := prometheus.Labels{}
	if vl.PluginInstance != "" {
		labels[vl.Plugin] = vl.PluginInstance
	}
	if vl.TypeInstance != "" {
		if vl.PluginInstance == "" {
			labels[vl.Plugin] = vl.TypeInstance
		} else {
			labels["type"] = vl.TypeInstance
		}
	}
	labels["instance"] = vl.Host

	return labels
}

// newDesc converts one data source of a value list to a Prometheus description.
func newDesc(vl api.ValueList, index int) *prometheus.Desc {
	help := fmt.Sprintf("Collectd exporter: '%s' Type: '%s' Dstype: '%T' Dsname: '%s'",
		vl.Plugin, vl.Type, vl.Values[index], vl.DSName(index))

	return prometheus.NewDesc(newName(vl, index), help, []string{}, newLabels(vl))
}

// newMetric converts one data source of a value list to a Prometheus metric.
func newMetric(vl api.ValueList, index int) (prometheus.Metric, error) {
	var value float64
	var valueType prometheus.ValueType

	switch v := vl.Values[index].(type) {
	case api.Gauge:
		value = float64(v)
		valueType = prometheus.GaugeValue
	case api.Derive:
		value = float64(v)
		valueType = prometheus.CounterValue
	case api.Counter:
		value = float64(v)
		valueType = prometheus.CounterValue
	default:
		return nil, fmt.Errorf("unknown value type: %T", v)
	}

	return prometheus.NewConstMetric(newDesc(vl, index), valueType, value)
}

type collectdCollector struct {
	ch         chan api.ValueList
	valueLists map[string]api.ValueList
	mu         *sync.Mutex
}

func newCollectdCollector() *collectdCollector {
	c := &collectdCollector{
		ch:         make(chan api.ValueList, 0),
		valueLists: make(map[string]api.ValueList),
		mu:         &sync.Mutex{},
	}
	go c.processSamples()
	return c
}

func (c *collectdCollector) collectdPost(w http.ResponseWriter, r *http.Request) {
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var valueLists []api.ValueList
	if err := json.Unmarshal(data, &valueLists); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	for _, vl := range valueLists {
		c.Write(vl)
	}
}

func (c *collectdCollector) processSamples() {
	ticker := time.NewTicker(time.Minute).C
	for {
		select {
		case vl := <-c.ch:
			id := vl.Identifier.String()
			c.mu.Lock()
			c.valueLists[id] = vl
			c.mu.Unlock()

		case <-ticker:
			// Garbage collect expired value lists.
			now := time.Now()
			c.mu.Lock()
			for id, vl := range c.valueLists {
				validUntil := vl.Time.Add(timeout * vl.Interval)
				if validUntil.Before(now) {
					delete(c.valueLists, id)
				}
			}
			c.mu.Unlock()
		}
	}
}

// Collect implements prometheus.Collector.
func (c collectdCollector) Collect(ch chan<- prometheus.Metric) {
	ch <- lastPush

	c.mu.Lock()
	valueLists := make([]api.ValueList, 0, len(c.valueLists))
	for _, vl := range c.valueLists {
		valueLists = append(valueLists, vl)
	}
	c.mu.Unlock()

	now := time.Now()
	for _, vl := range valueLists {
		validUntil := vl.Time.Add(timeout * vl.Interval)
		if validUntil.Before(now) {
			continue
		}

		for i := range vl.Values {
			m, err := newMetric(vl, i)
			if err != nil {
				log.Printf("newMetric: %v", err)
				continue
			}

			ch <- m
		}
	}
}

// Describe implements prometheus.Collector.
func (c collectdCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- lastPush.Desc()
}

// Write writes "vl" to the collector's channel, to be (asynchronously)
// processed by processSamples(). It implements api.Writer.
func (c collectdCollector) Write(vl api.ValueList) error {
	lastPush.Set(float64(time.Now().UnixNano()) / 1e9)
	c.ch <- vl

	return nil
}

func startCollectdServer(w api.Writer) {
	if *collectdAddress == "" {
		return
	}

	srv := network.Server{
		Addr:   *collectdAddress,
		Writer: w,
	}

	if *collectdAuth != "" {
		srv.PasswordLookup = network.NewAuthFile(*collectdAuth)
	}

	switch strings.ToLower(*collectdSecurity) {
	case "", "none":
		srv.SecurityLevel = network.None
	case "sign":
		srv.SecurityLevel = network.Sign
	case "encrypt":
		srv.SecurityLevel = network.Encrypt
	default:
		log.Fatalf("Unknown security level %q. Must be one of \"None\", \"Sign\" and \"Encrypt\".", *collectdSecurity)
	}

	go log.Fatal(srv.ListenAndWrite())
}

func main() {
	flag.Parse()

	c := newCollectdCollector()
	prometheus.MustRegister(c)

	startCollectdServer(c)

	if *collectdPostPath != "" {
		http.HandleFunc(*collectdPostPath, c.collectdPost)
	}

	http.Handle(*metricsPath, prometheus.Handler())

	log.Infof("Starting Server: %s", *webAddress)
	http.ListenAndServe(*webAddress, nil)
}
<commit_msg>Fix duplicate log package import<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"
	"time"

	"collectd.org/api"
	"collectd.org/network"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"
)

// timeout specifies the number of iterations after which a metric times out,
// i.e. becomes stale and is removed from collectdCollector.valueLists. It is
// modeled and named after the top-level "Timeout" setting of collectd.
const timeout = 2

var (
	webAddress       = flag.String("web.listen-address", ":9103", "Address on which to expose metrics and web interface.")
	collectdAddress  = flag.String("collectd.listen-address", "", "Network address on which to accept collectd binary network packets, e.g. \":25826\".")
	collectdAuth     = flag.String("collectd.auth-file", "", "File mapping user names to pre-shared keys (passwords).")
	collectdSecurity = flag.String("collectd.security-level", "None", "Minimum required security level for accepted packets. Must be one of \"None\", \"Sign\" and \"Encrypt\".")
	metricsPath      = flag.String("web.telemetry-path", "/metrics", "Path under which to expose Prometheus metrics.")
	collectdPostPath = flag.String("web.collectd-push-path", "/collectd-post", "Path under which to accept POST requests from collectd.")
	lastPush         = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "collectd_last_push_timestamp_seconds",
			Help: "Unix timestamp of the last received collectd metrics push in seconds.",
		},
	)
)

// newName converts one data source of a value list to a string representation.
func newName(vl api.ValueList, index int) string {
	var name string
	if vl.Plugin == vl.Type || strings.HasPrefix(vl.Type, vl.Plugin+"_") {
		name = "collectd_" + vl.Type
	} else {
		name = "collectd_" + vl.Plugin + "_" + vl.Type
	}
	if dsname := vl.DSName(index); dsname != "value" {
		name += "_" + dsname
	}
	switch vl.Values[index].(type) {
	case api.Counter:
		name += "_total"
	}

	return name
}

// newLabels converts the plugin and type instance of vl to a set of prometheus.Labels.
func newLabels(vl api.ValueList) prometheus.Labels {
	labels := prometheus.Labels{}
	if vl.PluginInstance != "" {
		labels[vl.Plugin] = vl.PluginInstance
	}
	if vl.TypeInstance != "" {
		if vl.PluginInstance == "" {
			labels[vl.Plugin] = vl.TypeInstance
		} else {
			labels["type"] = vl.TypeInstance
		}
	}
	labels["instance"] = vl.Host

	return labels
}

// newDesc converts one data source of a value list to a Prometheus description.
func newDesc(vl api.ValueList, index int) *prometheus.Desc {
	help := fmt.Sprintf("Collectd exporter: '%s' Type: '%s' Dstype: '%T' Dsname: '%s'",
		vl.Plugin, vl.Type, vl.Values[index], vl.DSName(index))

	return prometheus.NewDesc(newName(vl, index), help, []string{}, newLabels(vl))
}

// newMetric converts one data source of a value list to a Prometheus metric.
func newMetric(vl api.ValueList, index int) (prometheus.Metric, error) {
	var value float64
	var valueType prometheus.ValueType

	switch v := vl.Values[index].(type) {
	case api.Gauge:
		value = float64(v)
		valueType = prometheus.GaugeValue
	case api.Derive:
		value = float64(v)
		valueType = prometheus.CounterValue
	case api.Counter:
		value = float64(v)
		valueType = prometheus.CounterValue
	default:
		return nil, fmt.Errorf("unknown value type: %T", v)
	}

	return prometheus.NewConstMetric(newDesc(vl, index), valueType, value)
}

type collectdCollector struct {
	ch         chan api.ValueList
	valueLists map[string]api.ValueList
	mu         *sync.Mutex
}

func newCollectdCollector() *collectdCollector {
	c := &collectdCollector{
		ch:         make(chan api.ValueList, 0),
		valueLists: make(map[string]api.ValueList),
		mu:         &sync.Mutex{},
	}
	go c.processSamples()
	return c
}

func (c *collectdCollector) collectdPost(w http.ResponseWriter, r *http.Request) {
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var valueLists []api.ValueList
	if err := json.Unmarshal(data, &valueLists); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	for _, vl := range valueLists {
		c.Write(vl)
	}
}

func (c *collectdCollector) processSamples() {
	ticker := time.NewTicker(time.Minute).C
	for {
		select {
		case vl := <-c.ch:
			id := vl.Identifier.String()
			c.mu.Lock()
			c.valueLists[id] = vl
			c.mu.Unlock()

		case <-ticker:
			// Garbage collect expired value lists.
			now := time.Now()
			c.mu.Lock()
			for id, vl := range c.valueLists {
				validUntil := vl.Time.Add(timeout * vl.Interval)
				if validUntil.Before(now) {
					delete(c.valueLists, id)
				}
			}
			c.mu.Unlock()
		}
	}
}

// Collect implements prometheus.Collector.
func (c collectdCollector) Collect(ch chan<- prometheus.Metric) {
	ch <- lastPush

	c.mu.Lock()
	valueLists := make([]api.ValueList, 0, len(c.valueLists))
	for _, vl := range c.valueLists {
		valueLists = append(valueLists, vl)
	}
	c.mu.Unlock()

	now := time.Now()
	for _, vl := range valueLists {
		validUntil := vl.Time.Add(timeout * vl.Interval)
		if validUntil.Before(now) {
			continue
		}

		for i := range vl.Values {
			m, err := newMetric(vl, i)
			if err != nil {
				log.Printf("newMetric: %v", err)
				continue
			}

			ch <- m
		}
	}
}

// Describe implements prometheus.Collector.
func (c collectdCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- lastPush.Desc()
}

// Write writes "vl" to the collector's channel, to be (asynchronously)
// processed by processSamples(). It implements api.Writer.
func (c collectdCollector) Write(vl api.ValueList) error {
	lastPush.Set(float64(time.Now().UnixNano()) / 1e9)
	c.ch <- vl

	return nil
}

func startCollectdServer(w api.Writer) {
	if *collectdAddress == "" {
		return
	}

	srv := network.Server{
		Addr:   *collectdAddress,
		Writer: w,
	}

	if *collectdAuth != "" {
		srv.PasswordLookup = network.NewAuthFile(*collectdAuth)
	}

	switch strings.ToLower(*collectdSecurity) {
	case "", "none":
		srv.SecurityLevel = network.None
	case "sign":
		srv.SecurityLevel = network.Sign
	case "encrypt":
		srv.SecurityLevel = network.Encrypt
	default:
		log.Fatalf("Unknown security level %q. Must be one of \"None\", \"Sign\" and \"Encrypt\".", *collectdSecurity)
	}

	go log.Fatal(srv.ListenAndWrite())
}

func main() {
	flag.Parse()

	c := newCollectdCollector()
	prometheus.MustRegister(c)

	startCollectdServer(c)

	if *collectdPostPath != "" {
		http.HandleFunc(*collectdPostPath, c.collectdPost)
	}

	http.Handle(*metricsPath, prometheus.Handler())

	log.Infof("Starting Server: %s", *webAddress)
	http.ListenAndServe(*webAddress, nil)
}
<|endoftext|>