<commit_before>// Copyright 2018 Jigsaw Operations LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metrics

import (
	"errors"
	"io"
	"net"
	"time"

	onet "github.com/Jigsaw-Code/outline-ss-server/net"
	"github.com/oschwald/geoip2-golang"
	"github.com/prometheus/client_golang/prometheus"
)

// ShadowsocksMetrics registers metrics for the Shadowsocks service.
type ShadowsocksMetrics interface {
	GetLocation(net.Addr) (string, error)

	SetNumAccessKeys(numKeys int, numPorts int)
	AddUDPPacketFromClient(clientLocation, accessKey, status string, clientProxyBytes, proxyTargetBytes int)
	AddUDPPacketFromTarget(clientLocation, accessKey, status string, targetProxyBytes, proxyClientBytes int)
	AddOpenTCPConnection(clientLocation string)
	AddClosedTCPConnection(clientLocation, accessKey, status string, data ProxyMetrics, duration time.Duration)

	AddUdpNatEntry()
	RemoveUdpNatEntry()
}

type shadowsocksMetrics struct {
	ipCountryDB *geoip2.Reader

	accessKeys prometheus.Gauge
	ports      prometheus.Gauge
	dataBytes  *prometheus.CounterVec
	// TODO: Add time to first byte.

	tcpOpenConnections   *prometheus.CounterVec
	tcpClosedConnections *prometheus.CounterVec
	// TODO: Define a time window for the duration summary (e.g. 1 hour)
	tcpConnectionDurationMs *prometheus.SummaryVec

	udpAddedNatEntries   prometheus.Counter
	udpRemovedNatEntries prometheus.Counter
}

func NewShadowsocksMetrics(ipCountryDB *geoip2.Reader) ShadowsocksMetrics {
	m := &shadowsocksMetrics{
		ipCountryDB: ipCountryDB,
		accessKeys: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: "shadowsocks",
			Name:      "keys",
			Help:      "Count of access keys",
		}),
		ports: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: "shadowsocks",
			Name:      "ports",
			Help:      "Count of open Shadowsocks ports",
		}),
		tcpOpenConnections: prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: "shadowsocks",
			Subsystem: "tcp",
			Name:      "connections_opened",
			Help:      "Count of open TCP connections",
		}, []string{"location"}),
		tcpClosedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: "shadowsocks",
			Subsystem: "tcp",
			Name:      "connections_closed",
			Help:      "Count of closed TCP connections",
		}, []string{"location", "status", "access_key"}),
		tcpConnectionDurationMs: prometheus.NewSummaryVec(
			prometheus.SummaryOpts{
				Namespace:  "shadowsocks",
				Subsystem:  "tcp",
				Name:       "connection_duration_ms",
				Help:       "TCP connection duration distributions.",
				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
			}, []string{"location", "status", "access_key"}),
		dataBytes: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: "shadowsocks",
				Name:      "data_bytes",
				Help:      "Bytes tranferred by the proxy",
			}, []string{"dir", "proto", "location", "status", "access_key"}),
		udpAddedNatEntries: prometheus.NewCounter(
			prometheus.CounterOpts{
				Namespace: "shadowsocks",
				Subsystem: "udp",
				Name:      "nat_entries_added",
				Help:      "Entries added to the UDP NAT table",
			}),
		udpRemovedNatEntries: prometheus.NewCounter(
			prometheus.CounterOpts{
				Namespace: "shadowsocks",
				Subsystem: "udp",
				Name:      "nat_entries_removed",
				Help:      "Entries removed from the UDP NAT table",
			}),
	}
	// TODO: Is it possible to pass where to register the collectors?
	prometheus.MustRegister(m.accessKeys, m.ports, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs,
		m.dataBytes, m.udpAddedNatEntries, m.udpRemovedNatEntries)
	return m
}

func (m *shadowsocksMetrics) GetLocation(addr net.Addr) (string, error) {
	if m.ipCountryDB == nil {
		return "", nil
	}
	hostname, _, err := net.SplitHostPort(addr.String())
	if err != nil {
		return "", errors.New("Failed to split hostname and port")
	}
	ip := net.ParseIP(hostname)
	if ip == nil {
		return "", errors.New("Failed to parse address as IP")
	}
	record, err := m.ipCountryDB.Country(ip)
	if err != nil {
		return "", errors.New("Did not find country for IP")
	}
	return record.Country.IsoCode, nil
}

func (m *shadowsocksMetrics) SetNumAccessKeys(numKeys int, ports int) {
	m.accessKeys.Set(float64(numKeys))
	m.ports.Set(float64(ports))
}

func (m *shadowsocksMetrics) AddOpenTCPConnection(clientLocation string) {
	m.tcpOpenConnections.WithLabelValues(clientLocation).Inc()
}

func (m *shadowsocksMetrics) AddClosedTCPConnection(clientLocation, accessKey, status string, data ProxyMetrics, duration time.Duration) {
	m.tcpClosedConnections.WithLabelValues(clientLocation, status, accessKey).Inc()
	m.tcpConnectionDurationMs.WithLabelValues(clientLocation, status, accessKey).Observe(duration.Seconds() * 1000)
	m.dataBytes.WithLabelValues("c>p", "tcp", clientLocation, status, accessKey).Add(float64(data.ClientProxy))
	m.dataBytes.WithLabelValues("p>t", "tcp", clientLocation, status, accessKey).Add(float64(data.ProxyTarget))
	m.dataBytes.WithLabelValues("p<t", "tcp", clientLocation, status, accessKey).Add(float64(data.TargetProxy))
	m.dataBytes.WithLabelValues("c<p", "tcp", clientLocation, status, accessKey).Add(float64(data.ProxyClient))
}

func (m *shadowsocksMetrics) AddUDPPacketFromClient(clientLocation, accessKey, status string, clientProxyBytes, proxyTargetBytes int) {
	m.dataBytes.WithLabelValues("c>p", "udp", clientLocation, status, accessKey).Add(float64(clientProxyBytes))
	m.dataBytes.WithLabelValues("p>t", "udp", clientLocation, status, accessKey).Add(float64(proxyTargetBytes))
}

func (m *shadowsocksMetrics) AddUDPPacketFromTarget(clientLocation, accessKey, status string, targetProxyBytes, proxyClientBytes int) {
	m.dataBytes.WithLabelValues("p<t", "udp", clientLocation, status, accessKey).Add(float64(targetProxyBytes))
	m.dataBytes.WithLabelValues("c<p", "udp", clientLocation, status, accessKey).Add(float64(proxyClientBytes))
}

func (m *shadowsocksMetrics) AddUdpNatEntry() {
	m.udpAddedNatEntries.Inc()
}

func (m *shadowsocksMetrics) RemoveUdpNatEntry() {
	m.udpRemovedNatEntries.Inc()
}

type ProxyMetrics struct {
	ClientProxy int64
	ProxyTarget int64
	TargetProxy int64
	ProxyClient int64
}

func (m *ProxyMetrics) add(other ProxyMetrics) {
	m.ClientProxy += other.ClientProxy
	m.ProxyTarget += other.ProxyTarget
	m.TargetProxy += other.TargetProxy
	m.ProxyClient += other.ProxyClient
}

type measuredConn struct {
	onet.DuplexConn
	io.WriterTo
	readCount *int64
	io.ReaderFrom
	writeCount *int64
}

func (c *measuredConn) Read(b []byte) (int, error) {
	n, err := c.DuplexConn.Read(b)
	*c.readCount += int64(n)
	return n, err
}

func (c *measuredConn) WriteTo(w io.Writer) (int64, error) {
	n, err := io.Copy(w, c.DuplexConn)
	*c.readCount += n
	return n, err
}

func (c *measuredConn) Write(b []byte) (int, error) {
	n, err := c.DuplexConn.Write(b)
	*c.writeCount += int64(n)
	return n, err
}

func (c *measuredConn) ReadFrom(r io.Reader) (int64, error) {
	n, err := io.Copy(c.DuplexConn, r)
	*c.writeCount += n
	return n, err
}

func MeasureConn(conn onet.DuplexConn, bytesSent, bytesRceived *int64) onet.DuplexConn {
	return &measuredConn{DuplexConn: conn, writeCount: bytesSent, readCount: bytesRceived}
}
<commit_msg>Add empty ISO check<commit_after>// Copyright 2018 Jigsaw Operations LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metrics

import (
	"errors"
	"io"
	"net"
	"time"

	onet "github.com/Jigsaw-Code/outline-ss-server/net"
	"github.com/oschwald/geoip2-golang"
	"github.com/prometheus/client_golang/prometheus"
)

// ShadowsocksMetrics registers metrics for the Shadowsocks service.
type ShadowsocksMetrics interface {
	GetLocation(net.Addr) (string, error)

	SetNumAccessKeys(numKeys int, numPorts int)
	AddUDPPacketFromClient(clientLocation, accessKey, status string, clientProxyBytes, proxyTargetBytes int)
	AddUDPPacketFromTarget(clientLocation, accessKey, status string, targetProxyBytes, proxyClientBytes int)
	AddOpenTCPConnection(clientLocation string)
	AddClosedTCPConnection(clientLocation, accessKey, status string, data ProxyMetrics, duration time.Duration)

	AddUdpNatEntry()
	RemoveUdpNatEntry()
}

type shadowsocksMetrics struct {
	ipCountryDB *geoip2.Reader

	accessKeys prometheus.Gauge
	ports      prometheus.Gauge
	dataBytes  *prometheus.CounterVec
	// TODO: Add time to first byte.

	tcpOpenConnections   *prometheus.CounterVec
	tcpClosedConnections *prometheus.CounterVec
	// TODO: Define a time window for the duration summary (e.g. 1 hour)
	tcpConnectionDurationMs *prometheus.SummaryVec

	udpAddedNatEntries   prometheus.Counter
	udpRemovedNatEntries prometheus.Counter
}

func NewShadowsocksMetrics(ipCountryDB *geoip2.Reader) ShadowsocksMetrics {
	m := &shadowsocksMetrics{
		ipCountryDB: ipCountryDB,
		accessKeys: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: "shadowsocks",
			Name:      "keys",
			Help:      "Count of access keys",
		}),
		ports: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: "shadowsocks",
			Name:      "ports",
			Help:      "Count of open Shadowsocks ports",
		}),
		tcpOpenConnections: prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: "shadowsocks",
			Subsystem: "tcp",
			Name:      "connections_opened",
			Help:      "Count of open TCP connections",
		}, []string{"location"}),
		tcpClosedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: "shadowsocks",
			Subsystem: "tcp",
			Name:      "connections_closed",
			Help:      "Count of closed TCP connections",
		}, []string{"location", "status", "access_key"}),
		tcpConnectionDurationMs: prometheus.NewSummaryVec(
			prometheus.SummaryOpts{
				Namespace:  "shadowsocks",
				Subsystem:  "tcp",
				Name:       "connection_duration_ms",
				Help:       "TCP connection duration distributions.",
				Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
			}, []string{"location", "status", "access_key"}),
		dataBytes: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: "shadowsocks",
				Name:      "data_bytes",
				Help:      "Bytes tranferred by the proxy",
			}, []string{"dir", "proto", "location", "status", "access_key"}),
		udpAddedNatEntries: prometheus.NewCounter(
			prometheus.CounterOpts{
				Namespace: "shadowsocks",
				Subsystem: "udp",
				Name:      "nat_entries_added",
				Help:      "Entries added to the UDP NAT table",
			}),
		udpRemovedNatEntries: prometheus.NewCounter(
			prometheus.CounterOpts{
				Namespace: "shadowsocks",
				Subsystem: "udp",
				Name:      "nat_entries_removed",
				Help:      "Entries removed from the UDP NAT table",
			}),
	}
	// TODO: Is it possible to pass where to register the collectors?
	prometheus.MustRegister(m.accessKeys, m.ports, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs,
		m.dataBytes, m.udpAddedNatEntries, m.udpRemovedNatEntries)
	return m
}

func (m *shadowsocksMetrics) GetLocation(addr net.Addr) (string, error) {
	if m.ipCountryDB == nil {
		return "", nil
	}
	hostname, _, err := net.SplitHostPort(addr.String())
	if err != nil {
		return "", errors.New("Failed to split hostname and port")
	}
	ip := net.ParseIP(hostname)
	if ip == nil {
		return "", errors.New("Failed to parse address as IP")
	}
	record, err := m.ipCountryDB.Country(ip)
	if err != nil {
		return "", errors.New("Did not find country for IP")
	}
	if record.Country.IsoCode == "" {
		return "", errors.New("No ISO code")
	}
	return record.Country.IsoCode, nil
}

func (m *shadowsocksMetrics) SetNumAccessKeys(numKeys int, ports int) {
	m.accessKeys.Set(float64(numKeys))
	m.ports.Set(float64(ports))
}

func (m *shadowsocksMetrics) AddOpenTCPConnection(clientLocation string) {
	m.tcpOpenConnections.WithLabelValues(clientLocation).Inc()
}

func (m *shadowsocksMetrics) AddClosedTCPConnection(clientLocation, accessKey, status string, data ProxyMetrics, duration time.Duration) {
	m.tcpClosedConnections.WithLabelValues(clientLocation, status, accessKey).Inc()
	m.tcpConnectionDurationMs.WithLabelValues(clientLocation, status, accessKey).Observe(duration.Seconds() * 1000)
	m.dataBytes.WithLabelValues("c>p", "tcp", clientLocation, status, accessKey).Add(float64(data.ClientProxy))
	m.dataBytes.WithLabelValues("p>t", "tcp", clientLocation, status, accessKey).Add(float64(data.ProxyTarget))
	m.dataBytes.WithLabelValues("p<t", "tcp", clientLocation, status, accessKey).Add(float64(data.TargetProxy))
	m.dataBytes.WithLabelValues("c<p", "tcp", clientLocation, status, accessKey).Add(float64(data.ProxyClient))
}

func (m *shadowsocksMetrics) AddUDPPacketFromClient(clientLocation, accessKey, status string, clientProxyBytes, proxyTargetBytes int) {
	m.dataBytes.WithLabelValues("c>p", "udp", clientLocation, status, accessKey).Add(float64(clientProxyBytes))
	m.dataBytes.WithLabelValues("p>t", "udp", clientLocation, status, accessKey).Add(float64(proxyTargetBytes))
}

func (m *shadowsocksMetrics) AddUDPPacketFromTarget(clientLocation, accessKey, status string, targetProxyBytes, proxyClientBytes int) {
	m.dataBytes.WithLabelValues("p<t", "udp", clientLocation, status, accessKey).Add(float64(targetProxyBytes))
	m.dataBytes.WithLabelValues("c<p", "udp", clientLocation, status, accessKey).Add(float64(proxyClientBytes))
}

func (m *shadowsocksMetrics) AddUdpNatEntry() {
	m.udpAddedNatEntries.Inc()
}

func (m *shadowsocksMetrics) RemoveUdpNatEntry() {
	m.udpRemovedNatEntries.Inc()
}

type ProxyMetrics struct {
	ClientProxy int64
	ProxyTarget int64
	TargetProxy int64
	ProxyClient int64
}

func (m *ProxyMetrics) add(other ProxyMetrics) {
	m.ClientProxy += other.ClientProxy
	m.ProxyTarget += other.ProxyTarget
	m.TargetProxy += other.TargetProxy
	m.ProxyClient += other.ProxyClient
}

type measuredConn struct {
	onet.DuplexConn
	io.WriterTo
	readCount *int64
	io.ReaderFrom
	writeCount *int64
}

func (c *measuredConn) Read(b []byte) (int, error) {
	n, err := c.DuplexConn.Read(b)
	*c.readCount += int64(n)
	return n, err
}

func (c *measuredConn) WriteTo(w io.Writer) (int64, error) {
	n, err := io.Copy(w, c.DuplexConn)
	*c.readCount += n
	return n, err
}

func (c *measuredConn) Write(b []byte) (int, error) {
	n, err := c.DuplexConn.Write(b)
	*c.writeCount += int64(n)
	return n, err
}

func (c *measuredConn) ReadFrom(r io.Reader) (int64, error) {
	n, err := io.Copy(c.DuplexConn, r)
	*c.writeCount += n
	return n, err
}

func MeasureConn(conn onet.DuplexConn, bytesSent, bytesRceived *int64) onet.DuplexConn {
	return &measuredConn{DuplexConn: conn, writeCount: bytesSent, readCount: bytesRceived}
}
<|endoftext|>


<commit_before>// The metrics package defines prometheus metric types and provides
// convenience methods to add accounting to various parts of the pipeline.
//
// When defining new operations or metrics, these are helpful values to track:
//  - things coming into or go out of the system: requests, files, tests, api calls.
//  - the success or error status of any of the above.
//  - the distribution of processing latency.
package metrics

import (
	"math"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

func init() {
	// Register the metrics defined with Prometheus's default registry.
	prometheus.MustRegister(WorkerCount)
	prometheus.MustRegister(TaskCount)
	prometheus.MustRegister(BigQueryInsert)
	prometheus.MustRegister(DurationHistogram)
	prometheus.MustRegister(InsertionHistogram)
	prometheus.MustRegister(FileSizeHistogram)
}

var (
	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_worker_count
	// Example usage:
	//   metrics.TaskCount.Inc() / .Dec()
	WorkerCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "etl_worker_count",
		Help: "Number of active workers.",
	})

	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_task_count{worker, status}
	// Example usage:
	//   metrics.TaskCount.WithLabelValues("ndt", "ok").Inc()
	TaskCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_task_count",
			Help: "Number of tasks/archive files processed.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// Counts the number of into BigQuery insert operations.
	//
	// Provides metrics:
	//   etl_worker_bigquery_insert_total{worker, status}
	// Usage example:
	//   metrics.BigQueryInsert.WithLabelValues("ndt", "200").Inc()
	BigQueryInsert = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_worker_bigquery_insert_total",
			Help: "Number of BigQuery insert operations.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// A histogram of bigquery insertion times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_insertion_time_seconds_bucket{type="...", le="..."}
	//   ...
	//   etl_insertion_time_seconds_sum{type="..."}
	//   etl_insertion_time_seconds_count{type="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.InsertionHistogram.WithLabelValues(
	//           "ndt_test", "ok").Observe(time.Since(t).Seconds())
	InsertionHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_insertion_time_seconds",
			Help: "Insertion time distributions.",
			Buckets: []float64{
				0.001, 0.003, 0.01, 0.03, 0.1, 0.2, 0.5, 1.0, 2.0,
				5.0, 10.0, 20.0, 50.0, 100.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"table", "status"},
	)

	// A histogram of worker processing times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_worker_duration_seconds_bucket{worker="...", le="..."}
	//   ...
	//   etl_worker_duration_seconds_sum{worker="..."}
	//   etl_worker_duration_seconds_count{worker="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.DurationHistogram.WithLabelValues(
	//           "ndt").Observe(time.Since(t).Seconds())
	DurationHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_worker_duration_seconds",
			Help: "Worker execution time distributions.",
			Buckets: []float64{
				0.001, 0.01, 0.1, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
				600.0, 1800.0, 3600.0, 7200.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		// TODO(soltesz): support a status field based on HTTP status.
		[]string{"worker"},
	)

	FileSizeHistogram = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name: "etl_web100_snaplog_file_size_bytes",
			Help: "Size of individual snaplog files.",
			Buckets: []float64{
				100000,
				1000000,   // mb
				2000000,   // mb
				4000000,   // mb
				8000000,   // mb
				10000000,  // 10 mb
				20000000,  // 20
				40000000,  // 40
				80000000,  // 80
				100000000, // 100 mb
				200000000, // 200
				400000000, // 400
				800000000, // 800
			},
		},
	)
)

// DurationHandler wraps the call of an inner http.HandlerFunc and records the runtime.
func DurationHandler(name string, inner http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()
		inner.ServeHTTP(w, r)
		// TODO(soltesz): collect success or failure status.
		DurationHistogram.WithLabelValues(name).Observe(time.Since(t).Seconds())
	}
}
<commit_msg>Add 1gb to file size histogram.<commit_after>// The metrics package defines prometheus metric types and provides
// convenience methods to add accounting to various parts of the pipeline.
//
// When defining new operations or metrics, these are helpful values to track:
//  - things coming into or go out of the system: requests, files, tests, api calls.
//  - the success or error status of any of the above.
//  - the distribution of processing latency.
package metrics

import (
	"math"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

func init() {
	// Register the metrics defined with Prometheus's default registry.
	prometheus.MustRegister(WorkerCount)
	prometheus.MustRegister(TaskCount)
	prometheus.MustRegister(BigQueryInsert)
	prometheus.MustRegister(DurationHistogram)
	prometheus.MustRegister(InsertionHistogram)
	prometheus.MustRegister(FileSizeHistogram)
}

var (
	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_worker_count
	// Example usage:
	//   metrics.TaskCount.Inc() / .Dec()
	WorkerCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "etl_worker_count",
		Help: "Number of active workers.",
	})

	// Counts the number of tasks processed by the pipeline.
	//
	// Provides metrics:
	//   etl_task_count{worker, status}
	// Example usage:
	//   metrics.TaskCount.WithLabelValues("ndt", "ok").Inc()
	TaskCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_task_count",
			Help: "Number of tasks/archive files processed.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// Counts the number of into BigQuery insert operations.
	//
	// Provides metrics:
	//   etl_worker_bigquery_insert_total{worker, status}
	// Usage example:
	//   metrics.BigQueryInsert.WithLabelValues("ndt", "200").Inc()
	BigQueryInsert = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "etl_worker_bigquery_insert_total",
			Help: "Number of BigQuery insert operations.",
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"worker", "status"},
	)

	// A histogram of bigquery insertion times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_insertion_time_seconds_bucket{type="...", le="..."}
	//   ...
	//   etl_insertion_time_seconds_sum{type="..."}
	//   etl_insertion_time_seconds_count{type="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.InsertionHistogram.WithLabelValues(
	//           "ndt_test", "ok").Observe(time.Since(t).Seconds())
	InsertionHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_insertion_time_seconds",
			Help: "Insertion time distributions.",
			Buckets: []float64{
				0.001, 0.003, 0.01, 0.03, 0.1, 0.2, 0.5, 1.0, 2.0,
				5.0, 10.0, 20.0, 50.0, 100.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		[]string{"table", "status"},
	)

	// A histogram of worker processing times. The buckets should use
	// periods that are intuitive for people.
	//
	// Provides metrics:
	//   etl_worker_duration_seconds_bucket{worker="...", le="..."}
	//   ...
	//   etl_worker_duration_seconds_sum{worker="..."}
	//   etl_worker_duration_seconds_count{worker="..."}
	// Usage example:
	//   t := time.Now()
	//   // do some stuff.
	//   metrics.DurationHistogram.WithLabelValues(
	//           "ndt").Observe(time.Since(t).Seconds())
	DurationHistogram = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "etl_worker_duration_seconds",
			Help: "Worker execution time distributions.",
			Buckets: []float64{
				0.001, 0.01, 0.1, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
				600.0, 1800.0, 3600.0, 7200.0, math.Inf(+1),
			},
		},
		// Worker type, e.g. ndt, sidestream, ptr, etc.
		// TODO(soltesz): support a status field based on HTTP status.
		[]string{"worker"},
	)

	FileSizeHistogram = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name: "etl_web100_snaplog_file_size_bytes",
			Help: "Size of individual snaplog files.",
			Buckets: []float64{
				100000,
				1000000,    // mb
				2000000,    // mb
				4000000,    // mb
				8000000,    // mb
				10000000,   // 10 mb
				20000000,   // 20
				40000000,   // 40
				80000000,   // 80
				100000000,  // 100 mb
				200000000,  // 200
				400000000,  // 400
				800000000,  // 800
				1000000000, // 1 gb
			},
		},
	)
)

// DurationHandler wraps the call of an inner http.HandlerFunc and records the runtime.
func DurationHandler(name string, inner http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()
		inner.ServeHTTP(w, r)
		// TODO(soltesz): collect success or failure status.
		DurationHistogram.WithLabelValues(name).Observe(time.Since(t).Seconds())
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"fmt"
	"regexp"

	"github.com/blang/semver"
	"github.com/spf13/pflag"

	"k8s.io/component-base/version"
)

// Options has all parameters needed for exposing metrics from components
type Options struct {
	ShowHiddenMetricsForVersion string
	DisabledMetrics             []string
	AllowListMapping            map[string]string
}

// NewOptions returns default metrics options
func NewOptions() *Options {
	return &Options{}
}

// Validate validates metrics flags options.
func (o *Options) Validate() []error {
	var errs []error
	err := validateShowHiddenMetricsVersion(parseVersion(version.Get()), o.ShowHiddenMetricsForVersion)
	if err != nil {
		errs = append(errs, err)
	}

	if err := validateAllowMetricLabel(o.AllowListMapping); err != nil {
		errs = append(errs, err)
	}

	if len(errs) == 0 {
		return nil
	}
	return errs
}

// AddFlags adds flags for exposing component metrics.
func (o *Options) AddFlags(fs *pflag.FlagSet) {
	if o != nil {
		o = NewOptions()
	}
	fs.StringVar(&o.ShowHiddenMetricsForVersion, "show-hidden-metrics-for-version", o.ShowHiddenMetricsForVersion,
		"The previous version for which you want to show hidden metrics. "+
			"Only the previous minor version is meaningful, other values will not be allowed. "+
			"The format is <major>.<minor>, e.g.: '1.16'. "+
			"The purpose of this format is make sure you have the opportunity to notice if the next release hides additional metrics, "+
			"rather than being surprised when they are permanently removed in the release after that.")
	fs.StringSliceVar(&o.DisabledMetrics,
		"disabled-metrics",
		o.DisabledMetrics,
		"This flag provides an escape hatch for misbehaving metrics. "+
			"You must provide the fully qualified metric name in order to disable it. "+
			"Disclaimer: disabling metrics is higher in precedence than showing hidden metrics.")
	fs.StringToStringVar(&o.AllowListMapping, "allow-metric-labels", o.AllowListMapping,
		"The map from metric-label to value allow-list of this label. The key's format is <MetricName>,<LabelName>. "+
			"The value's format is <allowed_value>,<allowed_value>..."+
			"e.g. metric1,label1='v1,v2,v3', metric1,label2='v1,v2,v3' metric2,label1='v1,v2,v3'.")
}

// Apply applies parameters into global configuration of metrics.
func (o *Options) Apply() {
	if o == nil {
		return
	}
	if len(o.ShowHiddenMetricsForVersion) > 0 {
		SetShowHidden()
	}
	// set disabled metrics
	for _, metricName := range o.DisabledMetrics {
		SetDisabledMetric(metricName)
	}
	if o.AllowListMapping != nil {
		SetLabelAllowListFromCLI(o.AllowListMapping)
	}
}

func validateShowHiddenMetricsVersion(currentVersion semver.Version, targetVersionStr string) error {
	if targetVersionStr == "" {
		return nil
	}

	validVersionStr := fmt.Sprintf("%d.%d", currentVersion.Major, currentVersion.Minor-1)
	if targetVersionStr != validVersionStr {
		return fmt.Errorf("--show-hidden-metrics-for-version must be omitted or have the value '%v'. Only the previous minor version is allowed", validVersionStr)
	}

	return nil
}

func validateAllowMetricLabel(allowListMapping map[string]string) error {
	if allowListMapping == nil {
		return nil
	}
	metricNameRegex := `[a-zA-Z_:][a-zA-Z0-9_:]*`
	labelRegex := `[a-zA-Z_][a-zA-Z0-9_]*`
	for k := range allowListMapping {
		reg := regexp.MustCompile(metricNameRegex + `,` + labelRegex)
		if reg.FindString(k) != k {
			return fmt.Errorf("--allow-metric-labels must has a list of kv pair with format `metricName:labelName=labelValue, labelValue,...`")
		}
	}
	return nil
}
<commit_msg>parameter 'disabled-metrics' is invalid<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"fmt"
	"regexp"

	"github.com/blang/semver"
	"github.com/spf13/pflag"

	"k8s.io/component-base/version"
)

// Options has all parameters needed for exposing metrics from components
type Options struct {
	ShowHiddenMetricsForVersion string
	DisabledMetrics             []string
	AllowListMapping            map[string]string
}

// NewOptions returns default metrics options
func NewOptions() *Options {
	return &Options{}
}

// Validate validates metrics flags options.
func (o *Options) Validate() []error {
	var errs []error
	err := validateShowHiddenMetricsVersion(parseVersion(version.Get()), o.ShowHiddenMetricsForVersion)
	if err != nil {
		errs = append(errs, err)
	}

	if err := validateAllowMetricLabel(o.AllowListMapping); err != nil {
		errs = append(errs, err)
	}

	if len(errs) == 0 {
		return nil
	}
	return errs
}

// AddFlags adds flags for exposing component metrics.
func (o *Options) AddFlags(fs *pflag.FlagSet) {
	if o == nil {
		return
	}
	fs.StringVar(&o.ShowHiddenMetricsForVersion, "show-hidden-metrics-for-version", o.ShowHiddenMetricsForVersion,
		"The previous version for which you want to show hidden metrics. "+
			"Only the previous minor version is meaningful, other values will not be allowed. "+
			"The format is <major>.<minor>, e.g.: '1.16'. "+
			"The purpose of this format is make sure you have the opportunity to notice if the next release hides additional metrics, "+
			"rather than being surprised when they are permanently removed in the release after that.")
	fs.StringSliceVar(&o.DisabledMetrics,
		"disabled-metrics",
		o.DisabledMetrics,
		"This flag provides an escape hatch for misbehaving metrics. "+
			"You must provide the fully qualified metric name in order to disable it. "+
			"Disclaimer: disabling metrics is higher in precedence than showing hidden metrics.")
	fs.StringToStringVar(&o.AllowListMapping, "allow-metric-labels", o.AllowListMapping,
		"The map from metric-label to value allow-list of this label. The key's format is <MetricName>,<LabelName>. "+
			"The value's format is <allowed_value>,<allowed_value>..."+
			"e.g. metric1,label1='v1,v2,v3', metric1,label2='v1,v2,v3' metric2,label1='v1,v2,v3'.")
}

// Apply applies parameters into global configuration of metrics.
func (o *Options) Apply() {
	if o == nil {
		return
	}
	if len(o.ShowHiddenMetricsForVersion) > 0 {
		SetShowHidden()
	}
	// set disabled metrics
	for _, metricName := range o.DisabledMetrics {
		SetDisabledMetric(metricName)
	}
	if o.AllowListMapping != nil {
		SetLabelAllowListFromCLI(o.AllowListMapping)
	}
}

func validateShowHiddenMetricsVersion(currentVersion semver.Version, targetVersionStr string) error {
	if targetVersionStr == "" {
		return nil
	}

	validVersionStr := fmt.Sprintf("%d.%d", currentVersion.Major, currentVersion.Minor-1)
	if targetVersionStr != validVersionStr {
		return fmt.Errorf("--show-hidden-metrics-for-version must be omitted or have the value '%v'. Only the previous minor version is allowed", validVersionStr)
	}

	return nil
}

func validateAllowMetricLabel(allowListMapping map[string]string) error {
	if allowListMapping == nil {
		return nil
	}
	metricNameRegex := `[a-zA-Z_:][a-zA-Z0-9_:]*`
	labelRegex := `[a-zA-Z_][a-zA-Z0-9_]*`
	for k := range allowListMapping {
		reg := regexp.MustCompile(metricNameRegex + `,` + labelRegex)
		if reg.FindString(k) != k {
			return fmt.Errorf("--allow-metric-labels must has a list of kv pair with format `metricName:labelName=labelValue, labelValue,...`")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// +build darwin,!kqueue

package notify

import (
	"reflect"
	"testing"
)

func TestSplitflags(t *testing.T) {
	cases := [...]struct {
		set   uint32
		flags []uint32
	}{
		{0, nil},
		{0xD, []uint32{0x1, 0x4, 0x8}},
		{0x0010 | 0x0040 | 0x0080 | 0x01000, []uint32{0x0010, 0x0040, 0x0080, 0x01000}},
		{0x40000 | 0x00100 | 0x00200, []uint32{0x00100, 0x00200, 0x40000}},
	}
	for i, cas := range cases {
		if flags := splitflags(cas.set); !reflect.DeepEqual(flags, cas.flags) {
			t.Errorf("want flags=%v; got %v (i=%d)", cas.flags, flags, i)
		}
	}
}

func TestWatchStrip(t *testing.T) {
	const (
		create = uint32(FSEventsCreated)
		remove = uint32(FSEventsRemoved)
		rename = uint32(FSEventsRenamed)
		write  = uint32(FSEventsModified)
		inode  = uint32(FSEventsInodeMetaMod)
		owner  = uint32(FSEventsChangeOwner)
	)
	cases := [...][]struct {
		path string
		flag uint32
		diff uint32
	}{
		// 1.
		{
			{"file", create | write, create | write},
			{"file", create | write | inode, write | inode},
		},
		// 2.
		{
			{"file", create, create},
			{"file", create | remove, remove},
			{"file", create | remove, create},
		},
		// 3.
		{
			{"file", create | write, create | write},
			{"file", create | write | owner, write | owner},
		},
		// 4.
		{
			{"file", create | write, create | write},
			{"file", write | inode, write | inode},
			{"file", remove | write | inode, remove},
		},
		{
			{"file", remove | write | inode, remove},
		},
	}
Test:
	for i, cas := range cases {
		if len(cas) == 0 {
			t.Log("skipped")
			continue
		}
		w := &watch{prev: make(map[string]uint32)}
		for j, cas := range cas {
			if diff := w.strip(cas.path, cas.flag); diff != cas.diff {
				t.Errorf("want diff=%v; got %v (i=%d, j=%d)", Event(cas.diff),
					Event(diff), i, j)
				continue Test
			}
		}
	}
}

// Test for cases 3) and 5) with shadowed write&create events.
//
// See comment for (flagdiff).diff method.
func TestWatcherShadowedWriteCreate(t *testing.T) {
	w := NewWatcherTest(t, "testdata/vfs.txt")
	defer w.Close()

	cases := [...]WCase{
		create(w, "src/github.com/rjeczalik/fs/.fs.go.swp"),
		write(w, "src/github.com/rjeczalik/fs/.fs.go.swp", []byte("XD")),
		write(w, "src/github.com/rjeczalik/fs/.fs.go.swp", []byte("XD")),
		remove(w, "src/github.com/rjeczalik/fs/.fs.go.swp"),
		create(w, "src/github.com/rjeczalik/fs/.fs.go.swp"),
		write(w, "src/github.com/rjeczalik/fs/.fs.go.swp", []byte("XD")),
	}

	w.ExpectAny(cases[:])
}
<commit_msg>skip test for #62<commit_after>// +build darwin,!kqueue

package notify

import (
	"reflect"
	"testing"
)

func TestSplitflags(t *testing.T) {
	cases := [...]struct {
		set   uint32
		flags []uint32
	}{
		{0, nil},
		{0xD, []uint32{0x1, 0x4, 0x8}},
		{0x0010 | 0x0040 | 0x0080 | 0x01000, []uint32{0x0010, 0x0040, 0x0080, 0x01000}},
		{0x40000 | 0x00100 | 0x00200, []uint32{0x00100, 0x00200, 0x40000}},
	}
	for i, cas := range cases {
		if flags := splitflags(cas.set); !reflect.DeepEqual(flags, cas.flags) {
			t.Errorf("want flags=%v; got %v (i=%d)", cas.flags, flags, i)
		}
	}
}

func TestWatchStrip(t *testing.T) {
	const (
		create = uint32(FSEventsCreated)
		remove = uint32(FSEventsRemoved)
		rename = uint32(FSEventsRenamed)
		write  = uint32(FSEventsModified)
		inode  = uint32(FSEventsInodeMetaMod)
		owner  = uint32(FSEventsChangeOwner)
	)
	cases := [...][]struct {
		path string
		flag uint32
		diff uint32
	}{
		// 1.
		{
			{"file", create | write, create | write},
			{"file", create | write | inode, write | inode},
		},
		// 2.
		{
			{"file", create, create},
			{"file", create | remove, remove},
			{"file", create | remove, create},
		},
		// 3.
		{
			{"file", create | write, create | write},
			{"file", create | write | owner, write | owner},
		},
		// 4.
		{
			{"file", create | write, create | write},
			{"file", write | inode, write | inode},
			{"file", remove | write | inode, remove},
		},
		{
			{"file", remove | write | inode, remove},
		},
	}
Test:
	for i, cas := range cases {
		if len(cas) == 0 {
			t.Log("skipped")
			continue
		}
		w := &watch{prev: make(map[string]uint32)}
		for j, cas := range cas {
			if diff := w.strip(cas.path, cas.flag); diff != cas.diff {
				t.Errorf("want diff=%v; got %v (i=%d, j=%d)", Event(cas.diff),
					Event(diff), i, j)
				continue Test
			}
		}
	}
}

// Test for cases 3) and 5) with shadowed write&create events.
//
// See comment for (flagdiff).diff method.
func TestWatcherShadowedWriteCreate(t *testing.T) {
	w := NewWatcherTest(t, "testdata/vfs.txt")
	defer w.Close()

	cases := [...]WCase{
		// i=0
		create(w, "src/github.com/rjeczalik/fs/.fs.go.swp"),
		// i=1
		write(w, "src/github.com/rjeczalik/fs/.fs.go.swp", []byte("XD")),
		// i=2
		write(w, "src/github.com/rjeczalik/fs/.fs.go.swp", []byte("XD")),
		// i=3
		remove(w, "src/github.com/rjeczalik/fs/.fs.go.swp"),
		// i=4
		create(w, "src/github.com/rjeczalik/fs/.fs.go.swp"),
		// i=5
		write(w, "src/github.com/rjeczalik/fs/.fs.go.swp", []byte("XD")),
	}

	w.ExpectAny(cases[:5]) // BUG(rjeczalik): #62
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/caiofilipini/grpc-weather/weather"
	"github.com/caiofilipini/grpc-weather/weather_server/provider"

	"google.golang.org/grpc"

	"golang.org/x/net/context"
)

const (
	defaultPort = 9000
)

type server struct {
	providers []*provider.WeatherProvider
}

func (s *server) registerProvider(p provider.WeatherProvider) {
	s.providers = append(s.providers, &p)
}

func (s server) queryProviders(q string) (*weather.WeatherResponse, error) {
	var responses []*provider.WeatherInfo
	var err error

	for _, p := range s.providers {
		prov := *p
		resp, e := prov.Query(q)
		if e != nil {
			err = e
		} else {
			log.Printf("[WeatherServer] Temperature obtained from %s is %.1f\n",
				prov.Name(), resp.Temperature)
			responses = append(responses, &resp)
		}
	}

	if len(responses) == 0 {
		return nil, err
	}

	return s.avg(responses), nil
}

func (s server) avg(responses []*provider.WeatherInfo) *weather.WeatherResponse {
	var sumTemp float64 = 0
	for _, r := range responses {
		if r.Found {
			sumTemp += r.Temperature
		}
	}
	avgTemp := sumTemp / float64(len(responses))

	return &weather.WeatherResponse{
		Temperature: avgTemp,
		Description: responses[0].Description,
		Found:       true,
	}
}

func (s server) CurrentConditions(ctx context.Context, req *weather.WeatherRequest) (*weather.WeatherResponse, error) {
	log.Println("[WeatherServer] Fetching weather information for", req.Location)
	defer elapsed(time.Now())

	return s.queryProviders(req.Location)
}

func main() {
	owmApiKey := strings.TrimSpace(os.Getenv("OPEN_WEATHER_MAP_API_KEY"))
	wuApiKey := strings.TrimSpace(os.Getenv("WEATHER_UNDERGROUND_API_KEY"))
	if owmApiKey == "" {
		log.Fatal("Missing API key for OpenWeatherMap")
	}
	if wuApiKey == "" {
		log.Fatal("Missing API key for Weather Underground")
	}

	weatherServer := &server{}
	weatherServer.registerProvider(provider.OpenWeatherMap{ApiKey: owmApiKey})
	weatherServer.registerProvider(provider.WeatherUnderground{ApiKey: wuApiKey})

	conn := listen()
	grpcServer := grpc.NewServer()
	weather.RegisterWeatherServer(grpcServer, weatherServer)
	grpcServer.Serve(conn)
}

func listen() net.Listener {
	port := assignPort()
	listenAddr := fmt.Sprintf(":%d", port)

	conn, err := net.Listen("tcp", listenAddr)
	if err != nil {
		log.Fatalf("Failed to listen on %s: %v", port, err)
	}

	log.Println("[WeatherServer] Listening on", port)
	return conn
}

func assignPort() int {
	if p := os.Getenv("PORT"); p != "" {
		port, err := strconv.Atoi(p)
		if err != nil {
			log.Fatalf("Invalid port %s", p)
		}
		return port
	}
	return defaultPort
}

func elapsed(start time.Time) {
	log.Printf("[WeatherServer] Request took %s\n", time.Since(start))
}
<commit_msg>Fix printf string format<commit_after>package main

import (
	"fmt"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/caiofilipini/grpc-weather/weather"
	"github.com/caiofilipini/grpc-weather/weather_server/provider"

	"google.golang.org/grpc"

	"golang.org/x/net/context"
)

const (
	defaultPort = 9000
)

type server struct {
	providers []*provider.WeatherProvider
}

func (s *server) registerProvider(p provider.WeatherProvider) {
	s.providers = append(s.providers, &p)
}

func (s server) queryProviders(q string) (*weather.WeatherResponse, error) {
	var responses []*provider.WeatherInfo
	var err error

	for _, p := range s.providers {
		prov := *p
		resp, e := prov.Query(q)
		if e != nil {
			err = e
		} else {
			log.Printf("[WeatherServer] Temperature obtained from %s is %.1f\n",
				prov.Name(), resp.Temperature)
			responses = append(responses, &resp)
		}
	}

	if len(responses) == 0 {
		return nil, err
	}

	return s.avg(responses), nil
}

func (s server) avg(responses []*provider.WeatherInfo) *weather.WeatherResponse {
	var sumTemp float64 = 0
	for _, r := range responses {
		if r.Found {
			sumTemp += r.Temperature
		}
	}
	avgTemp := sumTemp / float64(len(responses))

	return &weather.WeatherResponse{
		Temperature: avgTemp,
		Description: responses[0].Description,
		Found:       true,
	}
}

func (s server) CurrentConditions(ctx context.Context, req *weather.WeatherRequest) (*weather.WeatherResponse, error) {
	log.Println("[WeatherServer] Fetching weather information for", req.Location)
	defer elapsed(time.Now())

	return s.queryProviders(req.Location)
}

func main() {
	owmApiKey := strings.TrimSpace(os.Getenv("OPEN_WEATHER_MAP_API_KEY"))
	wuApiKey := strings.TrimSpace(os.Getenv("WEATHER_UNDERGROUND_API_KEY"))
	if owmApiKey == "" {
		log.Fatal("Missing API key for OpenWeatherMap")
	}
	if wuApiKey == "" {
		log.Fatal("Missing API key for Weather Underground")
	}

	weatherServer := &server{}
	weatherServer.registerProvider(provider.OpenWeatherMap{ApiKey: owmApiKey})
	weatherServer.registerProvider(provider.WeatherUnderground{ApiKey: wuApiKey})

	conn := listen()
	grpcServer := grpc.NewServer()
	weather.RegisterWeatherServer(grpcServer, weatherServer)
	grpcServer.Serve(conn)
}

func listen() net.Listener {
	port := assignPort()
	listenAddr := fmt.Sprintf(":%d", port)

	conn, err := net.Listen("tcp", listenAddr)
	if err != nil {
		log.Fatalf("Failed to listen on %d: %v", port, err)
	}

	log.Println("[WeatherServer] Listening on", port)
	return conn
}

func assignPort() int {
	if p := os.Getenv("PORT"); p != "" {
		port, err := strconv.Atoi(p)
		if err != nil {
			log.Fatalf("Invalid port %s", p)
		}
		return port
	}
	return defaultPort
}

func elapsed(start time.Time) {
	log.Printf("[WeatherServer] Request took %s\n", time.Since(start))
}
<|endoftext|>


<commit_before>package cache

import (
	"time"

	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// Document holds the key-value pair for mongo cache
type Document struct {
	Key      string      `bson:"_id" json:"_id"`
	Value    interface{} `bson:"value" json:"value"`
	ExpireAt time.Time   `bson:"expireAt" json:"expireAt"`
}

// getKey fetches the key with its key
func (m *MongoCache) get(key string) (*Document, error) {
	keyValue := new(Document)

	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{
			"_id": key,
			"expireAt": bson.M{
				"$gt": time.Now().UTC(),
			}}).One(&keyValue)
	}

	err := m.run(m.CollectionName, query)
	if err != nil {
		return nil, err
	}

	return keyValue, nil
}

func (m *MongoCache) set(key string, duration time.Duration, value interface{}) error {
	update := bson.M{
		"_id":      key,
		"value":    value,
		"expireAt": time.Now().Add(duration),
	}

	query := func(c *mgo.Collection) error {
		_, err := c.UpsertId(key, update)
		return err
	}

	return m.run(m.CollectionName, query)
}

// deleteKey removes the key-value from mongoDB
func (m *MongoCache) delete(key string) error {
	query := func(c *mgo.Collection) error {
		err := c.RemoveId(key)
		return err
	}

	return m.run(m.CollectionName, query)
}

func (m *MongoCache) deleteExpiredKeys() error {
	var selector = bson.M{"expireAt": bson.M{
		"$lte": time.Now().UTC(),
	}}

	query := func(c *mgo.Collection) error {
		_, err := c.RemoveAll(selector)
		return err
	}

	return m.run(m.CollectionName, query)
}

func (m *MongoCache) run(collection string, s func(*mgo.Collection) error) error {
	session := m.mongeSession.Copy()
	defer session.Close()

	c := session.DB("").C(collection)
	return s(c)
}
<commit_msg>cache: update func comments<commit_after>package cache

import (
	"time"

	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// Document holds the key-value pair for mongo cache
type Document struct {
	Key      string      `bson:"_id" json:"_id"`
	Value    interface{} `bson:"value" json:"value"`
	ExpireAt time.Time   `bson:"expireAt" json:"expireAt"`
}

// get fetches the key with its given key input
func (m *MongoCache) get(key string) (*Document, error) {
	keyValue := new(Document)

	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{
			"_id": key,
			"expireAt": bson.M{
				"$gt": time.Now().UTC(),
			}}).One(&keyValue)
	}

	err := m.run(m.CollectionName, query)
	if err != nil {
		return nil, err
	}

	return keyValue, nil
}

func (m *MongoCache) set(key string, duration time.Duration, value interface{}) error {
	update := bson.M{
		"_id":      key,
		"value":    value,
		"expireAt": time.Now().Add(duration),
	}

	query := func(c *mgo.Collection) error {
		_, err := c.UpsertId(key, update)
		return err
	}

	return m.run(m.CollectionName, query)
}

// delete removes the key-value from mongoDB
func (m *MongoCache) delete(key string) error {
	query := func(c *mgo.Collection) error {
		err := c.RemoveId(key)
		return err
	}

	return m.run(m.CollectionName, query)
}

func (m *MongoCache) deleteExpiredKeys() error {
	var selector = bson.M{"expireAt": bson.M{
		"$lte": time.Now().UTC(),
	}}

	query := func(c *mgo.Collection) error {
		_, err := c.RemoveAll(selector)
		return err
	}

	return m.run(m.CollectionName, query)
}

func (m *MongoCache) run(collection string, s func(*mgo.Collection) error) error {
	session := m.mongeSession.Copy()
	defer session.Close()

	c := session.DB("").C(collection)
	return s(c)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"syscall"
	"unsafe"
)

// pcStat: page cache status
// Bytes: size of the file (from os.File.Stat())
// Pages: array of booleans: true if cached, false otherwise
type pcStat struct {
	Name     string
	Bytes    int64
	Pages    int
	Cached   int
	Uncached int
	Status   []bool
}

func main() {
	flag.Parse()
	for _, fname := range flag.Args() {
		pcs := getMincore(fname)
		percent := (pcs.Cached / pcs.Pages) * 100
		log.Printf("%s: Size: %d bytes, Pages Cached %d, Uncached: %d, %d%% cached\n",
			pcs.Name, pcs.Bytes, pcs.Cached, pcs.Uncached, percent)
	}
}

func getMincore(fname string) pcStat {
	f, err := os.Open(fname)
	if err != nil {
		log.Fatalf("Could not open file '%s' for read: %s\n", fname, err)
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		log.Fatalf("Could not stat file %s: %s\n", fname, err)
	}
	if fi.Size() == 0 {
		log.Fatalf("%s appears to be 0 bytes in length\n", fname)
	}

	mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_NONE, syscall.MAP_SHARED)
	if err != nil {
		log.Fatalf("Could not mmap file '%s': %s\n", fname, err)
	}
	// TODO: check for MAP_FAILED which is ((void *) -1)
	// but maybe unnecessary since it looks like errno is always set when MAP_FAILED

	// one byte per page, only LSB is used, remainder is reserved and clear
	vecsz := (fi.Size() + int64(os.Getpagesize()) - 1) / int64(os.Getpagesize())
	vec := make([]byte, vecsz)

	mmap_ptr := uintptr(unsafe.Pointer(&mmap[0]))
	size_ptr := uintptr(fi.Size())
	vec_ptr := uintptr(unsafe.Pointer(&vec[0]))
	ret, _, err := syscall.RawSyscall(syscall.SYS_MINCORE, mmap_ptr, size_ptr, vec_ptr)
	if ret != 0 {
		log.Fatalf("syscall SYS_MINCORE failed: %s", err)
	}
	defer syscall.Munmap(mmap)

	pcs := pcStat{fname, fi.Size(), int(vecsz), 0, 0, make([]bool, vecsz)}

	// expose no bitshift only bool
	for i, b := range vec {
		if b%2 == 1 {
			pcs.Status[i] = true
			pcs.Cached++
		} else {
			pcs.Status[i] = false
			pcs.Uncached++
		}
	}

	return pcs
}
<commit_msg>Refactor to support multiple output formats.<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"syscall"
	"unsafe"
)

// pcStat: page cache status
// Bytes: size of the file (from os.File.Stat())
// Pages: array of booleans: true if cached, false otherwise
type pcStat struct {
	Name     string `json:filename` // file name as specified on command line
	Size     int64  `json:size`     // file size in bytes
	Pages    int    `json:pages`    // total memory pages
	Cached   int    `json:cached`   // number of pages that are cached
	Uncached int    `json:uncached` // number of pages that are not cached
	Status   []bool `json:status`   // true for cached page, false otherwise
}

type pcStatList []pcStat

var jsonFlag bool

func init() {
	flag.BoolVar(&jsonFlag, "json", false, "return data in JSON format")
}

func main() {
	flag.Parse()

	stats := make(pcStatList, len(flag.Args()))

	for i, fname := range flag.Args() {
		stats[i] = getMincore(fname)
	}

	if jsonFlag {
		log.Fatal("Not implemented yet.")
	} else {
		stats.formatText()
	}
}

func (stats pcStatList) formatText() {
	// TODO: set a maximum padding length, possibly based on terminal info?
	maxName := 8
	for _, pcs := range stats {
		if len(pcs.Name) > maxName {
			maxName = len(pcs.Name)
		}
	}

	hr := "|--------------------+----------------+------------+-----------+---------|"
	fmt.Println(hr)
	fmt.Println("| Name               | Size           | Pages      | Cached    | Percent |")
	fmt.Println(hr)

	for _, pcs := range stats {
		percent := (pcs.Cached / pcs.Pages) * 100
		fmt.Printf("| %-19s| %-15d| %-11d| %-10d| %-7d |\n", pcs.Name, pcs.Size, pcs.Pages, pcs.Cached, percent)
	}
	fmt.Println(hr)
}

func getMincore(fname string) pcStat {
	f, err := os.Open(fname)
	if err != nil {
		log.Fatalf("Could not open file '%s' for read: %s\n", fname, err)
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		log.Fatalf("Could not stat file %s: %s\n", fname, err)
	}
	if fi.Size() == 0 {
		log.Fatalf("%s appears to be 0 bytes in length\n", fname)
	}

	mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_NONE, syscall.MAP_SHARED)
	if err != nil {
		log.Fatalf("Could not mmap file '%s': %s\n", fname, err)
	}
	// TODO: check for MAP_FAILED which is ((void *) -1)
	// but maybe unnecessary since it looks like errno is always set when MAP_FAILED

	// one byte per page, only LSB is used, remainder is reserved and clear
	vecsz := (fi.Size() + int64(os.Getpagesize()) - 1) / int64(os.Getpagesize())
	vec := make([]byte, vecsz)

	mmap_ptr := uintptr(unsafe.Pointer(&mmap[0]))
	size_ptr := uintptr(fi.Size())
	vec_ptr := uintptr(unsafe.Pointer(&vec[0]))
	ret, _, err := syscall.RawSyscall(syscall.SYS_MINCORE, mmap_ptr, size_ptr, vec_ptr)
	if ret != 0 {
		log.Fatalf("syscall SYS_MINCORE failed: %s", err)
	}
	defer syscall.Munmap(mmap)

	pcs := pcStat{fname, fi.Size(), int(vecsz), 0, 0, make([]bool, vecsz)}

	// expose no bitshift only bool
	for i, b := range vec {
		if b%2 == 1 {
			pcs.Status[i] = true
			pcs.Cached++
		} else {
			pcs.Status[i] = false
			pcs.Uncached++
		}
	}

	return pcs
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package mtail

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"sync"
	"syscall"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/exporter"
	"github.com/google/mtail/metrics"
	"github.com/google/mtail/tailer"
	"github.com/google/mtail/vm"
	"github.com/google/mtail/watcher"
	"github.com/pkg/errors"
	"github.com/spf13/afero"
)

// MtailServer contains the state of the main program object.
type MtailServer struct {
	lines chan *tailer.LogLine // Channel of lines from tailer to VM engine.
	store *metrics.Store       // Metrics storage.

	t *tailer.Tailer     // t tails the watched files and feeds lines to the VMs.
	l *vm.Loader         // l loads programs and manages the VM lifecycle.
	e *exporter.Exporter // e manages the export of metrics from the store.

	webquit   chan struct{} // Channel to signal shutdown from web UI.
	closeOnce sync.Once     // Ensure shutdown happens only once.

	o Options // Options passed in at creation time.
}

// StartTailing constructs a new Tailer and commences sending log lines into
// the lines channel.
func (m *MtailServer) StartTailing() error {
	var err error
	m.t, err = tailer.New(m.lines, m.o.FS, m.o.W)
	if m.o.OneShot {
		if err = m.t.SetOneShot(); err != nil {
			return err
		}
	}
	if err != nil {
		return errors.Wrap(err, "tailer.New")
	}

	for _, pattern := range m.o.LogPathPatterns {
		glog.V(1).Infof("Tail pattern %q", pattern)
		if err = m.t.TailPattern(pattern); err != nil {
			glog.Error(err)
		}
	}
	for _, fd := range m.o.LogFds {
		f := os.NewFile(uintptr(fd), strconv.Itoa(fd))
		if f == nil {
			glog.Errorf("Attempt to reopen fd %q returned nil", fd)
			continue
		}
		if err = m.t.TailHandle(f); err != nil {
			glog.Error(err)
		}
	}
	return nil
}

// InitLoader constructs a new program loader and performs the initial load of program files in the program directory.
func (m *MtailServer) InitLoader() error {
	var err error
	m.l, err = vm.NewLoader(m.o.Progs, m.store, m.lines, vm.Watcher(m.o.W), vm.Filesystem(m.o.FS), vm.OverrideLocation(m.o.OverrideLocation))
	if m.o.CompileOnly {
		m.l.SetOption(vm.CompileOnly)
		if m.o.OneShot {
			m.l.SetOption(vm.ErrorsAbort)
		}
	}
	if m.o.DumpAst {
		m.l.SetOption(vm.DumpAst)
	}
	if m.o.DumpAstTypes {
		m.l.SetOption(vm.DumpAstTypes)
	}
	if m.o.DumpBytecode {
		m.l.SetOption(vm.DumpBytecode)
	}
	if m.o.SyslogUseCurrentYear {
		m.l.SetOption(vm.SyslogUseCurrentYear)
	}
	if m.o.OmitMetricSource {
		m.l.SetOption(vm.OmitMetricSource)
	}
	if err != nil {
		return err
	}
	if m.o.Progs != "" {
		errs := m.l.LoadAllPrograms()
		if errs != nil {
			return errors.Errorf("Compile encountered errors:\n%s", errs)
		}
	}
	return nil
}

const statusTemplate = `

<title>mtail on {{.BindAddress}}</title>

mtail on {{.BindAddress}}

Build: {{.BuildInfo}}

Metrics: json, prometheus, varz

Debug: debug/pprof, debug/vars

`

func (m *MtailServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	t, err := template.New("status").Parse(statusTemplate)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	data := struct {
		BindAddress string
		BuildInfo   string
	}{
		m.o.BindAddress,
		m.o.BuildInfo,
	}
	w.Header().Add("Content-type", "text/html")
	w.WriteHeader(http.StatusFound)
	if err = t.Execute(w, data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	err = m.l.WriteStatusHTML(w)
	if err != nil {
		glog.Warningf("Error while writing loader status: %s", err)
	}
	err = m.t.WriteStatusHTML(w)
	if err != nil {
		glog.Warningf("Error while writing tailer status: %s", err)
	}
}

// Options contains all the parameters necessary for constructing a new MtailServer.
type Options struct {
	BindAddress          string
	Progs                string
	BuildInfo            string
	LogPathPatterns      []string
	LogFds               []int
	OneShot              bool
	CompileOnly          bool
	DumpAst              bool
	DumpAstTypes         bool
	DumpBytecode         bool
	SyslogUseCurrentYear bool
	OmitMetricSource     bool
	OmitProgLabel        bool

	OverrideLocation *time.Location
	Store            *metrics.Store

	W  watcher.Watcher // Not required, will use watcher.LogWatcher if zero.
	FS afero.Fs        // Not required, will use afero.OsFs if zero.
}

// New creates a MtailServer from the supplied Options.
func New(o Options) (*MtailServer, error) {
	store := o.Store
	if store == nil {
		store = metrics.NewStore()
	}
	if o.FS == nil {
		o.FS = &afero.OsFs{}
	}
	if o.W == nil {
		w, err := watcher.NewLogWatcher()
		if err != nil {
			return nil, err
		}
		o.W = w
	}
	m := &MtailServer{
		lines:   make(chan *tailer.LogLine),
		store:   store,
		webquit: make(chan struct{}),
		o:       o}

	err := m.InitLoader()
	if err != nil {
		return nil, err
	}

	m.e, err = exporter.New(exporter.Options{Store: m.store, OmitProgLabel: o.OmitProgLabel})
	if err != nil {
		return nil, err
	}

	return m, nil
}

// WriteMetrics dumps the current state of the metrics store in JSON format to
// the io.Writer.
func (m *MtailServer) WriteMetrics(w io.Writer) error {
	m.store.RLock()
	b, err := json.MarshalIndent(m.store.Metrics, "", "  ")
	m.store.RUnlock()
	if err != nil {
		return errors.Wrap(err, "failed to marshal metrics into json")
	}
	_, err = w.Write(b)
	return err
}

// Serve begins the webserver and awaits a shutdown instruction.
func (m *MtailServer) Serve() {
	http.Handle("/", m)
	http.HandleFunc("/json", http.HandlerFunc(m.e.HandleJSON))
	http.HandleFunc("/metrics", http.HandlerFunc(m.e.HandlePrometheusMetrics))
	http.HandleFunc("/varz", http.HandlerFunc(m.e.HandleVarz))
	http.HandleFunc("/quitquitquit", http.HandlerFunc(m.handleQuit))
	m.e.StartMetricPush()

	go func() {
		glog.Infof("Listening on port %s", m.o.BindAddress)
		err := http.ListenAndServe(m.o.BindAddress, nil)
		if err != nil {
			glog.Exit(err)
		}
	}()
	m.WaitForShutdown()
}

func (m *MtailServer) handleQuit(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Header().Add("Allow", "POST")
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	fmt.Fprintf(w, "Exiting...")
	close(m.webquit)
}

// WaitForShutdown handles shutdown requests from the system or the UI.
func (m *MtailServer) WaitForShutdown() {
	n := make(chan os.Signal, 1)
	signal.Notify(n, os.Interrupt, syscall.SIGTERM)
	select {
	case <-n:
		glog.Info("Received SIGTERM, exiting...")
	case <-m.webquit:
		glog.Info("Received Quit from HTTP, exiting...")
	}
	if err := m.Close(); err != nil {
		glog.Warning(err)
	}
}

// Close handles the graceful shutdown of this mtail instance, ensuring that it only occurs once.
func (m *MtailServer) Close() error {
	m.closeOnce.Do(func() {
		glog.Info("Shutdown requested.")
		// If we have a tailer (i.e. not in test) then signal the tailer to
		// shut down, which will cause the watcher to shut down and for the
		// lines channel to close, causing the loader to start shutdown.
		if m.t != nil {
			err := m.t.Close()
			if err != nil {
				glog.Infof("tailer close failed: %s", err)
			}
		} else {
			// Without a tailer, MtailServer has ownership of the lines channel.
			glog.V(2).Info("No tailer, closing lines channel directly.")
			close(m.lines)
		}
		// If we have a loader, wait for it to signal that it has completed shutdown.
		if m.l != nil {
			<-m.l.VMsDone
		} else {
			glog.V(2).Info("No loader, so not waiting for loader shutdown.")
		}
		glog.Info("All done.")
	})
	return nil
}

// Run starts MtailServer's primary function, in which it watches the log
// files for changes and sends any new lines found into the lines channel for
// pick up by the virtual machines. If OneShot mode is enabled, it will exit.
func (m *MtailServer) Run() error {
	if m.o.CompileOnly {
		return nil
	}
	err := m.StartTailing()
	if err != nil {
		glog.Exitf("tailing failed: %s", err)
	}
	if m.o.OneShot {
		err := m.Close()
		if err != nil {
			return err
		}
		fmt.Printf("Metrics store:")
		if err := m.WriteMetrics(os.Stdout); err != nil {
			return err
		}
	} else {
		m.Serve()
	}
	return nil
}
<commit_msg>Simplify consrtuction of a program loader options.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package mtail

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"sync"
	"syscall"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/exporter"
	"github.com/google/mtail/metrics"
	"github.com/google/mtail/tailer"
	"github.com/google/mtail/vm"
	"github.com/google/mtail/watcher"
	"github.com/pkg/errors"
	"github.com/spf13/afero"
)

// MtailServer contains the state of the main program object.
type MtailServer struct {
	lines chan *tailer.LogLine // Channel of lines from tailer to VM engine.
	store *metrics.Store       // Metrics storage.

	t *tailer.Tailer     // t tails the watched files and feeds lines to the VMs.
	l *vm.Loader         // l loads programs and manages the VM lifecycle.
	e *exporter.Exporter // e manages the export of metrics from the store.

	webquit   chan struct{} // Channel to signal shutdown from web UI.
	closeOnce sync.Once     // Ensure shutdown happens only once.

	o Options // Options passed in at creation time.
}

// StartTailing constructs a new Tailer and commences sending log lines into
// the lines channel.
func (m *MtailServer) StartTailing() error {
	var err error
	m.t, err = tailer.New(m.lines, m.o.FS, m.o.W)
	if m.o.OneShot {
		if err = m.t.SetOneShot(); err != nil {
			return err
		}
	}
	if err != nil {
		return errors.Wrap(err, "tailer.New")
	}

	for _, pattern := range m.o.LogPathPatterns {
		glog.V(1).Infof("Tail pattern %q", pattern)
		if err = m.t.TailPattern(pattern); err != nil {
			glog.Error(err)
		}
	}
	for _, fd := range m.o.LogFds {
		f := os.NewFile(uintptr(fd), strconv.Itoa(fd))
		if f == nil {
			glog.Errorf("Attempt to reopen fd %q returned nil", fd)
			continue
		}
		if err = m.t.TailHandle(f); err != nil {
			glog.Error(err)
		}
	}
	return nil
}

// InitLoader constructs a new program loader and performs the initial load of program files in the program directory.
func (m *MtailServer) InitLoader() error {
	opts := []func(*vm.Loader) error{
		vm.Watcher(m.o.W),
		vm.Filesystem(m.o.FS),
	}
	if m.o.CompileOnly {
		opts = append(opts, vm.CompileOnly)
		if m.o.OneShot {
			opts = append(opts, vm.ErrorsAbort)
		}
	}
	if m.o.DumpAst {
		opts = append(opts, vm.DumpAst)
	}
	if m.o.DumpAstTypes {
		opts = append(opts, vm.DumpAstTypes)
	}
	if m.o.DumpBytecode {
		opts = append(opts, vm.DumpBytecode)
	}
	if m.o.SyslogUseCurrentYear {
		opts = append(opts, vm.SyslogUseCurrentYear)
	}
	if m.o.OmitMetricSource {
		opts = append(opts, vm.OmitMetricSource)
	}
	if m.o.OverrideLocation != nil {
		opts = append(opts, vm.OverrideLocation(m.o.OverrideLocation))
	}
	var err error
	m.l, err = vm.NewLoader(m.o.Progs, m.store, m.lines, opts...)
	if err != nil {
		return err
	}
	if m.o.Progs == "" {
		return nil
	}
	if errs := m.l.LoadAllPrograms(); errs != nil {
		return errors.Errorf("Compile encountered errors:\n%s", errs)
	}
	return nil
}

const statusTemplate = `

<title>mtail on {{.BindAddress}}</title>

mtail on {{.BindAddress}}

Build: {{.BuildInfo}}

Metrics: json, prometheus, varz

Debug: debug/pprof, debug/vars

`

func (m *MtailServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	t, err := template.New("status").Parse(statusTemplate)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	data := struct {
		BindAddress string
		BuildInfo   string
	}{
		m.o.BindAddress,
		m.o.BuildInfo,
	}
	w.Header().Add("Content-type", "text/html")
	w.WriteHeader(http.StatusFound)
	if err = t.Execute(w, data); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	err = m.l.WriteStatusHTML(w)
	if err != nil {
		glog.Warningf("Error while writing loader status: %s", err)
	}
	err = m.t.WriteStatusHTML(w)
	if err != nil {
		glog.Warningf("Error while writing tailer status: %s", err)
	}
}

// Options contains all the parameters necessary for constructing a new MtailServer.
type Options struct {
	BindAddress          string
	Progs                string
	BuildInfo            string
	LogPathPatterns      []string
	LogFds               []int
	OneShot              bool
	CompileOnly          bool
	DumpAst              bool
	DumpAstTypes         bool
	DumpBytecode         bool
	SyslogUseCurrentYear bool
	OmitMetricSource     bool
	OmitProgLabel        bool

	OverrideLocation *time.Location
	Store            *metrics.Store

	W  watcher.Watcher // Not required, will use watcher.LogWatcher if zero.
	FS afero.Fs        // Not required, will use afero.OsFs if zero.
}

// New creates a MtailServer from the supplied Options.
func New(o Options) (*MtailServer, error) {
	store := o.Store
	if store == nil {
		store = metrics.NewStore()
	}
	if o.FS == nil {
		o.FS = &afero.OsFs{}
	}
	if o.W == nil {
		w, err := watcher.NewLogWatcher()
		if err != nil {
			return nil, err
		}
		o.W = w
	}
	m := &MtailServer{
		lines:   make(chan *tailer.LogLine),
		store:   store,
		webquit: make(chan struct{}),
		o:       o}

	err := m.InitLoader()
	if err != nil {
		return nil, err
	}

	m.e, err = exporter.New(exporter.Options{Store: m.store, OmitProgLabel: o.OmitProgLabel})
	if err != nil {
		return nil, err
	}

	return m, nil
}

// WriteMetrics dumps the current state of the metrics store in JSON format to
// the io.Writer.
func (m *MtailServer) WriteMetrics(w io.Writer) error {
	m.store.RLock()
	b, err := json.MarshalIndent(m.store.Metrics, "", "  ")
	m.store.RUnlock()
	if err != nil {
		return errors.Wrap(err, "failed to marshal metrics into json")
	}
	_, err = w.Write(b)
	return err
}

// Serve begins the webserver and awaits a shutdown instruction.
func (m *MtailServer) Serve() {
	http.Handle("/", m)
	http.HandleFunc("/json", http.HandlerFunc(m.e.HandleJSON))
	http.HandleFunc("/metrics", http.HandlerFunc(m.e.HandlePrometheusMetrics))
	http.HandleFunc("/varz", http.HandlerFunc(m.e.HandleVarz))
	http.HandleFunc("/quitquitquit", http.HandlerFunc(m.handleQuit))
	m.e.StartMetricPush()

	go func() {
		glog.Infof("Listening on port %s", m.o.BindAddress)
		err := http.ListenAndServe(m.o.BindAddress, nil)
		if err != nil {
			glog.Exit(err)
		}
	}()
	m.WaitForShutdown()
}

func (m *MtailServer) handleQuit(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Header().Add("Allow", "POST")
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	fmt.Fprintf(w, "Exiting...")
	close(m.webquit)
}

// WaitForShutdown handles shutdown requests from the system or the UI.
func (m *MtailServer) WaitForShutdown() {
	n := make(chan os.Signal, 1)
	signal.Notify(n, os.Interrupt, syscall.SIGTERM)
	select {
	case <-n:
		glog.Info("Received SIGTERM, exiting...")
	case <-m.webquit:
		glog.Info("Received Quit from HTTP, exiting...")
	}
	if err := m.Close(); err != nil {
		glog.Warning(err)
	}
}

// Close handles the graceful shutdown of this mtail instance, ensuring that it only occurs once.
func (m *MtailServer) Close() error {
	m.closeOnce.Do(func() {
		glog.Info("Shutdown requested.")
		// If we have a tailer (i.e. not in test) then signal the tailer to
		// shut down, which will cause the watcher to shut down and for the
		// lines channel to close, causing the loader to start shutdown.
		if m.t != nil {
			err := m.t.Close()
			if err != nil {
				glog.Infof("tailer close failed: %s", err)
			}
		} else {
			// Without a tailer, MtailServer has ownership of the lines channel.
			glog.V(2).Info("No tailer, closing lines channel directly.")
			close(m.lines)
		}
		// If we have a loader, wait for it to signal that it has completed shutdown.
		if m.l != nil {
			<-m.l.VMsDone
		} else {
			glog.V(2).Info("No loader, so not waiting for loader shutdown.")
		}
		glog.Info("All done.")
	})
	return nil
}

// Run starts MtailServer's primary function, in which it watches the log
// files for changes and sends any new lines found into the lines channel for
// pick up by the virtual machines. If OneShot mode is enabled, it will exit.
func (m *MtailServer) Run() error {
	if m.o.CompileOnly {
		return nil
	}
	err := m.StartTailing()
	if err != nil {
		glog.Exitf("tailing failed: %s", err)
	}
	if m.o.OneShot {
		err := m.Close()
		if err != nil {
			return err
		}
		fmt.Printf("Metrics store:")
		if err := m.WriteMetrics(os.Stdout); err != nil {
			return err
		}
	} else {
		m.Serve()
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package mtail

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"sync"
	"syscall"

	"github.com/golang/glog"
	"github.com/google/mtail/exporter"
	"github.com/google/mtail/metrics"
	"github.com/google/mtail/tailer"
	"github.com/google/mtail/vm"
	"github.com/google/mtail/watcher"
	"github.com/spf13/afero"
)

// Mtail contains the state of the main program object.
type Mtail struct {
	lines chan string   // Channel of lines from tailer to VM engine.
	store metrics.Store // Metrics storage.

	t *tailer.Tailer     // t tails the watched files and feeds lines to the VMs.
	l *vm.Loader         // l loads programs and manages the VM lifecycle.
	e *exporter.Exporter // e manages the export of metrics from the store.

	webquit   chan struct{} // Channel to signal shutdown from web UI.
	closeOnce sync.Once     // Ensure shutdown happens only once.

	o Options // Options passed in at creation time.
}

// OneShot reads the contents of a log file into the lines channel from start to finish, terminating the program at the end.
func (m *Mtail) OneShot(logfile string) error {
	l, err := os.Open(logfile)
	if err != nil {
		return fmt.Errorf("failed to open log file %q: %s", logfile, err)
	}
	defer l.Close()

	r := bufio.NewReader(l)

	for {
		line, err := r.ReadString('\n')
		switch {
		case err == io.EOF:
			return nil
		case err != nil:
			return fmt.Errorf("failed to read from %q: %s", logfile, err)
		default:
			m.lines <- line
		}
	}
}

// StartTailing constructs a new Tailer and commences sending log lines into
// the lines channel.
func (m *Mtail) StartTailing() error {
	o := tailer.Options{Lines: m.lines, W: m.o.W, FS: m.o.FS}
	var err error
	m.t, err = tailer.New(o)
	if err != nil {
		return fmt.Errorf("couldn't create a log tailer: %s", err)
	}

	for _, pathname := range m.o.LogPaths {
		m.t.Tail(pathname)
	}
	for _, fd := range m.o.LogFds {
		f := os.NewFile(uintptr(fd), strconv.Itoa(fd))
		m.t.TailFile(f)
	}
	return nil
}

// InitLoader constructs a new program loader and performs the inital load of program files in the program directory.
func (m *Mtail) InitLoader() error {
	o := vm.LoaderOptions{Store: &m.store, Lines: m.lines, CompileOnly: m.o.CompileOnly, DumpBytecode: m.o.DumpBytecode, SyslogUseCurrentYear: m.o.SyslogUseCurrentYear, W: m.o.W, FS: m.o.FS}
	var err error
	m.l, err = vm.NewLoader(o)
	if err != nil {
		return err
	}
	if m.o.Progs != "" {
		errors := m.l.LoadProgs(m.o.Progs)
		if m.o.CompileOnly || m.o.DumpBytecode {
			return fmt.Errorf("Compile encountered errors:\n%s", errors)
		}
	}
	return nil
}

func (m *Mtail) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(200)
	w.Write([]byte(`<a href="/json">json</a>, <a href="/metrics">prometheus metrics</a>, <a href="/varz">varz</a>`))
}

// Options contains all the parameters necessary for constructing a new Mtail.
type Options struct {
	Progs                string
	LogPaths             []string
	LogFds               []int
	Port                 string
	OneShot              bool
	CompileOnly          bool
	DumpBytecode         bool
	SyslogUseCurrentYear bool

	W  watcher.Watcher // Not required, will use watcher.LogWatcher if zero.
	FS afero.Fs        // Not required, will use afero.OsFs if zero.
}

// New creates an Mtail from the supplied Options.
func New(o Options) (*Mtail, error) {
	m := &Mtail{
		lines:   make(chan string),
		webquit: make(chan struct{}),
		o:       o}

	err := m.InitLoader()
	if err != nil {
		return nil, err
	}

	m.e, err = exporter.New(exporter.Options{Store: &m.store})
	if err != nil {
		return nil, err
	}

	return m, nil
}

// WriteMetrics dumps the current state of the metrics store in JSON format to
// the io.Writer.
func (m *Mtail) WriteMetrics(w io.Writer) error {
	m.store.RLock()
	b, err := json.MarshalIndent(m.store.Metrics, "", "  ")
	m.store.RUnlock()
	if err != nil {
		return fmt.Errorf("failed to marshal metrics into json: %s", err)
	}
	w.Write(b)
	return nil
}

// RunOneShot performs the work of the one_shot commandline flag; after compiling programs mtail will read all of the log files in full, once, dump the metric results at the end, and then exit.
func (m *Mtail) RunOneShot() {
	for _, pathname := range m.o.LogPaths {
		err := m.OneShot(pathname)
		if err != nil {
			glog.Exitf("Failed one shot mode for %q: %s\n", pathname, err)
		}
	}
	if err := m.WriteMetrics(os.Stdout); err != nil {
		glog.Exit(err)
	}
	m.e.WriteMetrics()
	m.Close()
}

// Serve begins the long-running mode of mtail, in which it watches the log
// files for changes and sends any new lines found into the lines channel for
// pick up by the virtual machines.  It will continue to do so until it is
// signalled to exit.
func (m *Mtail) Serve() {
	err := m.StartTailing()
	if err != nil {
		glog.Exitf("tailing failed: %s", err)
	}

	http.Handle("/", m)
	http.HandleFunc("/json", http.HandlerFunc(m.e.HandleJSON))
	http.HandleFunc("/metrics", http.HandlerFunc(m.e.HandlePrometheusMetrics))
	http.HandleFunc("/varz", http.HandlerFunc(m.e.HandleVarz))
	http.HandleFunc("/quitquitquit", http.HandlerFunc(m.handleQuit))
	m.e.StartMetricPush()

	go func() {
		glog.Infof("Listening on port %s", m.o.Port)
		err := http.ListenAndServe(":"+m.o.Port, nil)
		if err != nil {
			glog.Exit(err)
		}
	}()
	m.shutdownHandler()
}

func (m *Mtail) handleQuit(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Header().Add("Allow", "POST")
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	fmt.Fprintf(w, "Exiting...")
	close(m.webquit)
}

// shutdownHandler handles external shutdown request events.
func (m *Mtail) shutdownHandler() {
	n := make(chan os.Signal)
	signal.Notify(n, os.Interrupt, syscall.SIGTERM)
	select {
	case <-n:
		glog.Info("Received SIGTERM, exiting...")
	case <-m.webquit:
		glog.Info("Received Quit from UI, exiting...")
	}
	m.Close()
}

// Close handles the graceful shutdown of this mtail instance, ensuring that it only occurs once.
func (m *Mtail) Close() {
	m.closeOnce.Do(func() {
		glog.Info("Shutdown requested.")
		if m.t != nil {
			m.t.Close()
		} else {
			glog.Info("Closing lines channel.")
			close(m.lines)
		}
		if m.l != nil {
			<-m.l.VMsDone
		}
		glog.Info("All done.")
	})
}

// Run starts Mtail in the configuration supplied in Options at creation.
func (m *Mtail) Run() {
	if m.o.OneShot {
		m.RunOneShot()
	} else {
		m.Serve()
	}
}
<commit_msg>Put benchmark results into the oneshot output.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package mtail

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"sync"
	"syscall"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/exporter"
	"github.com/google/mtail/metrics"
	"github.com/google/mtail/tailer"
	"github.com/google/mtail/vm"
	"github.com/google/mtail/watcher"
	"github.com/spf13/afero"
)

// Mtail contains the state of the main program object.
type Mtail struct {
	lines chan string   // Channel of lines from tailer to VM engine.
	store metrics.Store // Metrics storage.

	t *tailer.Tailer     // t tails the watched files and feeds lines to the VMs.
	l *vm.Loader         // l loads programs and manages the VM lifecycle.
	e *exporter.Exporter // e manages the export of metrics from the store.

	webquit   chan struct{} // Channel to signal shutdown from web UI.
	closeOnce sync.Once     // Ensure shutdown happens only once.

	o Options // Options passed in at creation time.
}

// OneShot reads the contents of a log file into the lines channel from start to finish, terminating the program at the end.
func (m *Mtail) OneShot(logfile string) error {
	glog.Infof("Oneshot %q", logfile)
	l, err := os.Open(logfile)
	if err != nil {
		return fmt.Errorf("failed to open log file %q: %s", logfile, err)
	}
	defer l.Close()

	r := bufio.NewReader(l)

	start := time.Now()

Loop:
	for {
		line, err := r.ReadString('\n')
		switch {
		case err == io.EOF:
			m.lines <- line
			break Loop
		case err != nil:
			return fmt.Errorf("failed to read from %q: %s", logfile, err)
		default:
			m.lines <- line
		}
	}
	duration := time.Since(start)
	count, err := strconv.Atoi(vm.LineCount.String())
	if err != nil {
		return err
	}
	µsPerL := float64(duration.Nanoseconds()) / (float64(count) * 1000)
	fmt.Printf("%s: %d lines, %6.3f µs/line\n", logfile, count, µsPerL)
	return nil
}

// StartTailing constructs a new Tailer and commences sending log lines into
// the lines channel.
func (m *Mtail) StartTailing() error {
	o := tailer.Options{Lines: m.lines, W: m.o.W, FS: m.o.FS}
	var err error
	m.t, err = tailer.New(o)
	if err != nil {
		return fmt.Errorf("couldn't create a log tailer: %s", err)
	}

	for _, pathname := range m.o.LogPaths {
		m.t.Tail(pathname)
	}
	for _, fd := range m.o.LogFds {
		f := os.NewFile(uintptr(fd), strconv.Itoa(fd))
		m.t.TailFile(f)
	}
	return nil
}

// InitLoader constructs a new program loader and performs the inital load of program files in the program directory.
func (m *Mtail) InitLoader() error {
	o := vm.LoaderOptions{Store: &m.store, Lines: m.lines, CompileOnly: m.o.CompileOnly, DumpBytecode: m.o.DumpBytecode, SyslogUseCurrentYear: m.o.SyslogUseCurrentYear, W: m.o.W, FS: m.o.FS}
	var err error
	m.l, err = vm.NewLoader(o)
	if err != nil {
		return err
	}
	if m.o.Progs != "" {
		errors := m.l.LoadProgs(m.o.Progs)
		if m.o.CompileOnly || m.o.DumpBytecode {
			return fmt.Errorf("Compile encountered errors:\n%s", errors)
		}
	}
	return nil
}

func (m *Mtail) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(200)
	w.Write([]byte(`<a href="/json">json</a>, <a href="/metrics">prometheus metrics</a>, <a href="/varz">varz</a>`))
}

// Options contains all the parameters necessary for constructing a new Mtail.
type Options struct {
	Progs                string
	LogPaths             []string
	LogFds               []int
	Port                 string
	OneShot              bool
	CompileOnly          bool
	DumpBytecode         bool
	SyslogUseCurrentYear bool

	W  watcher.Watcher // Not required, will use watcher.LogWatcher if zero.
	FS afero.Fs        // Not required, will use afero.OsFs if zero.
}

// New creates an Mtail from the supplied Options.
func New(o Options) (*Mtail, error) {
	m := &Mtail{
		lines:   make(chan string),
		webquit: make(chan struct{}),
		o:       o}

	err := m.InitLoader()
	if err != nil {
		return nil, err
	}

	m.e, err = exporter.New(exporter.Options{Store: &m.store})
	if err != nil {
		return nil, err
	}

	return m, nil
}

// WriteMetrics dumps the current state of the metrics store in JSON format to
// the io.Writer.
func (m *Mtail) WriteMetrics(w io.Writer) error {
	m.store.RLock()
	b, err := json.MarshalIndent(m.store.Metrics, "", "  ")
	m.store.RUnlock()
	if err != nil {
		return fmt.Errorf("failed to marshal metrics into json: %s", err)
	}
	w.Write(b)
	return nil
}

// RunOneShot performs the work of the one_shot commandline flag; after compiling programs mtail will read all of the log files in full, once, dump the metric results at the end, and then exit.
func (m *Mtail) RunOneShot() {
	fmt.Println("Oneshot results:")
	for _, pathname := range m.o.LogPaths {
		err := m.OneShot(pathname)
		if err != nil {
			glog.Exitf("Failed one shot mode for %q: %s\n", pathname, err)
		}
	}
	// if err := m.WriteMetrics(os.Stdout); err != nil {
	// 	glog.Exit(err)
	// }
	// m.e.WriteMetrics()
	m.Close()
}

// Serve begins the long-running mode of mtail, in which it watches the log
// files for changes and sends any new lines found into the lines channel for
// pick up by the virtual machines.  It will continue to do so until it is
// signalled to exit.
func (m *Mtail) Serve() {
	err := m.StartTailing()
	if err != nil {
		glog.Exitf("tailing failed: %s", err)
	}

	http.Handle("/", m)
	http.HandleFunc("/json", http.HandlerFunc(m.e.HandleJSON))
	http.HandleFunc("/metrics", http.HandlerFunc(m.e.HandlePrometheusMetrics))
	http.HandleFunc("/varz", http.HandlerFunc(m.e.HandleVarz))
	http.HandleFunc("/quitquitquit", http.HandlerFunc(m.handleQuit))
	m.e.StartMetricPush()

	go func() {
		glog.Infof("Listening on port %s", m.o.Port)
		err := http.ListenAndServe(":"+m.o.Port, nil)
		if err != nil {
			glog.Exit(err)
		}
	}()
	m.shutdownHandler()
}

func (m *Mtail) handleQuit(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Header().Add("Allow", "POST")
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	fmt.Fprintf(w, "Exiting...")
	close(m.webquit)
}

// shutdownHandler handles external shutdown request events.
func (m *Mtail) shutdownHandler() {
	n := make(chan os.Signal)
	signal.Notify(n, os.Interrupt, syscall.SIGTERM)
	select {
	case <-n:
		glog.Info("Received SIGTERM, exiting...")
	case <-m.webquit:
		glog.Info("Received Quit from UI, exiting...")
	}
	m.Close()
}

// Close handles the graceful shutdown of this mtail instance, ensuring that it only occurs once.
func (m *Mtail) Close() {
	m.closeOnce.Do(func() {
		glog.Info("Shutdown requested.")
		if m.t != nil {
			m.t.Close()
		} else {
			glog.Info("Closing lines channel.")
			close(m.lines)
		}
		if m.l != nil {
			<-m.l.VMsDone
		}
		glog.Info("All done.")
	})
}

// Run starts Mtail in the configuration supplied in Options at creation.
func (m *Mtail) Run() {
	if m.o.OneShot {
		m.RunOneShot()
	} else {
		m.Serve()
	}
}
<|endoftext|>


<commit_before>package bamstats

import (
	"github.com/biogo/hts/sam"
	"github.com/brentp/irelate/parsers"
)

func isSplit(r *sam.Record) bool {
	for _, op := range r.Cigar {
		if op.Type() == sam.CigarSkipped {
			return true
		}
	}
	return false
}

func isSplit1(r *parsers.Bam) bool {
	for _, op := range r.Cigar {
		if op.Type() == sam.CigarSkipped {
			return true
		}
	}
	return false
}

func isPrimary(r *sam.Record) bool {
	return r.Flags&sam.Secondary == 0
}

func isUnmapped(r *sam.Record) bool {
	return r.Flags&sam.Unmapped == sam.Unmapped
}

func isPaired(r *sam.Record) bool {
	return r.Flags&sam.Paired == sam.Paired
}

func isProperlyPaired(r *sam.Record) bool {
	return r.Flags&sam.ProperPair == sam.ProperPair
}

func isRead1(r *sam.Record) bool {
	return r.Flags&sam.Read1 == sam.Read1
}

func isRead2(r *sam.Record) bool {
	return r.Flags&sam.Read2 == sam.Read2
}

func hasMateUnmapped(r *sam.Record) bool {
	return r.Flags&sam.MateUnmapped == sam.MateUnmapped
}

func getBlocks(r *sam.Record) []location {
	blocks := make([]location, 0, 10)
	ref := r.Ref.Name()
	start := r.Pos
	end := r.Pos
	var con sam.Consume
	for _, co := range r.Cigar {
		coType := co.Type()
		if coType == sam.CigarSkipped {
			blocks = append(blocks, location{ref, start, end})
			start = end + co.Len()
			end = start
			continue
		}
		con = co.Type().Consumes()
		end += co.Len() * con.Reference
		// if con.Query != 0 {
		// 	end = max(end, start)
		// }
	}
	blocks = append(blocks, location{ref, start, end})
	return blocks
}
<commit_msg>sam.go - Add functions to check for first read in valid pair<commit_after>package bamstats

import (
	"github.com/biogo/hts/sam"
	"github.com/brentp/irelate/parsers"
)

func isSplit(r *sam.Record) bool {
	for _, op := range r.Cigar {
		if op.Type() == sam.CigarSkipped {
			return true
		}
	}
	return false
}

func isSplit1(r *parsers.Bam) bool {
	for _, op := range r.Cigar {
		if op.Type() == sam.CigarSkipped {
			return true
		}
	}
	return false
}

func isPrimary(r *sam.Record) bool {
	return r.Flags&sam.Secondary == 0
}

func isUnmapped(r *sam.Record) bool {
	return r.Flags&sam.Unmapped == sam.Unmapped
}

func isPaired(r *sam.Record) bool {
	return r.Flags&sam.Paired == sam.Paired
}

func isProperlyPaired(r *sam.Record) bool {
	return r.Flags&sam.ProperPair == sam.ProperPair
}

func isRead1(r *sam.Record) bool {
	return r.Flags&sam.Read1 == sam.Read1
}

func isRead2(r *sam.Record) bool {
	return r.Flags&sam.Read2 == sam.Read2
}

func hasMateUnmapped(r *sam.Record) bool {
	return r.Flags&sam.MateUnmapped == sam.MateUnmapped
}

func isFirstOfValidPair(r *sam.Record) bool {
	return isPaired(r) && isRead1(r) && isProperlyPaired(r) && !hasMateUnmapped(r)
}

func getBlocks(r *sam.Record) []location {
	blocks := make([]location, 0, 10)
	ref := r.Ref.Name()
	start := r.Pos
	end := r.Pos
	var con sam.Consume
	for _, co := range r.Cigar {
		coType := co.Type()
		if coType == sam.CigarSkipped {
			blocks = append(blocks, location{ref, start, end})
			start = end + co.Len()
			end = start
			continue
		}
		con = co.Type().Consumes()
		end += co.Len() * con.Reference
		// if con.Query != 0 {
		// 	end = max(end, start)
		// }
	}
	blocks = append(blocks, location{ref, start, end})
	return blocks
}
<|endoftext|>


<commit_before>/*
Open Source Initiative OSI - The MIT License (MIT):Licensing

The MIT License (MIT)
Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com)

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

// Package mapset implements a simple and generic set collection.
// Items stored within it are unordered and unique
// It supports typical set operations: membership testing, intersection, union, difference and symmetric difference

package mapset

import (
	"fmt"
	"strings"
)

// The primary type that represents a set
type Set map[interface{}]struct{}

// Creates and returns a reference to an empty set.
func NewSet() Set {
	return make(Set)
}

// Creates and returns a reference to a set from an existing slice
func NewSetFromSlice(s []interface{}) Set {
	a := NewSet()
	for _, item := range s {
		a.Add(item)
	}
	return a
}

// Adds an item to the current set if it doesn't already exist in the set.
func (set Set) Add(i interface{}) bool {
	_, found := set[i]
	set[i] = struct{}{}
	return !found //False if it existed already
}

// Determines if a given item is already in the set.
func (set Set) Contains(i interface{}) bool {
	_, found := set[i]
	return found
}

// Determines if the given items are all in the set
func (set Set) ContainsAll(i ...interface{}) bool {
	allSet := NewSetFromSlice(i)

	if allSet.IsSubset(set) {

		return true
	}

	return false
}

// Determines if every item in the other set is in this set.
func (set Set) IsSubset(other Set) bool {
	for key := range set {
		if !other.Contains(key) {
			return false
		}
	}
	return true
}

// Determines if every item of this set is in the other set.
func (set Set) IsSuperset(other Set) bool {
	return other.IsSubset(set)
}

// Returns a new set with all items in both sets.
func (set Set) Union(other Set) Set {
	unionedSet := NewSet()

	for key, _ := range set {
		unionedSet.Add(key)
	}
	for key, _ := range other {
		unionedSet.Add(key)
	}
	return unionedSet
}

// Returns a new set with items that exist only in both sets.
func (set Set) Intersect(other Set) Set {
	intersectedSet := NewSet()
	var smallerSet Set = nil
	var largerSet Set = nil

	//figure out the smaller of the two sets and loop on that one as an optimization.
	if set.Size() < other.Size() {
		smallerSet = set
		largerSet = other
	} else {
		smallerSet = other
		largerSet = set
	}

	for key, _ := range smallerSet {
		if largerSet.Contains(key) {
			intersectedSet.Add(key)
		}
	}
	return intersectedSet
}

// Returns a new set with items in the current set but not in the other set
func (set Set) Difference(other Set) Set {
	differencedSet := NewSet()

	for key, _ := range set {
		if !other.Contains(key) {
			differencedSet.Add(key)
		}
	}

	return differencedSet
}

// Returns a new set with items in the current set or the other set but not in both.
func (set Set) SymmetricDifference(other Set) Set {
	aDiff := set.Difference(other)
	bDiff := other.Difference(set)

	symDifferencedSet := aDiff.Union(bDiff)

	return symDifferencedSet
}

// Clears the entire set to be the empty set.
func (set *Set) Clear() {
	*set = make(map[interface{}]struct{})
}

// Allows the removal of a single item in the set.
func (set Set) Remove(i interface{}) {
	delete(set, i)
}

// Size returns the how many items are currently in the set.
func (set Set) Size() int {
	return len(set)
}

// Equal determines if two sets are equal to each other.
// If they both are the same size and have the same items they are considered equal.
// Order of items is not relevent for sets to be equal.
func (set Set) Equal(other Set) bool {
	if set.Size() != other.Size() {
		return false
	} else {
		for elem, _ := range set {
			if !other.Contains(elem) {
				return false
			}
		}
		return true
	}
}

// Provides a convenient string representation of the current state of the set.
func (set Set) String() string {
	items := make([]string, 0, len(set))

	for key := range set {
		items = append(items, fmt.Sprintf("%v", key))
	}
	return fmt.Sprintf("Set{%s}", strings.Join(items, ", "))
}
<commit_msg>Another tiny stylistic change. Sets have elements, not keys.<commit_after>/*
Open Source Initiative OSI - The MIT License (MIT):Licensing

The MIT License (MIT)
Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com)

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

// Package mapset implements a simple and generic set collection.
// Items stored within it are unordered and unique
// It supports typical set operations: membership testing, intersection, union, difference and symmetric difference

package mapset

import (
	"fmt"
	"strings"
)

// The primary type that represents a set
type Set map[interface{}]struct{}

// Creates and returns a reference to an empty set.
func NewSet() Set {
	return make(Set)
}

// Creates and returns a reference to a set from an existing slice
func NewSetFromSlice(s []interface{}) Set {
	a := NewSet()
	for _, item := range s {
		a.Add(item)
	}
	return a
}

// Adds an item to the current set if it doesn't already exist in the set.
func (set Set) Add(i interface{}) bool {
	_, found := set[i]
	set[i] = struct{}{}
	return !found //False if it existed already
}

// Determines if a given item is already in the set.
func (set Set) Contains(i interface{}) bool {
	_, found := set[i]
	return found
}

// Determines if the given items are all in the set
func (set Set) ContainsAll(i ...interface{}) bool {
	allSet := NewSetFromSlice(i)

	if allSet.IsSubset(set) {

		return true
	}

	return false
}

// Determines if every item in the other set is in this set.
func (set Set) IsSubset(other Set) bool {
	for elem := range set {
		if !other.Contains(elem) {
			return false
		}
	}
	return true
}

// Determines if every item of this set is in the other set.
func (set Set) IsSuperset(other Set) bool {
	return other.IsSubset(set)
}

// Returns a new set with all items in both sets.
func (set Set) Union(other Set) Set {
	unionedSet := NewSet()

	for key, _ := range set {
		unionedSet.Add(key)
	}
	for key, _ := range other {
		unionedSet.Add(key)
	}
	return unionedSet
}

// Returns a new set with items that exist only in both sets.
func (set Set) Intersect(other Set) Set {
	intersectedSet := NewSet()
	var smallerSet Set = nil
	var largerSet Set = nil

	//figure out the smaller of the two sets and loop on that one as an optimization.
	if set.Size() < other.Size() {
		smallerSet = set
		largerSet = other
	} else {
		smallerSet = other
		largerSet = set
	}

	for key, _ := range smallerSet {
		if largerSet.Contains(key) {
			intersectedSet.Add(key)
		}
	}
	return intersectedSet
}

// Returns a new set with items in the current set but not in the other set
func (set Set) Difference(other Set) Set {
	differencedSet := NewSet()

	for key, _ := range set {
		if !other.Contains(key) {
			differencedSet.Add(key)
		}
	}

	return differencedSet
}

// Returns a new set with items in the current set or the other set but not in both.
func (set Set) SymmetricDifference(other Set) Set {
	aDiff := set.Difference(other)
	bDiff := other.Difference(set)

	symDifferencedSet := aDiff.Union(bDiff)

	return symDifferencedSet
}

// Clears the entire set to be the empty set.
func (set *Set) Clear() {
	*set = make(map[interface{}]struct{})
}

// Allows the removal of a single item in the set.
func (set Set) Remove(i interface{}) {
	delete(set, i)
}

// Size returns the how many items are currently in the set.
func (set Set) Size() int {
	return len(set)
}

// Equal determines if two sets are equal to each other.
// If they both are the same size and have the same items they are considered equal.
// Order of items is not relevent for sets to be equal.
func (set Set) Equal(other Set) bool {
	if set.Size() != other.Size() {
		return false
	} else {
		for elem, _ := range set {
			if !other.Contains(elem) {
				return false
			}
		}
		return true
	}
}

// Provides a convenient string representation of the current state of the set.
func (set Set) String() string {
	items := make([]string, 0, len(set))

	for key := range set {
		items = append(items, fmt.Sprintf("%v", key))
	}
	return fmt.Sprintf("Set{%s}", strings.Join(items, ", "))
}
<|endoftext|>


<commit_before>package namesys

import (
	"fmt"
	"net"

	b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58"
	isd "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-is-domain"
	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
)

var ErrNotFound = fmt.Errorf("namesys: name not found")

// DNSResolver implements a Resolver on DNS domains
type DNSResolver struct {
	// TODO: maybe some sort of caching?
	// cache would need a timeout
}

// CanResolve implements Resolver
func (r *DNSResolver) CanResolve(name string) bool {
	return isd.IsDomain(name)
}

// Resolve implements Resolver
// TXT records for a given domain name should contain a b58
// encoded multihash.
func (r *DNSResolver) Resolve(name string) (string, error) {
	log.Info("DNSResolver resolving %v", name)
	txt, err := net.LookupTXT(name)
	if err != nil {
		return "", err
	}

	for _, t := range txt {
		chk := b58.Decode(t)
		if len(chk) == 0 {
			continue
		}

		_, err := mh.Cast(chk)
		if err != nil {
			continue
		}
		return t, nil
	}

	return "", ErrNotFound
}
<commit_msg>fix(namesys) use the error that already exists<commit_after>package namesys

import (
	"net"

	b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58"
	isd "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-is-domain"
	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
)

// DNSResolver implements a Resolver on DNS domains
type DNSResolver struct {
	// TODO: maybe some sort of caching?
	// cache would need a timeout
}

// CanResolve implements Resolver
func (r *DNSResolver) CanResolve(name string) bool {
	return isd.IsDomain(name)
}

// Resolve implements Resolver
// TXT records for a given domain name should contain a b58
// encoded multihash.
func (r *DNSResolver) Resolve(name string) (string, error) {
	log.Info("DNSResolver resolving %v", name)
	txt, err := net.LookupTXT(name)
	if err != nil {
		return "", err
	}

	for _, t := range txt {
		chk := b58.Decode(t)
		if len(chk) == 0 {
			continue
		}

		_, err := mh.Cast(chk)
		if err != nil {
			continue
		}
		return t, nil
	}

	return "", ErrResolveFailed
}
<|endoftext|>


<commit_before>package model

import (
	"encoding/json"
	"testing"
	"time"
)

func TestWorkerMarshalAndUnmarshal(t *testing.T) {
	var err error
	var expectedId WorkerId = "id"

	bytes, err := json.Marshal(&Worker{Id: expectedId, Timeout: time.Now()})
	if err != nil {
		t.Fatal("couldn't marshal worker: ", err)
	}

	worker := new(Worker)
	err = json.Unmarshal(bytes, worker)
	if err != nil {
		t.Fatal("couldn't unmarshal worker \"", string(bytes), "\": ",
			err)
	}

	if worker.Id != expectedId {
		t.Fatalf("expected id %v, but got %v", expectedId, worker.Id)
	}
}
<commit_msg>improved test<commit_after>package model

import (
	"encoding/json"
	"testing"
	"time"
)

func TestWorker2Json(t *testing.T) {
	var err error
	var expectedId WorkerId = "id"
	var expectedTimeout time.Time = time.Now()

	bytes, err := json.Marshal(&Worker{
		Id:      expectedId,
		Timeout: expectedTimeout,
	})
	if err != nil {
		t.Fatal("couldn't marshal worker: ", err)
	}

	worker := new(Worker)
	err = json.Unmarshal(bytes, worker)
	if err != nil {
		t.Fatal("couldn't unmarshal worker \"", string(bytes), "\": ",
			err)
	}

	if worker.Id != expectedId {
		t.Fatalf("expected id %v, but got %v",
			expectedId, worker.Id)
	}
	if worker.Timeout != expectedTimeout {
		t.Fatalf("expected timeout %v, but got %v",
			expectedTimeout, worker.Timeout)
	}
}
<|endoftext|>