<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/smtp"
	"net/url"
	"os"
	"os/user"
)

const (
	name   = "trafficjam"
	apiURL = "https://maps.googleapis.com/maps/api/distancematrix/json"
)

type config struct {
	Origins      string `json:"origins"`
	Destinations string `json:"destinations"`
	APIKey       string `json:"api_key"`
	Mode         string `json:"mode"`
	Avoid        string `json:"avoid"`
	TrafficModel string `json:"traffic_model"`
	MaxDuration  int    `json:"max_duration"`
	SMTP         struct {
		Host string `json:"host"`
		Port int    `json:"port"`
		User string `json:"user"`
		Pass string `json:"pass"`
	} `json:"smtp"`
	Recipient string `json:"recipient"`
}

type apiResponse struct {
	Rows []struct {
		Elements []struct {
			DurationInTraffic struct {
				Text  string `json:"text"`
				Value int    `json:"value"`
			} `json:"duration_in_traffic"`
			Status string `json:"status"`
		} `json:"elements"`
	} `json:"rows"`
	Status string `json:"status"`
}

func main() {
	log.SetPrefix(name + ": ")
	log.SetFlags(0)

	if len(os.Args) != 2 {
		log.Fatalf("usage: %s config.json", name)
	}

	conf, err := readConfig(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	params := map[string]string{
		"origins":        conf.Origins,
		"destinations":   conf.Destinations,
		"key":            conf.APIKey,
		"mode":           conf.Mode,
		"avoid":          conf.Avoid,
		"departure_time": "now",
		"traffic_model":  conf.TrafficModel,
	}
	apiResp, err := queryMapsAPI(params)
	if err != nil {
		log.Fatal(err)
	}

	duration := apiResp.Rows[0].Elements[0].DurationInTraffic.Value
	if duration > conf.MaxDuration*60 {
		if err := sendMail(conf, apiResp.Rows[0].Elements[0].DurationInTraffic.Text); err != nil {
			log.Fatal(err)
		}
	}
}

func readConfig(filename string) (*config, error) {
	var conf config

	confData, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(confData, &conf); err != nil {
		return nil, err
	}

	return &conf, nil
}

func queryMapsAPI(params map[string]string) (*apiResponse, error) {
	query := url.Values{}
	for key, val := range params {
		if val != "" {
			query.Set(key, val)
		}
	}

	uri, err := url.Parse(apiURL)
	if err != nil {
		return nil, err
	}
	uri.RawQuery = query.Encode()

	resp, err := http.Get(uri.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var apiResp apiResponse
	if err := json.Unmarshal(body, &apiResp); err != nil {
		return nil, err
	}
	if apiResp.Status != "OK" {
		return nil, fmt.Errorf("%s: bad response status: %s\n", name, apiResp.Status)
	}
	if len(apiResp.Rows) != 1 {
		return nil, fmt.Errorf("%s: response row count is not 1\n", name)
	}
	if len(apiResp.Rows[0].Elements) != 1 {
		return nil, fmt.Errorf("%s: response first row element count is not 1\n", name)
	}
	if apiResp.Rows[0].Elements[0].Status != "OK" {
		return nil, fmt.Errorf("%s: bad response first row first element status: %s\n", name, apiResp.Rows[0].Elements[0].Status)
	}

	return &apiResp, nil
}

func sendMail(conf *config, body string) error {
	user, err := user.Current()
	if err != nil {
		return err
	}

	hostname, err := os.Hostname()
	if err != nil {
		return err
	}

	auth := smtp.PlainAuth("", conf.SMTP.User, conf.SMTP.Pass, conf.SMTP.Host)
	sender := user.Username + "@" + hostname
	to := []string{conf.Recipient}
	msg := []byte("To: " + conf.Recipient + "\r\n" +
		"Subject: " + name + " alert\r\n" +
		"\r\n" +
		body + "\r\n")

	return smtp.SendMail(fmt.Sprintf("%s:%d", conf.SMTP.Host, conf.SMTP.Port), auth, sender, to, msg)
}
<commit_msg>Use lowercase field names for structs<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/smtp"
	"net/url"
	"os"
	"os/user"
)

const (
	name   = "trafficjam"
	apiURL = "https://maps.googleapis.com/maps/api/distancematrix/json"
)

type config struct {
	origins      string `json:"origins"`
	destinations string `json:"destinations"`
	apiKey       string `json:"api_key"`
	mode         string `json:"mode"`
	avoid        string `json:"avoid"`
	trafficModel string `json:"traffic_model"`
	maxDuration  int    `json:"max_duration"`
	smtp         struct {
		host string `json:"host"`
		port int    `json:"port"`
		user string `json:"user"`
		pass string `json:"pass"`
	} `json:"smtp"`
	recipient string `json:"recipient"`
}

type apiResponse struct {
	rows []struct {
		elements []struct {
			durationInTraffic struct {
				text  string `json:"text"`
				value int    `json:"value"`
			} `json:"duration_in_traffic"`
			status string `json:"status"`
		} `json:"elements"`
	} `json:"rows"`
	status string `json:"status"`
}

func main() {
	log.SetPrefix(name + ": ")
	log.SetFlags(0)

	if len(os.Args) != 2 {
		log.Fatalf("usage: %s config.json", name)
	}

	conf, err := readConfig(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	params := map[string]string{
		"origins":        conf.origins,
		"destinations":   conf.destinations,
		"key":            conf.apiKey,
		"mode":           conf.mode,
		"avoid":          conf.avoid,
		"departure_time": "now",
		"traffic_model":  conf.trafficModel,
	}
	apiResp, err := queryMapsAPI(params)
	if err != nil {
		log.Fatal(err)
	}

	duration := apiResp.rows[0].elements[0].durationInTraffic.value
	if duration > conf.maxDuration*60 {
		if err := sendMail(conf, apiResp.rows[0].elements[0].durationInTraffic.text); err != nil {
			log.Fatal(err)
		}
	}
}

func readConfig(filename string) (*config, error) {
	var conf config

	confData, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(confData, &conf); err != nil {
		return nil, err
	}

	return &conf, nil
}

func queryMapsAPI(params map[string]string) (*apiResponse, error) {
	query := url.Values{}
	for key, val := range params {
		if val != "" {
			query.Set(key, val)
		}
	}

	uri, err := url.Parse(apiURL)
	if err != nil {
		return nil, err
	}
	uri.RawQuery = query.Encode()

	resp, err := http.Get(uri.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var apiResp apiResponse
	if err := json.Unmarshal(body, &apiResp); err != nil {
		return nil, err
	}
	if apiResp.status != "OK" {
		return nil, fmt.Errorf("%s: bad response status: %s\n", name, apiResp.status)
	}
	if len(apiResp.rows) != 1 {
		return nil, fmt.Errorf("%s: response row count is not 1\n", name)
	}
	if len(apiResp.rows[0].elements) != 1 {
		return nil, fmt.Errorf("%s: response first row element count is not 1\n", name)
	}
	if apiResp.rows[0].elements[0].status != "OK" {
		return nil, fmt.Errorf("%s: bad response first row first element status: %s\n", name, apiResp.rows[0].elements[0].status)
	}

	return &apiResp, nil
}

func sendMail(conf *config, body string) error {
	user, err := user.Current()
	if err != nil {
		return err
	}

	hostname, err := os.Hostname()
	if err != nil {
		return err
	}

	auth := smtp.PlainAuth("", conf.smtp.user, conf.smtp.pass, conf.smtp.host)
	sender := user.Username + "@" + hostname
	to := []string{conf.recipient}
	msg := []byte("To: " + conf.recipient + "\r\n" +
		"Subject: " + name + " alert\r\n" +
		"\r\n" +
		body + "\r\n")

	return smtp.SendMail(fmt.Sprintf("%s:%d", conf.smtp.host, conf.smtp.port), auth, sender, to, msg)
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"testing"
)

func checkEq(a, b interface{}, t *testing.T) {
	if a != b {
		t.Error(fmt.Sprintf("%v != %v", a, b))
	}
}

// Shorthand for OpFromString.
func ofs(s string) Op {
	op, err := OpFromString(s)
	PanicOnError(err)
	return op
}

func TestInsert(t *testing.T) {
	op := Insert{Pos: 0, Value: "foo"}
	ds := op.ToString()
	checkEq(ds, "i0:foo", t)
	checkEq(ds, ofs(ds).ToString(), t)
}

func TestDelete(t *testing.T) {
	op := Delete{Pos: 2, Len: 4}
	ds := op.ToString()
	checkEq(ds, "d2:4", t)
	checkEq(ds, ofs(ds).ToString(), t)
}

func TestOpFromString(t *testing.T) {
	op := ofs("i2:bar")
	checkEq(*op.(*Insert), Insert{Pos: 2, Value: "bar"}, t)

	op = ofs("d5:2")
	checkEq(*op.(*Delete), Delete{Pos: 5, Len: 2}, t)
}

// Assumes OpFromString and Operator.ToString are tested.
// TODO: Share tests between Go and JS, i.e. use data-driven tests.
// TODO: Test TransformCompound.
func TestTransform(t *testing.T) {
	checkTransform := func(as, bs, aps, bps string, andReverse bool) {
		ap, bp := Transform(ofs(as), ofs(bs))
		checkEq(ap.ToString(), aps, t)
		checkEq(bp.ToString(), bps, t)

		if andReverse {
			bp, ap = Transform(ofs(bs), ofs(as))
			checkEq(ap.ToString(), aps, t)
			checkEq(bp.ToString(), bps, t)
		}
	}

	// Test insert-insert.
	checkTransform("i1:f", "i1:foo", "i4:f", "i1:foo", false)
	checkTransform("i1:foo", "i1:f", "i2:foo", "i1:f", false)
	checkTransform("i1:foo", "i1:foo", "i4:foo", "i1:foo", false)
	checkTransform("i1:foo", "i2:foo", "i1:foo", "i5:foo", true)
	checkTransform("i2:foo", "i1:foo", "i5:foo", "i1:foo", true)

	// Test insert-delete and delete-insert.
	checkTransform("i2:foo", "d0:1", "i1:foo", "d0:1", true)
	checkTransform("i2:foo", "d1:2", "i1:", "d1:5", true)
	checkTransform("i2:foo", "d2:2", "i2:foo", "d5:2", true)
	checkTransform("i2:foo", "d3:2", "i2:foo", "d6:2", true)
	checkTransform("i2:f", "d1:2", "i1:", "d1:3", true)
	checkTransform("i2:f", "d2:2", "i2:f", "d3:2", true)
	checkTransform("i2:f", "d3:2", "i2:f", "d4:2", true)
	checkTransform("i2:foo", "d1:1", "i1:foo", "d1:1", true)
	checkTransform("i2:foo", "d2:1", "i2:foo", "d5:1", true)
	checkTransform("i2:foo", "d3:1", "i2:foo", "d6:1", true)

	// Test delete-delete.
	checkTransform("d0:1", "d0:1", "d0:0", "d0:0", true)
	checkTransform("d0:1", "d0:2", "d0:0", "d0:1", true)
	// Hold b="d3:4" while shifting a forward.
	checkTransform("d0:2", "d3:4", "d0:2", "d1:4", true)
	checkTransform("d1:2", "d3:4", "d1:2", "d1:4", true)
	checkTransform("d2:2", "d3:4", "d2:1", "d2:3", true)
	checkTransform("d3:2", "d3:4", "d3:0", "d3:2", true)
	checkTransform("d4:2", "d3:4", "d3:0", "d3:2", true)
	checkTransform("d5:2", "d3:4", "d3:0", "d3:2", true)
	checkTransform("d6:2", "d3:4", "d3:1", "d3:3", true)
	checkTransform("d7:2", "d3:4", "d3:2", "d3:4", true)
	checkTransform("d8:2", "d3:4", "d4:2", "d3:4", true)
}

func checkTextEq(text *Text, s string, t *testing.T) {
	if text.Value != s {
		t.Error(fmt.Sprintf("%q != %q", text.Value, s))
	}
}

func TestNewText(t *testing.T) {
	checkTextEq(NewText(""), "", t)
	checkTextEq(NewText("foo"), "foo", t)
}

func TestApplyToEmpty(t *testing.T) {
	text := NewText("")
	op := &Insert{Pos: 0, Value: "foo"}
	text.Apply(op)
	checkTextEq(text, "foo", t)
}

func TestApplyTwice(t *testing.T) {
	text := NewText("")
	var op Op = &Insert{Pos: 0, Value: "foo"}
	text.Apply(op)
	text.Apply(op)
	op = &Delete{Pos: 2, Len: 1}
	text.Apply(op)
	text.Apply(op)
	checkTextEq(text, "fooo", t)
}

func TestApplyCompound(t *testing.T) {
	text := NewText("foobar")
	ops := []Op{
		&Delete{Pos: 0, Len: 3},
		&Insert{Pos: 2, Value: "seball"},
		&Delete{Pos: 8, Len: 1},
	}
	text.ApplyCompound(ops)
	checkTextEq(text, "baseball", t)
}
<commit_msg>move tests to main pkg for now<commit_after>package main

import (
	"fmt"
	"testing"
)

func checkEq(a, b interface{}, t *testing.T) {
	if a != b {
		t.Error(fmt.Sprintf("%v != %v", a, b))
	}
}

// Shorthand for OpFromString.
func ofs(s string) Op {
	op, err := OpFromString(s)
	PanicOnError(err)
	return op
}

func TestInsert(t *testing.T) {
	op := Insert{Pos: 0, Value: "foo"}
	ds := op.ToString()
	checkEq(ds, "i0:foo", t)
	checkEq(ds, ofs(ds).ToString(), t)
}

func TestDelete(t *testing.T) {
	op := Delete{Pos: 2, Len: 4}
	ds := op.ToString()
	checkEq(ds, "d2:4", t)
	checkEq(ds, ofs(ds).ToString(), t)
}

func TestOpFromString(t *testing.T) {
	op := ofs("i2:bar")
	checkEq(*op.(*Insert), Insert{Pos: 2, Value: "bar"}, t)

	op = ofs("d5:2")
	checkEq(*op.(*Delete), Delete{Pos: 5, Len: 2}, t)
}

// Assumes OpFromString and Operator.ToString are tested.
// TODO: Share tests between Go and JS, i.e. use data-driven tests.
// TODO: Test TransformCompound.
func TestTransform(t *testing.T) {
	checkTransform := func(as, bs, aps, bps string, andReverse bool) {
		ap, bp := Transform(ofs(as), ofs(bs))
		checkEq(ap.ToString(), aps, t)
		checkEq(bp.ToString(), bps, t)

		if andReverse {
			bp, ap = Transform(ofs(bs), ofs(as))
			checkEq(ap.ToString(), aps, t)
			checkEq(bp.ToString(), bps, t)
		}
	}

	// Test insert-insert.
	checkTransform("i1:f", "i1:foo", "i4:f", "i1:foo", false)
	checkTransform("i1:foo", "i1:f", "i2:foo", "i1:f", false)
	checkTransform("i1:foo", "i1:foo", "i4:foo", "i1:foo", false)
	checkTransform("i1:foo", "i2:foo", "i1:foo", "i5:foo", true)
	checkTransform("i2:foo", "i1:foo", "i5:foo", "i1:foo", true)

	// Test insert-delete and delete-insert.
	checkTransform("i2:foo", "d0:1", "i1:foo", "d0:1", true)
	checkTransform("i2:foo", "d1:2", "i1:", "d1:5", true)
	checkTransform("i2:foo", "d2:2", "i2:foo", "d5:2", true)
	checkTransform("i2:foo", "d3:2", "i2:foo", "d6:2", true)
	checkTransform("i2:f", "d1:2", "i1:", "d1:3", true)
	checkTransform("i2:f", "d2:2", "i2:f", "d3:2", true)
	checkTransform("i2:f", "d3:2", "i2:f", "d4:2", true)
	checkTransform("i2:foo", "d1:1", "i1:foo", "d1:1", true)
	checkTransform("i2:foo", "d2:1", "i2:foo", "d5:1", true)
	checkTransform("i2:foo", "d3:1", "i2:foo", "d6:1", true)

	// Test delete-delete.
	checkTransform("d0:1", "d0:1", "d0:0", "d0:0", true)
	checkTransform("d0:1", "d0:2", "d0:0", "d0:1", true)
	// Hold b="d3:4" while shifting a forward.
	checkTransform("d0:2", "d3:4", "d0:2", "d1:4", true)
	checkTransform("d1:2", "d3:4", "d1:2", "d1:4", true)
	checkTransform("d2:2", "d3:4", "d2:1", "d2:3", true)
	checkTransform("d3:2", "d3:4", "d3:0", "d3:2", true)
	checkTransform("d4:2", "d3:4", "d3:0", "d3:2", true)
	checkTransform("d5:2", "d3:4", "d3:0", "d3:2", true)
	checkTransform("d6:2", "d3:4", "d3:1", "d3:3", true)
	checkTransform("d7:2", "d3:4", "d3:2", "d3:4", true)
	checkTransform("d8:2", "d3:4", "d4:2", "d3:4", true)
}

func checkTextEq(text *Text, s string, t *testing.T) {
	if text.Value != s {
		t.Error(fmt.Sprintf("%q != %q", text.Value, s))
	}
}

func TestNewText(t *testing.T) {
	checkTextEq(NewText(""), "", t)
	checkTextEq(NewText("foo"), "foo", t)
}

func TestApplyToEmpty(t *testing.T) {
	text := NewText("")
	op := &Insert{Pos: 0, Value: "foo"}
	text.Apply(op)
	checkTextEq(text, "foo", t)
}

func TestApplyTwice(t *testing.T) {
	text := NewText("")
	var op Op = &Insert{Pos: 0, Value: "foo"}
	text.Apply(op)
	text.Apply(op)
	op = &Delete{Pos: 2, Len: 1}
	text.Apply(op)
	text.Apply(op)
	checkTextEq(text, "fooo", t)
}

func TestApplyCompound(t *testing.T) {
	text := NewText("foobar")
	ops := []Op{
		&Delete{Pos: 0, Len: 3},
		&Insert{Pos: 2, Value: "seball"},
		&Delete{Pos: 8, Len: 1},
	}
	text.ApplyCompound(ops)
	checkTextEq(text, "baseball", t)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"image"
	"image/color"
	"image/gif"
	"io"
	"log"
	"math"
	"math/rand"
	"net/http"
	"os"
	"time"
)

const (
	bgIndex = 0     // background color palette index
	fgIndex = 1     // foreground color palette index
	cycles  = 5     // default number of complete x oscillator revolutions
	res     = 0.001 // default angular resolution
	size    = 250   // default image canvas covers [-size..+size]
	nFrames = 64    // default number of animation frames
	delay   = 8     // default delay between frames in 10ms units
)

func main() {
	log.Print("Server running...")
	rand.Seed(time.Now().UTC().UnixNano())
	http.HandleFunc("/", handler)
	log.Fatal(http.ListenAndServe("localhost:8000", nil))
}

func handler(w http.ResponseWriter, r *http.Request) {
	lissajous(w)
}

func lissajous(out io.Writer) {
	palette := []color.Color{color.Black}
	for i := 0; i < nFrames; i++ {
		red := (uint8)((i * 255) / nFrames)
		green := (uint8)(((i + nFrames/2) * 255) / nFrames)
		blue := (uint8)(((nFrames/2 - i) * 255) / nFrames)
		fmt.Fprintf(os.Stderr, "RGB=(0x%02x%02x%02x)\n", red, green, blue)
		palette = append(palette, color.RGBA{red, green, blue, 0xff})
	}
	freq := rand.Float64() * 3.0 // relative frequency of y oscillator
	anim := gif.GIF{LoopCount: nFrames}
	phase := 0.0
	for i := 0; i < nFrames; i++ {
		rect := image.Rect(0, 0, 2*size+1, 2*size+1)
		img := image.NewPaletted(rect, palette)
		for t := 0.0; t < cycles*2*math.Pi; t += res {
			x := math.Sin(t)
			y := math.Sin(t*freq + phase)
			img.SetColorIndex(size+int(x*size+0.5), size+int(y*size+0.5),
				(uint8)(i+1))
		}
		phase += 0.1
		anim.Delay = append(anim.Delay, delay)
		anim.Image = append(anim.Image, img)
	}
	gif.EncodeAll(out, &anim)
}
<commit_msg>Change to using function literal, i.e. anonymous function.<commit_after>package main

import (
	"fmt"
	"image"
	"image/color"
	"image/gif"
	"io"
	"log"
	"math"
	"math/rand"
	"net/http"
	"os"
	"time"
)

const (
	bgIndex = 0     // background color palette index
	fgIndex = 1     // foreground color palette index
	cycles  = 5     // default number of complete x oscillator revolutions
	res     = 0.001 // default angular resolution
	size    = 250   // default image canvas covers [-size..+size]
	nFrames = 64    // default number of animation frames
	delay   = 8     // default delay between frames in 10ms units
)

func main() {
	log.Print("Server running...")
	rand.Seed(time.Now().UTC().UnixNano())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		lissajous(w)
	})
	log.Fatal(http.ListenAndServe("localhost:8000", nil))
}

func lissajous(out io.Writer) {
	palette := []color.Color{color.Black}
	for i := 0; i < nFrames; i++ {
		red := (uint8)((i * 255) / nFrames)
		green := (uint8)(((i + nFrames/2) * 255) / nFrames)
		blue := (uint8)(((nFrames/2 - i) * 255) / nFrames)
		fmt.Fprintf(os.Stderr, "RGB=(0x%02x%02x%02x)\n", red, green, blue)
		palette = append(palette, color.RGBA{red, green, blue, 0xff})
	}
	freq := rand.Float64() * 3.0 // relative frequency of y oscillator
	anim := gif.GIF{LoopCount: nFrames}
	phase := 0.0
	for i := 0; i < nFrames; i++ {
		rect := image.Rect(0, 0, 2*size+1, 2*size+1)
		img := image.NewPaletted(rect, palette)
		for t := 0.0; t < cycles*2*math.Pi; t += res {
			x := math.Sin(t)
			y := math.Sin(t*freq + phase)
			img.SetColorIndex(size+int(x*size+0.5), size+int(y*size+0.5),
				(uint8)(i+1))
		}
		phase += 0.1
		anim.Delay = append(anim.Delay, delay)
		anim.Image = append(anim.Image, img)
	}
	gif.EncodeAll(out, &anim)
}
<|endoftext|>


<commit_before>// Package sqlignorestore contains a SQL implementation of ignore.Store.
package sqlignorestore

import (
	"context"
	"fmt"
	"net/url"
	"sort"
	"strings"

	"github.com/cockroachdb/cockroach-go/v2/crdb/crdbpgx"
	"github.com/jackc/pgx/v4"
	"github.com/jackc/pgx/v4/pgxpool"

	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/golden/go/ignore"
	"go.skia.org/infra/golden/go/sql/schema"
)

type StoreImpl struct {
	db *pgxpool.Pool
}

// New returns a SQL based implementation of ignore.Store.
func New(db *pgxpool.Pool) *StoreImpl {
	return &StoreImpl{db: db}
}

// Create implements the ignore.Store interface. It will mark all traces that match the rule as
// "ignored".
func (s *StoreImpl) Create(ctx context.Context, rule ignore.Rule) error {
	v, err := url.ParseQuery(rule.Query)
	if err != nil {
		return skerr.Wrapf(err, "invalid ignore query %q", rule.Query)
	}
	err = crdbpgx.ExecuteTx(ctx, s.db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err := tx.Exec(ctx, `
INSERT INTO IgnoreRules (creator_email, updated_email, expires, note, query)
VALUES ($1, $2, $3, $4, $5)`, rule.CreatedBy, rule.CreatedBy, rule.Expires, rule.Note, v)
		return err // Don't wrap - crdbpgx might retry
	})
	if err != nil {
		return skerr.Wrapf(err, "creating ignore rule %#v", rule)
	}
	// We could be updating a lot of traces and values at head here. If done as one big transaction,
	// that could take a while to land if we are ingesting a lot of new data at the time. As such,
	// we do it in three independent transactions
	if err := markTracesAsIgnored(ctx, s.db, v); err != nil {
		return skerr.Wrap(err)
	}
	if err := markValuesAtHeadAsIgnored(ctx, s.db, v); err != nil {
		return skerr.Wrap(err)
	}
	return nil
}

// markTracesAsIgnored will update all Traces matching the given paramset as ignored.
func markTracesAsIgnored(ctx context.Context, db *pgxpool.Pool, ps map[string][]string) error {
	condition, arguments := ConvertIgnoreRules([]paramtools.ParamSet{ps})
	statement := `UPDATE Traces SET matches_any_ignore_rule = TRUE WHERE `
	statement += condition
	statement += ` RETURNING NOTHING`
	err := crdbpgx.ExecuteTx(ctx, db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err := tx.Exec(ctx, statement, arguments...)
		return err // Don't wrap - crdbpgx might retry
	})
	return skerr.Wrap(err)
}

// markValuesAtHeadAsIgnored will update all ValuesAtHead matching the given paramset as ignored.
func markValuesAtHeadAsIgnored(ctx context.Context, db *pgxpool.Pool, ps map[string][]string) error {
	condition, arguments := ConvertIgnoreRules([]paramtools.ParamSet{ps})
	statement := `UPDATE ValuesAtHead SET matches_any_ignore_rule = TRUE WHERE `
	statement += condition
	statement += ` RETURNING NOTHING`
	err := crdbpgx.ExecuteTx(ctx, db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err := tx.Exec(ctx, statement, arguments...)
		return err // Don't wrap - crdbpgx might retry
	})
	return skerr.Wrap(err)
}

// List implements the ignore.Store interface.
func (s *StoreImpl) List(ctx context.Context) ([]ignore.Rule, error) {
	var rv []ignore.Rule
	rows, err := s.db.Query(ctx, `SELECT * FROM IgnoreRules ORDER BY expires ASC`)
	if err != nil {
		return nil, skerr.Wrap(err)
	}
	defer rows.Close()
	for rows.Next() {
		var r schema.IgnoreRuleRow
		err := rows.Scan(&r.IgnoreRuleID, &r.CreatorEmail, &r.UpdatedEmail, &r.Expires, &r.Note, &r.Query)
		if err != nil {
			return nil, skerr.Wrap(err)
		}
		rv = append(rv, ignore.Rule{
			ID:        r.IgnoreRuleID.String(),
			CreatedBy: r.CreatorEmail,
			UpdatedBy: r.UpdatedEmail,
			Expires:   r.Expires.UTC(),
			Query:     url.Values(r.Query).Encode(),
			Note:      r.Note,
		})
	}
	return rv, nil
}

// Update implements the ignore.Store interface. If the rule paramset changes, it will mark the
// traces that match the old params as "ignored" or not depending on how the unchanged n-1 rules
// plus the new rule affect them. It will then update all traces that match the new rule as
// "ignored".
func (s *StoreImpl) Update(ctx context.Context, rule ignore.Rule) error {
	newParamSet, err := url.ParseQuery(rule.Query)
	if err != nil {
		return skerr.Wrapf(err, "invalid ignore query %q", rule.Query)
	}
	existingRulePS, err := s.getRuleParamSet(ctx, rule.ID)
	if err != nil {
		return skerr.Wrapf(err, "getting existing rule with id %s", rule.ID)
	}
	err = crdbpgx.ExecuteTx(ctx, s.db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err = tx.Exec(ctx, `
UPDATE IgnoreRules SET (updated_email, expires, note, query) = ($1, $2, $3, $4)
WHERE ignore_rule_id = $5`, rule.UpdatedBy, rule.Expires, rule.Note, newParamSet, rule.ID)
		return err // Don't wrap - crdbpgx might retry
	})
	if err != nil {
		return skerr.Wrapf(err, "updating rule with id %s to %#v %#v", rule.ID, rule, newParamSet)
	}
	if existingRulePS.Equal(newParamSet) {
		// We don't need to update Traces or ValuesAtHead because the query was unchanged.
		return nil
	}
	// We could be updating a lot of traces and values at head here. If done as one big transaction,
	// that could take a while to land if we are ingesting a lot of new data at the time. As such,
	// we update in separate transactions.
	combinedRules, err := s.getOtherRules(ctx, rule.ID)
	if err != nil {
		return skerr.Wrapf(err, "getting other rules when updating %s", rule.ID)
	}
	// Apply those old rules to the traces that match the old paramset
	if err := conditionallyMarkTracesAsIgnored(ctx, s.db, existingRulePS, combinedRules); err != nil {
		return skerr.Wrap(err)
	}
	if err := conditionallyMarkValuesAtHeadAsIgnored(ctx, s.db, existingRulePS, combinedRules); err != nil {
		return skerr.Wrap(err)
	}
	// Apply the result of the new rules.
	if err := markTracesAsIgnored(ctx, s.db, newParamSet); err != nil {
		return skerr.Wrap(err)
	}
	if err := markValuesAtHeadAsIgnored(ctx, s.db, newParamSet); err != nil {
		return skerr.Wrap(err)
	}
	return nil
}

// conditionallyMarkTracesAsIgnored applies the slice of rules to all traces that match the
// provided PatchSet.
func conditionallyMarkTracesAsIgnored(ctx context.Context, db *pgxpool.Pool, ps paramtools.ParamSet, rules []paramtools.ParamSet) error {
	matches, matchArgs := ConvertIgnoreRules(rules)
	condition, conArgs := convertIgnoreRules([]paramtools.ParamSet{ps}, len(matchArgs)+1)
	statement := `UPDATE Traces SET matches_any_ignore_rule = `
	statement += matches
	statement += ` WHERE `
	statement += condition
	statement += ` RETURNING NOTHING`
	matchArgs = append(matchArgs, conArgs...)
	err := crdbpgx.ExecuteTx(ctx, db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err := tx.Exec(ctx, statement, matchArgs...)
		return err // Don't wrap - crdbpgx might retry
	})
	if err != nil {
		return skerr.Wrapf(err, "updating traces to match %d rules", len(rules))
	}
	return nil
}

// conditionallyMarkValuesAtHeadAsIgnored applies the slice of rules to all ValuesAtHead that
// match the provided PatchSet.
func conditionallyMarkValuesAtHeadAsIgnored(ctx context.Context, db *pgxpool.Pool, ps paramtools.ParamSet, rules []paramtools.ParamSet) error {
	matches, matchArgs := ConvertIgnoreRules(rules)
	condition, conArgs := convertIgnoreRules([]paramtools.ParamSet{ps}, len(matchArgs)+1)
	statement := `UPDATE ValuesAtHead SET matches_any_ignore_rule = `
	statement += matches
	statement += ` WHERE `
	statement += condition
	statement += ` RETURNING NOTHING`
	matchArgs = append(matchArgs, conArgs...)
	err := crdbpgx.ExecuteTx(ctx, db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err := tx.Exec(ctx, statement, matchArgs...)
		return err // Don't wrap - crdbpgx might retry
	})
	if err != nil {
		return skerr.Wrapf(err, "updating traces to match %d rules", len(rules))
	}
	return nil
}

// Delete implements the ignore.Store interface. It will mark the traces that match the params of
// the deleted rule as "ignored" or not depending on how the unchanged n-1 rules affect them.
func (s *StoreImpl) Delete(ctx context.Context, id string) error {
	existingRulePS, err := s.getRuleParamSet(ctx, id)
	if err != nil {
		return skerr.Wrapf(err, "getting existing rule with id %s", id)
	}
	err = crdbpgx.ExecuteTx(ctx, s.db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err = tx.Exec(ctx, `
DELETE FROM IgnoreRules WHERE ignore_rule_id = $1`, id)
		return err // Don't wrap - crdbpgx might retry
	})
	// We could be updating a lot of traces and values at head here. If done as one big transaction,
	// that could take a while to land if we are ingesting a lot of new data at the time. As such,
	// we update in separate transactions.
	remainingRules, err := s.getOtherRules(ctx, id)
	if err != nil {
		return skerr.Wrapf(err, "getting other rules when deleting %s", id)
	}
	// Apply those old rules to the traces that match the old paramset
	if err := conditionallyMarkTracesAsIgnored(ctx, s.db, existingRulePS, remainingRules); err != nil {
		return skerr.Wrap(err)
	}
	if err := conditionallyMarkValuesAtHeadAsIgnored(ctx, s.db, existingRulePS, remainingRules); err != nil {
		return skerr.Wrap(err)
	}
	return nil
}

// getRuleParamSet returns the ParamSet for a given rule.
func (s *StoreImpl) getRuleParamSet(ctx context.Context, id string) (paramtools.ParamSet, error) {
	var ps paramtools.ParamSet
	row := s.db.QueryRow(ctx, `SELECT query FROM IgnoreRules where ignore_rule_id = $1`, id)
	if err := row.Scan(&ps); err != nil {
		return ps, skerr.Wrap(err)
	}
	return ps, nil
}

// getOtherRules returns a slice of params that has all rules not matching the given id.
func (s *StoreImpl) getOtherRules(ctx context.Context, id string) ([]paramtools.ParamSet, error) {
	var rules []paramtools.ParamSet
	rows, err := s.db.Query(ctx, `SELECT query FROM IgnoreRules where ignore_rule_id != $1`, id)
	if err != nil {
		return nil, skerr.Wrap(err)
	}
	defer rows.Close()
	for rows.Next() {
		var ps paramtools.ParamSet
		if err := rows.Scan(&ps); err != nil {
			return nil, skerr.Wrap(err)
		}
		rules = append(rules, ps)
	}
	return rules, nil
}

// Make sure Store fulfills the ignore.Store interface
var _ ignore.Store = (*StoreImpl)(nil)

// ConvertIgnoreRules turns a Paramset into a SQL clause that would match rows using a column
// named "keys". It is currently implemented with AND/OR clauses, but could potentially be done
// with UNION/INTERSECT depending on performance needs.
func ConvertIgnoreRules(rules []paramtools.ParamSet) (string, []interface{}) {
	return convertIgnoreRules(rules, 1)
}

// convertIgnoreRules takes a parameter that configures where the numbered params start.
// 1 is the lowest legal value. 2^16 is the biggest.
func convertIgnoreRules(rules []paramtools.ParamSet, startIndex int) (string, []interface{}) {
	if len(rules) == 0 {
		return "false", nil
	}
	conditions := make([]string, 0, len(rules))
	var arguments []interface{}
	argIdx := startIndex

	for _, rule := range rules {
		rule.Normalize()
		keys := make([]string, 0, len(rule))
		for key := range rule {
			keys = append(keys, key)
		}
		sort.Strings(keys) // sort the keys for determinism

		andParts := make([]string, 0, len(rules))
		for _, key := range keys {
			values := rule[key]
			// We need the COALESCE because if a trace has one key, but not another, it will
			// return NULL. We don't want this NULL to propagate (FALSE OR NULL == NULL), so
			// we coalesce it to false (since if a trace lacks a key, it cannot match the key:value
			// pair).
			subCondition := fmt.Sprintf("COALESCE(keys ->> $%d::STRING IN (", argIdx)
			argIdx++
			arguments = append(arguments, key)
			for i, value := range values {
				if i != 0 {
					subCondition += ", "
				}
				subCondition += fmt.Sprintf("$%d", argIdx)
				argIdx++
				arguments = append(arguments, value)
			}
			subCondition += "), FALSE)"
			andParts = append(andParts, subCondition)
		}
		condition := "(" + strings.Join(andParts, " AND ") + ")"
		conditions = append(conditions, condition)
	}
	combined := "(" + strings.Join(conditions, "\nOR ") + ")"
	return combined, arguments
}
<commit_msg>[gold] Add tracing to sql ignorestore<commit_after>// Package sqlignorestore contains a SQL implementation of ignore.Store.
package sqlignorestore

import (
	"context"
	"fmt"
	"net/url"
	"sort"
	"strings"

	"go.opencensus.io/trace"

	"github.com/cockroachdb/cockroach-go/v2/crdb/crdbpgx"
	"github.com/jackc/pgx/v4"
	"github.com/jackc/pgx/v4/pgxpool"

	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/golden/go/ignore"
	"go.skia.org/infra/golden/go/sql/schema"
)

type StoreImpl struct {
	db *pgxpool.Pool
}

// New returns a SQL based implementation of ignore.Store.
func New(db *pgxpool.Pool) *StoreImpl {
	return &StoreImpl{db: db}
}

// Create implements the ignore.Store interface. It will mark all traces that match the rule as
// "ignored".
func (s *StoreImpl) Create(ctx context.Context, rule ignore.Rule) error {
	ctx, span := trace.StartSpan(ctx, "ignorestore_Create", trace.WithSampler(trace.AlwaysSample()))
	defer span.End()
	v, err := url.ParseQuery(rule.Query)
	if err != nil {
		return skerr.Wrapf(err, "invalid ignore query %q", rule.Query)
	}
	err = crdbpgx.ExecuteTx(ctx, s.db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err := tx.Exec(ctx, `
INSERT INTO IgnoreRules (creator_email, updated_email, expires, note, query)
VALUES ($1, $2, $3, $4, $5)`, rule.CreatedBy, rule.CreatedBy, rule.Expires, rule.Note, v)
		return err // Don't wrap - crdbpgx might retry
	})
	if err != nil {
		return skerr.Wrapf(err, "creating ignore rule %#v", rule)
	}
	// We could be updating a lot of traces and values at head here. If done as one big transaction,
	// that could take a while to land if we are ingesting a lot of new data at the time. As such,
	// we do it in three independent transactions
	if err := markTracesAsIgnored(ctx, s.db, v); err != nil {
		return skerr.Wrap(err)
	}
	if err := markValuesAtHeadAsIgnored(ctx, s.db, v); err != nil {
		return skerr.Wrap(err)
	}
	return nil
}

// markTracesAsIgnored will update all Traces matching the given paramset as ignored.
func markTracesAsIgnored(ctx context.Context, db *pgxpool.Pool, ps map[string][]string) error {
	ctx, span := trace.StartSpan(ctx, "markTracesAsIgnored")
	defer span.End()
	condition, arguments := ConvertIgnoreRules([]paramtools.ParamSet{ps})
	statement := `UPDATE Traces SET matches_any_ignore_rule = TRUE WHERE `
	statement += condition
	statement += ` RETURNING NOTHING`
	err := crdbpgx.ExecuteTx(ctx, db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err := tx.Exec(ctx, statement, arguments...)
		return err // Don't wrap - crdbpgx might retry
	})
	return skerr.Wrap(err)
}

// markValuesAtHeadAsIgnored will update all ValuesAtHead matching the given paramset as ignored.
func markValuesAtHeadAsIgnored(ctx context.Context, db *pgxpool.Pool, ps map[string][]string) error {
	ctx, span := trace.StartSpan(ctx, "markValuesAtHeadAsIgnored")
	defer span.End()
	condition, arguments := ConvertIgnoreRules([]paramtools.ParamSet{ps})
	statement := `UPDATE ValuesAtHead SET matches_any_ignore_rule = TRUE WHERE `
	statement += condition
	statement += ` RETURNING NOTHING`
	err := crdbpgx.ExecuteTx(ctx, db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err := tx.Exec(ctx, statement, arguments...)
		return err // Don't wrap - crdbpgx might retry
	})
	return skerr.Wrap(err)
}

// List implements the ignore.Store interface.
func (s *StoreImpl) List(ctx context.Context) ([]ignore.Rule, error) {
	ctx, span := trace.StartSpan(ctx, "ignorestore_List", trace.WithSampler(trace.AlwaysSample()))
	defer span.End()
	var rv []ignore.Rule
	rows, err := s.db.Query(ctx, `SELECT * FROM IgnoreRules ORDER BY expires ASC`)
	if err != nil {
		return nil, skerr.Wrap(err)
	}
	defer rows.Close()
	for rows.Next() {
		var r schema.IgnoreRuleRow
		err := rows.Scan(&r.IgnoreRuleID, &r.CreatorEmail, &r.UpdatedEmail, &r.Expires, &r.Note, &r.Query)
		if err != nil {
			return nil, skerr.Wrap(err)
		}
		rv = append(rv, ignore.Rule{
			ID:        r.IgnoreRuleID.String(),
			CreatedBy: r.CreatorEmail,
			UpdatedBy: r.UpdatedEmail,
			Expires:   r.Expires.UTC(),
			Query:     url.Values(r.Query).Encode(),
			Note:      r.Note,
		})
	}
	return rv, nil
}

// Update implements the ignore.Store interface. If the rule paramset changes, it will mark the
// traces that match the old params as "ignored" or not depending on how the unchanged n-1 rules
// plus the new rule affect them. It will then update all traces that match the new rule as
// "ignored".
func (s *StoreImpl) Update(ctx context.Context, rule ignore.Rule) error {
	ctx, span := trace.StartSpan(ctx, "ignorestore_Update", trace.WithSampler(trace.AlwaysSample()))
	defer span.End()
	newParamSet, err := url.ParseQuery(rule.Query)
	if err != nil {
		return skerr.Wrapf(err, "invalid ignore query %q", rule.Query)
	}
	existingRulePS, err := s.getRuleParamSet(ctx, rule.ID)
	if err != nil {
		return skerr.Wrapf(err, "getting existing rule with id %s", rule.ID)
	}
	err = crdbpgx.ExecuteTx(ctx, s.db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err = tx.Exec(ctx, `
UPDATE IgnoreRules SET (updated_email, expires, note, query) = ($1, $2, $3, $4)
WHERE ignore_rule_id = $5`, rule.UpdatedBy, rule.Expires, rule.Note, newParamSet, rule.ID)
		return err // Don't wrap - crdbpgx might retry
	})
	if err != nil {
		return skerr.Wrapf(err, "updating rule with id %s to %#v %#v", rule.ID, rule, newParamSet)
	}
	if existingRulePS.Equal(newParamSet) {
		// We don't need to update Traces or ValuesAtHead because the query was unchanged.
		return nil
	}
	// We could be updating a lot of traces and values at head here. If done as one big transaction,
	// that could take a while to land if we are ingesting a lot of new data at the time. As such,
	// we update in separate transactions.
	combinedRules, err := s.getOtherRules(ctx, rule.ID)
	if err != nil {
		return skerr.Wrapf(err, "getting other rules when updating %s", rule.ID)
	}
	// Apply those old rules to the traces that match the old paramset
	if err := conditionallyMarkTracesAsIgnored(ctx, s.db, existingRulePS, combinedRules); err != nil {
		return skerr.Wrap(err)
	}
	if err := conditionallyMarkValuesAtHeadAsIgnored(ctx, s.db, existingRulePS, combinedRules); err != nil {
		return skerr.Wrap(err)
	}
	// Apply the result of the new rules.
	if err := markTracesAsIgnored(ctx, s.db, newParamSet); err != nil {
		return skerr.Wrap(err)
	}
	if err := markValuesAtHeadAsIgnored(ctx, s.db, newParamSet); err != nil {
		return skerr.Wrap(err)
	}
	return nil
}

// conditionallyMarkTracesAsIgnored applies the slice of rules to all traces that match the
// provided PatchSet.
func conditionallyMarkTracesAsIgnored(ctx context.Context, db *pgxpool.Pool, ps paramtools.ParamSet, rules []paramtools.ParamSet) error {
	ctx, span := trace.StartSpan(ctx, "conditionallyMarkTracesAsIgnored")
	defer span.End()
	matches, matchArgs := ConvertIgnoreRules(rules)
	condition, conArgs := convertIgnoreRules([]paramtools.ParamSet{ps}, len(matchArgs)+1)
	statement := `UPDATE Traces SET matches_any_ignore_rule = `
	statement += matches
	statement += ` WHERE `
	statement += condition
	statement += ` RETURNING NOTHING`
	matchArgs = append(matchArgs, conArgs...)
	err := crdbpgx.ExecuteTx(ctx, db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err := tx.Exec(ctx, statement, matchArgs...)
		return err // Don't wrap - crdbpgx might retry
	})
	if err != nil {
		return skerr.Wrapf(err, "updating traces to match %d rules", len(rules))
	}
	return nil
}

// conditionallyMarkValuesAtHeadAsIgnored applies the slice of rules to all ValuesAtHead that
// match the provided PatchSet.
func conditionallyMarkValuesAtHeadAsIgnored(ctx context.Context, db *pgxpool.Pool, ps paramtools.ParamSet, rules []paramtools.ParamSet) error {
	ctx, span := trace.StartSpan(ctx, "conditionallyMarkValuesAtHeadAsIgnored")
	defer span.End()
	matches, matchArgs := ConvertIgnoreRules(rules)
	condition, conArgs := convertIgnoreRules([]paramtools.ParamSet{ps}, len(matchArgs)+1)
	statement := `UPDATE ValuesAtHead SET matches_any_ignore_rule = `
	statement += matches
	statement += ` WHERE `
	statement += condition
	statement += ` RETURNING NOTHING`
	matchArgs = append(matchArgs, conArgs...)
	err := crdbpgx.ExecuteTx(ctx, db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err := tx.Exec(ctx, statement, matchArgs...)
		return err // Don't wrap - crdbpgx might retry
	})
	if err != nil {
		return skerr.Wrapf(err, "updating traces to match %d rules", len(rules))
	}
	return nil
}

// Delete implements the ignore.Store interface. It will mark the traces that match the params of
// the deleted rule as "ignored" or not depending on how the unchanged n-1 rules affect them.
func (s *StoreImpl) Delete(ctx context.Context, id string) error {
	ctx, span := trace.StartSpan(ctx, "ignorestore_Delete", trace.WithSampler(trace.AlwaysSample()))
	defer span.End()
	existingRulePS, err := s.getRuleParamSet(ctx, id)
	if err != nil {
		return skerr.Wrapf(err, "getting existing rule with id %s", id)
	}
	err = crdbpgx.ExecuteTx(ctx, s.db, pgx.TxOptions{}, func(tx pgx.Tx) error {
		_, err = tx.Exec(ctx, `
DELETE FROM IgnoreRules WHERE ignore_rule_id = $1`, id)
		return err // Don't wrap - crdbpgx might retry
	})
	// We could be updating a lot of traces and values at head here. If done as one big transaction,
	// that could take a while to land if we are ingesting a lot of new data at the time. As such,
	// we update in separate transactions.
	remainingRules, err := s.getOtherRules(ctx, id)
	if err != nil {
		return skerr.Wrapf(err, "getting other rules when deleting %s", id)
	}
	// Apply those old rules to the traces that match the old paramset
	if err := conditionallyMarkTracesAsIgnored(ctx, s.db, existingRulePS, remainingRules); err != nil {
		return skerr.Wrap(err)
	}
	if err := conditionallyMarkValuesAtHeadAsIgnored(ctx, s.db, existingRulePS, remainingRules); err != nil {
		return skerr.Wrap(err)
	}
	return nil
}

// getRuleParamSet returns the ParamSet for a given rule.
func (s *StoreImpl) getRuleParamSet(ctx context.Context, id string) (paramtools.ParamSet, error) {
	ctx, span := trace.StartSpan(ctx, "getRuleParamSet")
	defer span.End()
	var ps paramtools.ParamSet
	row := s.db.QueryRow(ctx, `SELECT query FROM IgnoreRules where ignore_rule_id = $1`, id)
	if err := row.Scan(&ps); err != nil {
		return ps, skerr.Wrap(err)
	}
	return ps, nil
}

// getOtherRules returns a slice of params that has all rules not matching the given id.
func (s *StoreImpl) getOtherRules(ctx context.Context, id string) ([]paramtools.ParamSet, error) {
	ctx, span := trace.StartSpan(ctx, "getOtherRules")
	defer span.End()
	var rules []paramtools.ParamSet
	rows, err := s.db.Query(ctx, `SELECT query FROM IgnoreRules where ignore_rule_id != $1`, id)
	if err != nil {
		return nil, skerr.Wrap(err)
	}
	defer rows.Close()
	for rows.Next() {
		var ps paramtools.ParamSet
		if err := rows.Scan(&ps); err != nil {
			return nil, skerr.Wrap(err)
		}
		rules = append(rules, ps)
	}
	return rules, nil
}

// Make sure Store fulfills the ignore.Store interface
var _ ignore.Store = (*StoreImpl)(nil)

// ConvertIgnoreRules turns a Paramset into a SQL clause that would match rows using a column
// named "keys". It is currently implemented with AND/OR clauses, but could potentially be done
// with UNION/INTERSECT depending on performance needs.
func ConvertIgnoreRules(rules []paramtools.ParamSet) (string, []interface{}) {
	return convertIgnoreRules(rules, 1)
}

// convertIgnoreRules takes a parameter that configures where the numbered params start.
// 1 is the lowest legal value. 2^16 is the biggest.
func convertIgnoreRules(rules []paramtools.ParamSet, startIndex int) (string, []interface{}) {
	if len(rules) == 0 {
		return "false", nil
	}
	conditions := make([]string, 0, len(rules))
	var arguments []interface{}
	argIdx := startIndex

	for _, rule := range rules {
		rule.Normalize()
		keys := make([]string, 0, len(rule))
		for key := range rule {
			keys = append(keys, key)
		}
		sort.Strings(keys) // sort the keys for determinism

		andParts := make([]string, 0, len(rules))
		for _, key := range keys {
			values := rule[key]
			// We need the COALESCE because if a trace has one key, but not another, it will
			// return NULL. We don't want this NULL to propagate (FALSE OR NULL == NULL), so
			// we coalesce it to false (since if a trace lacks a key, it cannot match the key:value
			// pair).
			subCondition := fmt.Sprintf("COALESCE(keys ->> $%d::STRING IN (", argIdx)
			argIdx++
			arguments = append(arguments, key)
			for i, value := range values {
				if i != 0 {
					subCondition += ", "
				}
				subCondition += fmt.Sprintf("$%d", argIdx)
				argIdx++
				arguments = append(arguments, value)
			}
			subCondition += "), FALSE)"
			andParts = append(andParts, subCondition)
		}
		condition := "(" + strings.Join(andParts, " AND ") + ")"
		conditions = append(conditions, condition)
	}
	combined := "(" + strings.Join(conditions, "\nOR ") + ")"
	return combined, arguments
}
<|endoftext|>


<commit_before>package fstree_test

import (
	"os"
	"runtime"
	"strconv"
	"strings"
	"testing"

	"github.com/caelifer/dups/balancer"
	"github.com/caelifer/dups/fstree"
)

func TestTreeWalk(t *testing.T) {
	// Recover from panic
	defer func() {
		if r := recover(); r != nil {
			t.Fatal(r)
		}
	}()

	testRoot := "/Users/timour/golang/src/github.com/caelifer/dups/t/"

	nprocs := runtime.NumCPU()
	t.Logf("Using %d CPU threads\n", nprocs)

	for i := 10; i < 20; i++ {
		results := make([]string, 0, 256)
		testdir := testRoot + strconv.Itoa(i)

		err := fstree.Walk(balancer.NewWorkQueue(nprocs), testdir, func(path string, info os.FileInfo, err error) error {
			results = append(results, path)
			return nil
		})

		if err != nil {
			t.Fatal(err)
		}

		t.Logf("Results for %d entries:\n%s\n", i+1, strings.Join(results, "\n"))
		if n := len(results); n != i+1 {
			t.Errorf("Expected %d nodes, got %d\n", i+1, n)
		}
	}
}

func TestTreeWalk11(t *testing.T) {
	// Recover from panic
	defer func() {
		if r := recover(); r != nil {
			t.Fatal(r)
		}
	}()

	testRoot := "/Users/timour/golang/src/github.com/caelifer/dups/t/"
	i := 11

	results := make([]string, 0, 256)
	testdir := testRoot + strconv.Itoa(i)

	nprocs := runtime.NumCPU()
	t.Logf("Using %d CPU threads\n", nprocs)

	err := fstree.Walk(balancer.NewWorkQueue(nprocs), testdir, func(path string, info os.FileInfo, err error) error {
		results = append(results, path)
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	t.Logf("Results for %d entries:\n%s\n", i+1, strings.Join(results, "\n"))
	if n := len(results); n != i+1 {
		t.Errorf("Expected %d nodes, got %d\n", i+1, n)
	}
}

func TestTreeWalk12(t *testing.T) {
	// Recover from panic
	defer func() {
		if r := recover(); r != nil {
			t.Fatal(r)
		}
	}()

	testRoot := "/Users/timour/golang/src/github.com/caelifer/dups/t/"
	i := 12

	results := make([]string, 0, 256)
	testdir := testRoot + strconv.Itoa(i)

	nprocs := runtime.NumCPU()
	t.Logf("Using %d CPU threads\n", nprocs)

	err := fstree.Walk(balancer.NewWorkQueue(nprocs), testdir, func(path string, info os.FileInfo, err error) error {
		results = append(results, path)
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	t.Logf("Results for %d entries:\n%s\n", i+1, strings.Join(results, "\n"))
	if n := len(results); n != i+1 {
		t.Errorf("Expected %d nodes, got %d\n", i+1, n)
	}
}

func TestTreeWalk13(t *testing.T) {
	// Recover from panic
	defer func() {
		if r := recover(); r != nil {
			t.Fatal(r)
		}
	}()

	testRoot := "/Users/timour/golang/src/github.com/caelifer/dups/t/"
	i := 13

	results := make([]string, 0, 256)
	testdir := testRoot + strconv.Itoa(i)

	nprocs := runtime.NumCPU()
	t.Logf("Using %d CPU threads\n", nprocs)

	err := fstree.Walk(balancer.NewWorkQueue(nprocs), testdir, func(path string, info os.FileInfo, err error) error {
		results = append(results, path)
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	t.Logf("Results for %d entries:\n%s\n", i+1, strings.Join(results, "\n"))
	if n := len(results); n != i+1 {
		t.Errorf("Expected %d nodes, got %d\n", i+1, n)
	}
}

func TestTreeWalkEmpty(t *testing.T) {
	// Recover from panic
	defer func() {
		if r := recover(); r != nil {
			t.Fatal(r)
		}
	}()

	testRoot := "/Users/timour/golang/src/github.com/caelifer/dups/t/c"
	i := 0

	results := make([]string, 0, 256)
	testdir := testRoot

	nprocs := runtime.NumCPU()
	t.Logf("Using %d CPU threads\n", nprocs)

	err := fstree.Walk(balancer.NewWorkQueue(nprocs), testdir, func(path string, info os.FileInfo, err error) error {
		results = append(results, path)
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	t.Logf("Results for %d entries:\n%s\n", i+1, strings.Join(results, "\n"))
	if n := len(results); n != i+1 {
		t.Errorf("Expected %d nodes, got %d\n", i+1, n)
	}
}
<commit_msg>Fixed test to use scheduler<commit_after>package fstree_test

import (
	"os"
	"runtime"
	"strconv"
	"strings"
	"testing"

	"github.com/caelifer/dups/fstree"
	"github.com/caelifer/scheduler"
)

func TestTreeWalk(t *testing.T) {
	// Recover from panic
	defer func() {
		if r := recover(); r != nil {
			t.Fatal(r)
		}
	}()

	testRoot := "/Users/timour/golang/src/github.com/caelifer/dups/t/"

	nprocs := runtime.NumCPU()
	t.Logf("Using %d CPU threads\n", nprocs)

	for i := 10; i < 20; i++ {
		results := make([]string, 0, 256)
		testdir := testRoot + strconv.Itoa(i)

		err := fstree.Walk(scheduler.New(nprocs, 1024), testdir, func(path string, info os.FileInfo, err error) error {
			results = append(results, path)
			return nil
		})

		if err != nil {
			t.Fatal(err)
		}

		t.Logf("Results for %d entries:\n%s\n", i+1, strings.Join(results, "\n"))
		if n := len(results); n != i+1 {
			t.Errorf("Expected %d nodes, got %d\n", i+1, n)
		}
	}
}

func TestTreeWalk11(t *testing.T) {
	// Recover from panic
	defer func() {
		if r := recover(); r != nil {
			t.Fatal(r)
		}
	}()

	testRoot := "/Users/timour/golang/src/github.com/caelifer/dups/t/"
	i := 11

	results := make([]string, 0, 256)
	testdir := testRoot + strconv.Itoa(i)

	nprocs := runtime.NumCPU()
	t.Logf("Using %d CPU threads\n", nprocs)

	err := fstree.Walk(scheduler.New(nprocs, 1024), testdir, func(path string, info os.FileInfo, err error) error {
		results = append(results, path)
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	t.Logf("Results for %d entries:\n%s\n", i+1, strings.Join(results, "\n"))
	if n := len(results); n != i+1 {
		t.Errorf("Expected %d nodes, got %d\n", i+1, n)
	}
}

func TestTreeWalk12(t *testing.T) {
	// Recover from panic
	defer func() {
		if r := recover(); r != nil {
			t.Fatal(r)
		}
	}()

	testRoot := "/Users/timour/golang/src/github.com/caelifer/dups/t/"
	i := 12

	results := make([]string, 0, 256)
	testdir := testRoot + strconv.Itoa(i)

	nprocs := runtime.NumCPU()
	t.Logf("Using %d CPU threads\n", nprocs)

	err := fstree.Walk(scheduler.New(nprocs, 1024), testdir, func(path string, info os.FileInfo, err error) error {
		results = append(results, path)
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	t.Logf("Results for %d entries:\n%s\n", i+1, strings.Join(results, "\n"))
	if n := len(results); n != i+1 {
		t.Errorf("Expected %d nodes, got %d\n", i+1, n)
	}
}

func TestTreeWalk13(t *testing.T) {
	// Recover from panic
	defer func() {
		if r := recover(); r != nil {
			t.Fatal(r)
		}
	}()

	testRoot := "/Users/timour/golang/src/github.com/caelifer/dups/t/"
	i := 13

	results := make([]string, 0, 256)
	testdir := testRoot + strconv.Itoa(i)

	nprocs := runtime.NumCPU()
	t.Logf("Using %d CPU threads\n", nprocs)

	err := fstree.Walk(scheduler.New(nprocs, 1024), testdir, func(path string, info os.FileInfo, err error) error {
		results = append(results, path)
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	t.Logf("Results for %d entries:\n%s\n", i+1, strings.Join(results, "\n"))
	if n := len(results); n != i+1 {
		t.Errorf("Expected %d nodes, got %d\n", i+1, n)
	}
}

func TestTreeWalkEmpty(t *testing.T) {
	// Recover from panic
	defer func() {
		if r := recover(); r != nil {
			t.Fatal(r)
		}
	}()

	testRoot := "/Users/timour/golang/src/github.com/caelifer/dups/t/c/a"
	i := 0

	results := make([]string, 0, 256)
	testdir := testRoot

	nprocs := runtime.NumCPU()
	t.Logf("Using %d CPU threads\n", nprocs)

	err := fstree.Walk(scheduler.New(nprocs, 1024), testdir, func(path string, info os.FileInfo, err error) error {
		results = append(results, path)
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	t.Logf("Results for %d entries:\n%s\n", i+1, strings.Join(results, "\n"))
	if n := len(results); n != i+1 {
		t.Errorf("Expected %d nodes, got %d\n", i+1, n)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Config is used for reading a config file and flags.
// Inspired from spf13/viper.
package config

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"gopkg.in/yaml.v2"
)

var (
	override = make(map[string]string)
	config   = make(map[string]string)
	defaults = make(map[string]string)

	configPath = filepath.Join(os.Getenv("HOME"), ".nehmconfig")

	ErrNotExist = errors.New("config file doesn't exist")
)

// Get has the behavior of returning the value associated with the first
// place from where it is set. Get will check value in the following order:
// flag, config file, defaults.
//
// Get returns a string. For a specific value you can use one of the Get____ methods.
func Get(key string) string {
	if value, exists := override[key]; exists {
		return value
	}
	if value, exists := config[key]; exists {
		return value
	}
	return defaults[key]
}

// ReadInConfig will discover and load the config file from disk.
// It will term the program, if there is an error.
func ReadInConfig() error {
	configFile, err := os.Open(configPath)
	if os.IsNotExist(err) {
		return ErrNotExist
	}
	if err != nil {
		return fmt.Errorf("couldn't open the config file: %v", err)
	}
	defer configFile.Close()

	configData, err := ioutil.ReadAll(configFile)
	if err != nil {
		return fmt.Errorf("couldn't read the config file: %v", err)
	}

	if err := yaml.Unmarshal(configData, config); err != nil {
		return fmt.Errorf("couldn't unmarshal the config file: %v", err)
	}

	return nil
}

// Set sets the value for the key in the override regiser.
func Set(key, value string) {
	override[key] = value
}

// SetDefault sets the value for the key in the default regiser.
func SetDefault(key, value string) {
	defaults[key] = value
}
<commit_msg>Fix comment in config.ReadInConfig<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Config is used for reading a config file and flags.
// Inspired from spf13/viper.
package config

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"gopkg.in/yaml.v2"
)

var (
	override = make(map[string]string)
	config   = make(map[string]string)
	defaults = make(map[string]string)

	configPath = filepath.Join(os.Getenv("HOME"), ".nehmconfig")

	ErrNotExist = errors.New("config file doesn't exist")
)

// Get has the behavior of returning the value associated with the first
// place from where it is set. Get will check value in the following order:
// flag, config file, defaults.
//
// Get returns a string. For a specific value you can use one of the Get____ methods.
func Get(key string) string {
	if value, exists := override[key]; exists {
		return value
	}
	if value, exists := config[key]; exists {
		return value
	}
	return defaults[key]
}

// ReadInConfig will discover and load the config file from disk, searching
// in the defined path.
func ReadInConfig() error {
	configFile, err := os.Open(configPath)
	if os.IsNotExist(err) {
		return ErrNotExist
	}
	if err != nil {
		return fmt.Errorf("couldn't open the config file: %v", err)
	}
	defer configFile.Close()

	configData, err := ioutil.ReadAll(configFile)
	if err != nil {
		return fmt.Errorf("couldn't read the config file: %v", err)
	}

	if err := yaml.Unmarshal(configData, config); err != nil {
		return fmt.Errorf("couldn't unmarshal the config file: %v", err)
	}

	return nil
}

// Set sets the value for the key in the override regiser.
func Set(key, value string) {
	override[key] = value
}

// SetDefault sets the value for the key in the default regiser.
func SetDefault(key, value string) {
	defaults[key] = value
}
<|endoftext|>


<commit_before>package config

import (
	"errors"
	"fmt"
	"net/url"
	"time"
)

const (
	ContentTypeJSON = "application/json"
)

const serverConfigFile = "petze.yml"

type Expect struct {
	Max      *int64
	Min      *int64
	Count    *int64
	Contains string
	Equals   interface{}
}

type Check struct {
	Comment     string
	Data        map[string]Expect
	Goquery     map[string]Expect
	Header      map[string][]string
	Duration    time.Duration
	StatusCode  int64
	ContentType string `yaml:"content-type"`
}

type Call struct {
	URI         string
	URL         string
	Method      string
	Data        interface{}
	ContentType string `yaml:"content-type"`
	Check       []Check
	Header      map[string][]string
}

// Service a service to monitor
type Service struct {
	ID       string
	Endpoint string
	Interval time.Duration
	Session  []Call
}

type Server struct {
	Address       string
	BasicAuthFile string
	TLS *struct {
		Address string
		Cert    string
		Key     string
	}
}

func (s *Service) GetURL() (u *url.URL, e error) {
	return url.Parse(s.Endpoint)
}

func (s *Service) IsValid() (valid bool, err error) {
	valid = false
	_, errURL := s.GetURL()
	if errURL != nil {

		err = errors.New("endpoint is invalid: " + errURL.Error())
		return
	}
	for callIndex, call := range s.Session {
		_, callErr := call.IsValid()
		if callErr != nil {
			err = errors.New(fmt.Sprint("invalid call in session @", callIndex, " : ", callErr))
			return
		}
	}
	valid = true
	return
}
func (c *Call) GetURL() (u *url.URL, e error) {
	return url.Parse(c.URI)
}

func (c *Call) IsValid() (valid bool, err error) {
	valid = true
	_, errURL := c.GetURL()
	if errURL != nil {
		err = errors.New("invalid uri " + c.URI + " : " + errURL.Error())
		return
	}
	return
}
<commit_msg>Reformat config file with gofmt<commit_after>package config

import (
	"errors"
	"fmt"
	"net/url"
	"time"
)

const (
	ContentTypeJSON = "application/json"
)

const serverConfigFile = "petze.yml"

type Expect struct {
	Max      *int64
	Min      *int64
	Count    *int64
	Contains string
	Equals   interface{}
}

type Check struct {
	Comment     string
	Data        map[string]Expect
	Goquery     map[string]Expect
	Header      map[string][]string
	Duration    time.Duration
	StatusCode  int64
	ContentType string `yaml:"content-type"`
}

type Call struct {
	URI         string
	URL         string
	Method      string
	Data        interface{}
	ContentType string `yaml:"content-type"`
	Check       []Check
	Header      map[string][]string
}

// Service a service to monitor
type Service struct {
	ID       string
	Endpoint string
	Interval time.Duration
	Session  []Call
}

type Server struct {
	Address       string
	BasicAuthFile string
	TLS           *struct {
		Address string
		Cert    string
		Key     string
	}
}

func (s *Service) GetURL() (u *url.URL, e error) {
	return url.Parse(s.Endpoint)
}

func (s *Service) IsValid() (valid bool, err error) {
	valid = false
	_, errURL := s.GetURL()
	if errURL != nil {

		err = errors.New("endpoint is invalid: " + errURL.Error())
		return
	}
	for callIndex, call := range s.Session {
		_, callErr := call.IsValid()
		if callErr != nil {
			err = errors.New(fmt.Sprint("invalid call in session @", callIndex, " : ", callErr))
			return
		}
	}
	valid = true
	return
}
func (c *Call) GetURL() (u *url.URL, e error) {
	return url.Parse(c.URI)
}

func (c *Call) IsValid() (valid bool, err error) {
	valid = true
	_, errURL := c.GetURL()
	if errURL != nil {
		err = errors.New("invalid uri " + c.URI + " : " + errURL.Error())
		return
	}
	return
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/mholt/caddy"
)

// Config is a configuration for browsing in a particualr path.
type Config struct {
	PathScope   string
	Root        http.FileSystem
	BaseURL     string
	StyleSheet  string // Costum stylesheet
	FrontMatter string // Default frontmatter to save files in
	HugoEnabled bool   // Enables the Hugo plugin for File Manager
}

// Parse parses the configuration set by the user so it can
// be used by the middleware
func Parse(c *caddy.Controller) ([]Config, error) {
	var configs []Config

	appendConfig := func(cfg Config) error {
		for _, c := range configs {
			if c.PathScope == cfg.PathScope {
				return fmt.Errorf("duplicate file managing config for %s", c.PathScope)
			}
		}
		configs = append(configs, cfg)
		return nil
	}

	for c.Next() {
		var cfg = Config{PathScope: ".", BaseURL: "", FrontMatter: "yaml", HugoEnabled: false}
		for c.NextBlock() {
			switch c.Val() {
			case "show":
				if !c.NextArg() {
					return configs, c.ArgErr()
				}
				cfg.PathScope = c.Val()
				cfg.PathScope = strings.TrimSuffix(cfg.PathScope, "/")
			case "on":
				if !c.NextArg() {
					return configs, c.ArgErr()
				}
				cfg.BaseURL = c.Val()
				cfg.BaseURL = strings.TrimPrefix(cfg.BaseURL, "/")
				cfg.BaseURL = strings.TrimSuffix(cfg.BaseURL, "/")
				cfg.BaseURL = "/" + cfg.BaseURL
			case "styles":
				if !c.NextArg() {
					return configs, c.ArgErr()
				}
				tplBytes, err := ioutil.ReadFile(c.Val())
				if err != nil {
					return configs, err
				}
				cfg.StyleSheet = string(tplBytes)
			}
		}

		cfg.Root = http.Dir(cfg.PathScope)
		if err := appendConfig(cfg); err != nil {
			return configs, err
		}
	}

	return configs, nil
}
<commit_msg>add frontmatter to options<commit_after>package config

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/mholt/caddy"
)

// Config is a configuration for browsing in a particualr path.
type Config struct {
	PathScope   string
	Root        http.FileSystem
	BaseURL     string
	StyleSheet  string // Costum stylesheet
	FrontMatter string // Default frontmatter to save files in
	HugoEnabled bool   // Enables the Hugo plugin for File Manager
}

// Parse parses the configuration set by the user so it can
// be used by the middleware
func Parse(c *caddy.Controller) ([]Config, error) {
	var configs []Config

	appendConfig := func(cfg Config) error {
		for _, c := range configs {
			if c.PathScope == cfg.PathScope {
				return fmt.Errorf("duplicate file managing config for %s", c.PathScope)
			}
		}
		configs = append(configs, cfg)
		return nil
	}

	for c.Next() {
		var cfg = Config{PathScope: ".", BaseURL: "", FrontMatter: "yaml", HugoEnabled: false}
		for c.NextBlock() {
			switch c.Val() {
			case "show":
				if !c.NextArg() {
					return configs, c.ArgErr()
				}
				cfg.PathScope = c.Val()
				cfg.PathScope = strings.TrimSuffix(cfg.PathScope, "/")
			case "frontmatter":
				if !c.NextArg() {
					return configs, c.ArgErr()
				}
				cfg.FrontMatter = c.Val()
				if cfg.FrontMatter != "yaml" && cfg.FrontMatter != "json" && cfg.FrontMatter != "toml" {
					return configs, c.Err("frontmatter type not supported")
				}
			case "on":
				if !c.NextArg() {
					return configs, c.ArgErr()
				}
				cfg.BaseURL = c.Val()
				cfg.BaseURL = strings.TrimPrefix(cfg.BaseURL, "/")
				cfg.BaseURL = strings.TrimSuffix(cfg.BaseURL, "/")
				cfg.BaseURL = "/" + cfg.BaseURL
			case "styles":
				if !c.NextArg() {
					return configs, c.ArgErr()
				}
				tplBytes, err := ioutil.ReadFile(c.Val())
				if err != nil {
					return configs, err
				}
				cfg.StyleSheet = string(tplBytes)
			}
		}

		cfg.Root = http.Dir(cfg.PathScope)
		if err := appendConfig(cfg); err != nil {
			return configs, err
		}
	}

	return configs, nil
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	flag "github.com/spf13/pflag"
	"github.com/tendermint/confer"
)

var rootDir string
var App *confer.Config

// NOTE: If you change this, maybe also change initDefaults()
var defaultConfig = `# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

Network =         "tendermint_testnet0"
ListenAddr =      "0.0.0.0:8080"
# First node to connect to.  Command-line overridable.
SeedNode =        "23.239.22.253:8080"

[DB]
# The only other available backend is "memdb"
Backend =         "leveldb"
# Dir =             "~/.tendermint/data"

[Log.Stdout]
Level =           "info"

[Log.File]
Level =           "debug"
# Dir =             "~/.tendermint/log"

[RPC.HTTP]
# For the RPC API HTTP server.  Port required.
ListenAddr = "127.0.0.1:8081"

[Alert]
# TODO: Document options

[SMTP]
# TODO: Document options
`

// NOTE: If you change this, maybe also change defaultConfig
func initDefaults() {
	App.SetDefault("Network", "tendermint_testnet0")
	App.SetDefault("ListenAddr", "0.0.0.0:8080")
	App.SetDefault("DB.Backend", "leveldb")
	App.SetDefault("DB.Dir", rootDir+"/data")
	App.SetDefault("Log.Stdout.Level", "info")
	App.SetDefault("Log.File.Dir", rootDir+"/log")
	App.SetDefault("Log.File.Level", "debug")
	App.SetDefault("RPC.HTTP.ListenAddr", "127.0.0.1:8081")

	App.SetDefault("GenesisFile", rootDir+"/genesis.json")
	App.SetDefault("AddrBookFile", rootDir+"/addrbook.json")
	App.SetDefault("PrivValidatorfile", rootDir+"/priv_validator.json")
}

func init() {

	// Get RootDir
	rootDir = os.Getenv("TMROOT")
	if rootDir == "" {
		rootDir = os.Getenv("HOME") + "/.tendermint"
	}
	configFile := rootDir + "/config.toml"

	// Write default config file if missing.
	if _, err := os.Stat(configFile); os.IsNotExist(err) {
		if strings.Index(configFile, "/") != -1 {
			err := os.MkdirAll(filepath.Dir(configFile), 0700)
			if err != nil {
				fmt.Printf("Could not create directory: %v", err)
				os.Exit(1)
			}
		}
		err := ioutil.WriteFile(configFile, []byte(defaultConfig), 0600)
		if err != nil {
			fmt.Printf("Could not write config file: %v", err)
			os.Exit(1)
		}
		fmt.Printf("Config file written to %v. Please edit & run again\n", configFile)
		os.Exit(1)
	}

	// Initialize Config
	App = confer.NewConfig()
	initDefaults()
	paths := []string{configFile}
	if err := App.ReadPaths(paths...); err != nil {
		log.Warn("Error reading configuration", "paths", paths, "error", err)
	}

	// Confused?
	// App.Debug()
}

func ParseFlags(args []string) {
	var flags = flag.NewFlagSet("main", flag.ExitOnError)
	var printHelp = false

	// Declare flags
	flags.BoolVar(&printHelp, "help", false, "Print this help message.")
	flags.String("listen_addr", App.GetString("ListenAddr"), "Listen address. (0.0.0.0:0 means any interface, any port)")
	flags.String("seed_node", App.GetString("SeedNode"), "Address of seed node")
	flags.String("rpc_http_listen_addr", App.GetString("RPC.HTTP.ListenAddr"), "RPC listen address. Port required")
	flags.Parse(args)
	if printHelp {
		flags.PrintDefaults()
		os.Exit(0)
	}

	// Merge parsed flag values onto App.
	App.BindPFlag("ListenAddr", flags.Lookup("listen_addr"))
	App.BindPFlag("SeedNode", flags.Lookup("seed_node"))
	App.BindPFlag("RPC.HTTP.ListenAddr", flags.Lookup("rpc_http_listen_addr"))

	// Confused?
	//App.Debug()
}
<commit_msg>config: hardcode default genesis.json<commit_after>package config

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	flag "github.com/spf13/pflag"
	"github.com/tendermint/confer"
)

var rootDir string
var App *confer.Config

// NOTE: If you change this, maybe also change initDefaults()
var defaultConfig = `# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

Network =         "tendermint_testnet0"
ListenAddr =      "0.0.0.0:8080"
# First node to connect to.  Command-line overridable.
SeedNode =        "23.239.22.253:8080"

[DB]
# The only other available backend is "memdb"
Backend =         "leveldb"
# Dir =             "~/.tendermint/data"

[Log.Stdout]
Level =           "info"

[Log.File]
Level =           "debug"
# Dir =             "~/.tendermint/log"

[RPC.HTTP]
# For the RPC API HTTP server.  Port required.
ListenAddr = "127.0.0.1:8081"

[Alert]
# TODO: Document options

[SMTP]
# TODO: Document options
`

var defaultGenesis = `
{
  "Accounts": [
    {
      "Address": "553722287BF1230C081C270908C1F453E7D1C397",
      "Amount":  200000000
    },
    {
      "Address": "AC89A6DDF4C309A89A2C4078CE409A5A7B282270",
      "Amount":  200000000
    }
  ],
  "Validators": [
    {
      "PubKey": [1, "932A857D334BA5A38DD8E0D9CDE9C84687C21D0E5BEE64A1EDAB9C6C32344F1A"],
      "Amount": 100000000,
      "UnbondTo": [
        {
          "Address": "553722287BF1230C081C270908C1F453E7D1C397",
          "Amount":  100000000
        }
      ]
    }
  ]
}
`

// NOTE: If you change this, maybe also change defaultConfig
func initDefaults() {
	App.SetDefault("Network", "tendermint_testnet0")
	App.SetDefault("ListenAddr", "0.0.0.0:8080")
	App.SetDefault("DB.Backend", "leveldb")
	App.SetDefault("DB.Dir", rootDir+"/data")
	App.SetDefault("Log.Stdout.Level", "info")
	App.SetDefault("Log.File.Dir", rootDir+"/log")
	App.SetDefault("Log.File.Level", "debug")
	App.SetDefault("RPC.HTTP.ListenAddr", "127.0.0.1:8081")

	App.SetDefault("GenesisFile", rootDir+"/genesis.json")
	App.SetDefault("AddrBookFile", rootDir+"/addrbook.json")
	App.SetDefault("PrivValidatorfile", rootDir+"/priv_validator.json")
}

// Check if a file exists; if not, ensure the directory is made and write the file
func checkWriteFile(configFile, contents string) {
	if _, err := os.Stat(configFile); os.IsNotExist(err) {
		if strings.Index(configFile, "/") != -1 {
			err := os.MkdirAll(filepath.Dir(configFile), 0700)
			if err != nil {
				fmt.Printf("Could not create directory: %v", err)
				os.Exit(1)
			}
		}
		err := ioutil.WriteFile(configFile, []byte(contents), 0600)
		if err != nil {
			fmt.Printf("Could not write config file: %v", err)
			os.Exit(1)
		}
		fmt.Printf("Config file written to %v.\n", configFile)
	}
}

func init() {

	// Get RootDir
	rootDir = os.Getenv("TMROOT")
	if rootDir == "" {
		rootDir = os.Getenv("HOME") + "/.tendermint"
	}
	configFile := path.Join(rootDir, "config.toml")
	genesisFile := path.Join(rootDir, "genesis.json")

	// Write default config file if missing.
	checkWriteFile(configFile, defaultConfig)
	checkWriteFile(genesisFile, defaultGenesis)

	// Initialize Config
	App = confer.NewConfig()
	initDefaults()
	paths := []string{configFile}
	if err := App.ReadPaths(paths...); err != nil {
		log.Warn("Error reading configuration", "paths", paths, "error", err)
	}

	// Confused?
	// App.Debug()
}

func ParseFlags(args []string) {
	var flags = flag.NewFlagSet("main", flag.ExitOnError)
	var printHelp = false

	// Declare flags
	flags.BoolVar(&printHelp, "help", false, "Print this help message.")
	flags.String("listen_addr", App.GetString("ListenAddr"), "Listen address. (0.0.0.0:0 means any interface, any port)")
	flags.String("seed_node", App.GetString("SeedNode"), "Address of seed node")
	flags.String("rpc_http_listen_addr", App.GetString("RPC.HTTP.ListenAddr"), "RPC listen address. Port required")
	flags.Parse(args)
	if printHelp {
		flags.PrintDefaults()
		os.Exit(0)
	}

	// Merge parsed flag values onto App.
	App.BindPFlag("ListenAddr", flags.Lookup("listen_addr"))
	App.BindPFlag("SeedNode", flags.Lookup("seed_node"))
	App.BindPFlag("RPC.HTTP.ListenAddr", flags.Lookup("rpc_http_listen_addr"))

	// Confused?
	//App.Debug()
}
<|endoftext|>


<commit_before>// Copyright 2014 The Chihaya Authors. All rights reserved.
// Use of this source code is governed by the BSD 2-Clause license,
// which can be found in the LICENSE file.

// Package config implements the configuration for a BitTorrent tracker
package config

import (
	"encoding/json"
	"errors"
	"io"
	"os"
	"time"
)

// ErrMissingRequiredParam is used by drivers to indicate that an entry required
// to be within the DriverConfig.Params map is not present.
var ErrMissingRequiredParam = errors.New("A parameter that was required by a driver is not present")

// Duration wraps a time.Duration and adds JSON marshalling.
type Duration struct{ time.Duration }

// MarshalJSON transforms a duration into JSON.
func (d *Duration) MarshalJSON() ([]byte, error) {
	return json.Marshal(d.String())
}

// UnmarshalJSON transform JSON into a Duration.
func (d *Duration) UnmarshalJSON(b []byte) error {
	var str string
	err := json.Unmarshal(b, &str)
	d.Duration, err = time.ParseDuration(str)
	return err
}

// DriverConfig is the configuration used to connect to a tracker.Driver or
// a backend.Driver.
type DriverConfig struct {
	Name   string            `json:"driver"`
	Params map[string]string `json:"params,omitempty"`
}

// Config is a configuration for a Server.
type Config struct {
	Addr    string       `json:"addr"`
	Tracker DriverConfig `json:"tracker"`
	Backend DriverConfig `json:"backend"`

	Private   bool `json:"private"`
	Freeleech bool `json:"freeleech"`
	Whitelist bool `json:"whitelist"`

	PurgeInactiveTorrents bool `json:"purge_inactive_torrents"`

	Announce        Duration `json:"announce"`
	MinAnnounce     Duration `json:"min_announce"`
	RequestTimeout  Duration `json:"request_timeout"`
	NumWantFallback int      `json:"default_num_want"`

	PreferredSubnet     bool `json:"preferred_subnet,omitempty"`
	PreferredIPv4Subnet int  `json:"preferred_ipv4_subnet,omitempty"`
	PreferredIPv6Subnet int  `json:"preferred_ipv6_subnet,omitempty"`
}

// DefaultConfig is a configuration that can be used as a fallback value.
var DefaultConfig = Config{
	Addr: "127.0.0.1:6881",

	Tracker: DriverConfig{
		Name: "memory",
	},

	Backend: DriverConfig{
		Name: "noop",
	},

	Private:   false,
	Freeleech: false,
	Whitelist: false,

	PurgeInactiveTorrents: true,

	Announce:        Duration{30 * time.Minute},
	MinAnnounce:     Duration{15 * time.Minute},
	RequestTimeout:  Duration{10 * time.Second},
	NumWantFallback: 50,
}

// Open is a shortcut to open a file, read it, and generate a Config.
// It supports relative and absolute paths. Given "", it returns DefaultConfig.
func Open(path string) (*Config, error) {
	if path == "" {
		return &DefaultConfig, nil
	}

	f, err := os.Open(os.ExpandEnv(path))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	conf, err := Decode(f)
	if err != nil {
		return nil, err
	}
	return conf, nil
}

// Decode casts an io.Reader into a JSONDecoder and decodes it into a *Config.
func Decode(r io.Reader) (*Config, error) {
	conf := &Config{}
	err := json.NewDecoder(r).Decode(conf)
	if err != nil {
		return nil, err
	}
	return conf, nil
}
<commit_msg>moved together bools<commit_after>// Copyright 2014 The Chihaya Authors. All rights reserved.
// Use of this source code is governed by the BSD 2-Clause license,
// which can be found in the LICENSE file.

// Package config implements the configuration for a BitTorrent tracker
package config

import (
	"encoding/json"
	"errors"
	"io"
	"os"
	"time"
)

// ErrMissingRequiredParam is used by drivers to indicate that an entry required
// to be within the DriverConfig.Params map is not present.
var ErrMissingRequiredParam = errors.New("A parameter that was required by a driver is not present")

// Duration wraps a time.Duration and adds JSON marshalling.
type Duration struct{ time.Duration }

// MarshalJSON transforms a duration into JSON.
func (d *Duration) MarshalJSON() ([]byte, error) {
	return json.Marshal(d.String())
}

// UnmarshalJSON transform JSON into a Duration.
func (d *Duration) UnmarshalJSON(b []byte) error {
	var str string
	err := json.Unmarshal(b, &str)
	d.Duration, err = time.ParseDuration(str)
	return err
}

// DriverConfig is the configuration used to connect to a tracker.Driver or
// a backend.Driver.
type DriverConfig struct {
	Name   string            `json:"driver"`
	Params map[string]string `json:"params,omitempty"`
}

// Config is a configuration for a Server.
type Config struct {
	Addr    string       `json:"addr"`
	Tracker DriverConfig `json:"tracker"`
	Backend DriverConfig `json:"backend"`

	Private               bool `json:"private"`
	Freeleech             bool `json:"freeleech"`
	Whitelist             bool `json:"whitelist"`
	PurgeInactiveTorrents bool `json:"purge_inactive_torrents"`

	Announce        Duration `json:"announce"`
	MinAnnounce     Duration `json:"min_announce"`
	RequestTimeout  Duration `json:"request_timeout"`
	NumWantFallback int      `json:"default_num_want"`

	PreferredSubnet     bool `json:"preferred_subnet,omitempty"`
	PreferredIPv4Subnet int  `json:"preferred_ipv4_subnet,omitempty"`
	PreferredIPv6Subnet int  `json:"preferred_ipv6_subnet,omitempty"`
}

// DefaultConfig is a configuration that can be used as a fallback value.
var DefaultConfig = Config{
	Addr: "127.0.0.1:6881",

	Tracker: DriverConfig{
		Name: "memory",
	},

	Backend: DriverConfig{
		Name: "noop",
	},

	Private:               false,
	Freeleech:             false,
	Whitelist:             false,
	PurgeInactiveTorrents: true,

	Announce:        Duration{30 * time.Minute},
	MinAnnounce:     Duration{15 * time.Minute},
	RequestTimeout:  Duration{10 * time.Second},
	NumWantFallback: 50,
}

// Open is a shortcut to open a file, read it, and generate a Config.
// It supports relative and absolute paths. Given "", it returns DefaultConfig.
func Open(path string) (*Config, error) {
	if path == "" {
		return &DefaultConfig, nil
	}

	f, err := os.Open(os.ExpandEnv(path))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	conf, err := Decode(f)
	if err != nil {
		return nil, err
	}
	return conf, nil
}

// Decode casts an io.Reader into a JSONDecoder and decodes it into a *Config.
func Decode(r io.Reader) (*Config, error) {
	conf := &Config{}
	err := json.NewDecoder(r).Decode(conf)
	if err != nil {
		return nil, err
	}
	return conf, nil
}
<|endoftext|>


<commit_before>package config

import (
	"github.com/freeformz/shh/utils"
	"time"
)

const (
	VERSION              = "0.0.23"
	DEFAULT_INTERVAL     = "10s"                                                              // Default tick interval for pollers
	DEFAULT_OUTPUTTER    = "stdoutl2metder"                                                   // Default outputter
	DEFAULT_POLLERS      = "conntrack,cpu,df,disk,listen,load,mem,nif,ntpdate,processes,self" // Default pollers
	DEFAULT_PROFILE_PORT = "0"
	DEFAULT_DF_TYPES     = "btrfs,ext3,ext4,tmpfs,xfs"
	DEFAULT_NIF_DEVICES  = "eth0,lo"
)

var (
	Interval            = utils.GetEnvWithDefaultDuration("SHH_INTERVAL", DEFAULT_INTERVAL)                      // Polling Interval
	Outputter           = utils.GetEnvWithDefault("SHH_OUTPUTTER", DEFAULT_OUTPUTTER)                            // Outputter
	Pollers             = utils.GetEnvWithDefaultStrings("SHH_POLLERS", DEFAULT_POLLERS)                         // Pollers to poll
	Source              = utils.GetEnvWithDefault("SHH_SOURCE", "")                                              // Source to emit
	Prefix              = utils.GetEnvWithDefault("SHH_PREFIX", "")                                              // Metric prefix to use
	ProfilePort         = utils.GetEnvWithDefault("SHH_PROFILE_PORT", DEFAULT_PROFILE_PORT)                      // Profile Port
	DfTypes             = utils.GetEnvWithDefaultStrings("SHH_DF_TYPES", DEFAULT_DF_TYPES)                       // Default DF types
	Listen              = utils.GetEnvWithDefault("SHH_LISTEN", "unix,/tmp/shh")                                 // Default network socket info for listen
	NifDevices          = utils.GetEnvWithDefaultStrings("SHH_NIF_DEVICES", DEFAULT_NIF_DEVICES)                 // Devices to poll
	NtpdateServers      = utils.GetEnvWithDefaultStrings("SHH_NTPDATE_SERVERS", "0.pool.ntp.org,1.pool.ntp.org") // NTP Servers
	LibratoUser         = utils.GetEnvWithDefault("SHH_LIBRATO_USER", "")                                        // The Librato API User
	LibratoToken        = utils.GetEnvWithDefault("SHH_LIBRATO_TOKEN", "")                                       // The Librato API TOken
	LibratoBatchSize    = utils.GetEnvWithDefaultInt("SHH_LIBRATO_BATCH_SIZE", 50)                               // The max number of metrics to submit in a single request
	LibratoBatchTimeout = utils.GetEnvWithDefaultDuration("SHH_LIBRATO_BATCH_TIMEOUT", "500ms")                  // The max time metrics will sit un-delivered

	Start = time.Now() // Start time
)
<commit_msg>bump version<commit_after>package config

import (
	"github.com/freeformz/shh/utils"
	"time"
)

const (
	VERSION              = "0.0.24"
	DEFAULT_INTERVAL     = "10s"                                                              // Default tick interval for pollers
	DEFAULT_OUTPUTTER    = "stdoutl2metder"                                                   // Default outputter
	DEFAULT_POLLERS      = "conntrack,cpu,df,disk,listen,load,mem,nif,ntpdate,processes,self" // Default pollers
	DEFAULT_PROFILE_PORT = "0"
	DEFAULT_DF_TYPES     = "btrfs,ext3,ext4,tmpfs,xfs"
	DEFAULT_NIF_DEVICES  = "eth0,lo"
)

var (
	Interval            = utils.GetEnvWithDefaultDuration("SHH_INTERVAL", DEFAULT_INTERVAL)                      // Polling Interval
	Outputter           = utils.GetEnvWithDefault("SHH_OUTPUTTER", DEFAULT_OUTPUTTER)                            // Outputter
	Pollers             = utils.GetEnvWithDefaultStrings("SHH_POLLERS", DEFAULT_POLLERS)                         // Pollers to poll
	Source              = utils.GetEnvWithDefault("SHH_SOURCE", "")                                              // Source to emit
	Prefix              = utils.GetEnvWithDefault("SHH_PREFIX", "")                                              // Metric prefix to use
	ProfilePort         = utils.GetEnvWithDefault("SHH_PROFILE_PORT", DEFAULT_PROFILE_PORT)                      // Profile Port
	DfTypes             = utils.GetEnvWithDefaultStrings("SHH_DF_TYPES", DEFAULT_DF_TYPES)                       // Default DF types
	Listen              = utils.GetEnvWithDefault("SHH_LISTEN", "unix,/tmp/shh")                                 // Default network socket info for listen
	NifDevices          = utils.GetEnvWithDefaultStrings("SHH_NIF_DEVICES", DEFAULT_NIF_DEVICES)                 // Devices to poll
	NtpdateServers      = utils.GetEnvWithDefaultStrings("SHH_NTPDATE_SERVERS", "0.pool.ntp.org,1.pool.ntp.org") // NTP Servers
	LibratoUser         = utils.GetEnvWithDefault("SHH_LIBRATO_USER", "")                                        // The Librato API User
	LibratoToken        = utils.GetEnvWithDefault("SHH_LIBRATO_TOKEN", "")                                       // The Librato API TOken
	LibratoBatchSize    = utils.GetEnvWithDefaultInt("SHH_LIBRATO_BATCH_SIZE", 50)                               // The max number of metrics to submit in a single request
	LibratoBatchTimeout = utils.GetEnvWithDefaultDuration("SHH_LIBRATO_BATCH_TIMEOUT", "500ms")                  // The max time metrics will sit un-delivered

	Start = time.Now() // Start time
)
<|endoftext|>


<commit_before>package proxyconfig

import (
	"fmt"
	"io/ioutil"

	"github.com/jacksontj/promxy/servergroup"
	"github.com/prometheus/prometheus/config"

	"gopkg.in/yaml.v2"
)

func ConfigFromFile(path string) (*Config, error) {
	// load the config file
	cfg := &Config{}
	configBytes, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("Error loading config: %v", err)
	}
	err = yaml.Unmarshal([]byte(configBytes), &cfg)
	if err != nil {
		return nil, fmt.Errorf("Error unmarshaling config: %v", err)
	}

	return cfg, nil
}

// Common configuration for all storage nodes
type Config struct {
	PromConfig config.Config `yaml:",inline"`

	// Our own configs
	ServerGroups []*servergroup.Config `yaml:"server_groups"`
}
<commit_msg>Have default prom config<commit_after>package proxyconfig

import (
	"fmt"
	"io/ioutil"

	"github.com/jacksontj/promxy/servergroup"
	"github.com/prometheus/prometheus/config"

	"gopkg.in/yaml.v2"
)

func ConfigFromFile(path string) (*Config, error) {
	// load the config file
	cfg := &Config{
		PromConfig: config.DefaultConfig,
	}
	configBytes, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("Error loading config: %v", err)
	}
	err = yaml.Unmarshal([]byte(configBytes), &cfg)
	if err != nil {
		return nil, fmt.Errorf("Error unmarshaling config: %v", err)
	}

	return cfg, nil
}

// Common configuration for all storage nodes
type Config struct {
	PromConfig config.Config `yaml:",inline"`

	// Our own configs
	ServerGroups []*servergroup.Config `yaml:"server_groups"`
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2015 Ludovic Fauvet
// Licensed under the MIT license

package config

import (
	"fmt"
	"github.com/etix/mirrorbits/core"
	"github.com/op/go-logging"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"strings"
	"sync"
)

var (
	log = logging.MustGetLogger("main")

	defaultConfig = configuration{
		Repository:             "",
		Templates:              "",
		OutputMode:             "auto",
		ListenAddress:          ":8080",
		Gzip:                   false,
		RedisAddress:           "127.0.0.1:6379",
		RedisPassword:          "",
		LogDir:                 "",
		GeoipDatabasePath:      "/usr/share/GeoIP/",
		ConcurrentSync:         2,
		ScanInterval:           30,
		CheckInterval:          1,
		RepositoryScanInterval: 5,
		Hashes: hashing{
			SHA1:   true,
			SHA256: false,
			MD5:    false,
		},
		DisallowRedirects:       false,
		WeightDistributionRange: 1.5,
		DisableOnMissingFile:    false,
	}
	config      *configuration
	configMutex sync.RWMutex
)

type configuration struct {
	Repository              string     `yaml:"Repository"`
	Templates               string     `yaml:"Templates"`
	OutputMode              string     `yaml:"OutputMode"`
	ListenAddress           string     `yaml:"ListenAddress"`
	Gzip                    bool       `yaml:"Gzip"`
	RedisAddress            string     `yaml:"RedisAddress"`
	RedisPassword           string     `yaml:"RedisPassword"`
	LogDir                  string     `yaml:"LogDir"`
	GeoipDatabasePath       string     `yaml:"GeoipDatabasePath"`
	ConcurrentSync          int        `yaml:"ConcurrentSync"`
	ScanInterval            int        `yaml:"ScanInterval"`
	CheckInterval           int        `yaml:"CheckInterval"`
	RepositoryScanInterval  int        `yaml:"RepositoryScanInterval"`
	Hashes                  hashing    `yaml:"Hashes"`
	DisallowRedirects       bool       `yaml:"DisallowRedirects"`
	WeightDistributionRange float32    `yaml:"WeightDistributionRange"`
	DisableOnMissingFile    bool       `yaml:"DisableOnMissingFile"`
	Fallbacks               []fallback `yaml:"Fallbacks"`

	RedisSentinelMasterName string      `yaml:"RedisSentinelMasterName"`
	RedisSentinels          []sentinels `yaml:"RedisSentinels"`
}

type fallback struct {
	Url           string `yaml:"URL"`
	CountryCode   string `yaml:"CountryCode"`
	ContinentCode string `yaml:"ContinentCode"`
}

type sentinels struct {
	Host string `yaml:"Host"`
}

type hashing struct {
	SHA1   bool `yaml:"SHA1"`
	SHA256 bool `yaml:"SHA256"`
	MD5    bool `yaml:"MD5"`
}

// LoadConfig loads the configuration file if it has not yet been loaded
func LoadConfig() {
	if config != nil {
		return
	}
	err := ReloadConfig()
	if err != nil {
		log.Fatal(err)
	}
}

// ReloadConfig reloads the configuration file and update it globally
func ReloadConfig() error {
	if core.ConfigFile == "" {
		if fileExists("./mirrorbits.conf") {
			core.ConfigFile = "./mirrorbits.conf"
		} else if fileExists("/etc/mirrorbits.conf") {
			core.ConfigFile = "/etc/mirrorbits.conf"
		}
	}

	content, err := ioutil.ReadFile(core.ConfigFile)
	if err != nil {
		fmt.Println("Configuration could not be found.\n\tUse -config <path>")
		os.Exit(-1)
	}

	if os.Getenv("DEBUG") != "" {
		fmt.Println("Reading configuration from", core.ConfigFile)
	}

	c := defaultConfig

	// Overload the default configuration with the user's one
	err = yaml.Unmarshal(content, &c)
	if err != nil {
		return fmt.Errorf("%s in %s", err, core.ConfigFile)
	}

	// Sanitize
	if c.WeightDistributionRange <= 0 {
		return fmt.Errorf("WeightDistributionRange must be > 0")
	}
	if !isInSlice(c.OutputMode, []string{"auto", "json", "redirect"}) {
		return fmt.Errorf("Config: outputMode can only be set to 'auto', 'json' or 'redirect'")
	}
	c.Repository = strings.TrimRight(c.Repository, "/")
	if c.RepositoryScanInterval < 0 {
		c.RepositoryScanInterval = 0
	}

	if config != nil &&
		(c.RedisAddress != config.RedisAddress ||
			c.RedisPassword != config.RedisPassword ||
			!testSentinelsEq(c.RedisSentinels, config.RedisSentinels)) {
		// TODO reload redis connections
		// Currently established connections will be updated only in case of disconnection
	}

	configMutex.Lock()
	config = &c
	configMutex.Unlock()
	return nil
}

// GetConfig returns a pointer to a configuration object
// FIXME reading from the pointer could cause a race!
func GetConfig() *configuration {
	configMutex.RLock()
	defer configMutex.RUnlock()

	if config == nil {
		panic("Configuration not loaded")
	}

	return config
}

func fileExists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func testSentinelsEq(a, b []sentinels) bool {
	if len(a) != len(b) {
		return false
	}

	for i := range a {
		if a[i].Host != b[i].Host {
			return false
		}
	}

	return true
}

//DUPLICATE
func isInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}
<commit_msg>config: add a simple mechanism to subscribe to configuration reloads<commit_after>// Copyright (c) 2014-2015 Ludovic Fauvet
// Licensed under the MIT license

package config

import (
	"fmt"
	"github.com/etix/mirrorbits/core"
	"github.com/op/go-logging"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"strings"
	"sync"
)

var (
	log = logging.MustGetLogger("main")

	defaultConfig = configuration{
		Repository:             "",
		Templates:              "",
		OutputMode:             "auto",
		ListenAddress:          ":8080",
		Gzip:                   false,
		RedisAddress:           "127.0.0.1:6379",
		RedisPassword:          "",
		LogDir:                 "",
		GeoipDatabasePath:      "/usr/share/GeoIP/",
		ConcurrentSync:         2,
		ScanInterval:           30,
		CheckInterval:          1,
		RepositoryScanInterval: 5,
		Hashes: hashing{
			SHA1:   true,
			SHA256: false,
			MD5:    false,
		},
		DisallowRedirects:       false,
		WeightDistributionRange: 1.5,
		DisableOnMissingFile:    false,
	}
	config      *configuration
	configMutex sync.RWMutex

	subscribers     []chan bool
	subscribersLock sync.RWMutex
)

type configuration struct {
	Repository              string     `yaml:"Repository"`
	Templates               string     `yaml:"Templates"`
	OutputMode              string     `yaml:"OutputMode"`
	ListenAddress           string     `yaml:"ListenAddress"`
	Gzip                    bool       `yaml:"Gzip"`
	RedisAddress            string     `yaml:"RedisAddress"`
	RedisPassword           string     `yaml:"RedisPassword"`
	LogDir                  string     `yaml:"LogDir"`
	GeoipDatabasePath       string     `yaml:"GeoipDatabasePath"`
	ConcurrentSync          int        `yaml:"ConcurrentSync"`
	ScanInterval            int        `yaml:"ScanInterval"`
	CheckInterval           int        `yaml:"CheckInterval"`
	RepositoryScanInterval  int        `yaml:"RepositoryScanInterval"`
	Hashes                  hashing    `yaml:"Hashes"`
	DisallowRedirects       bool       `yaml:"DisallowRedirects"`
	WeightDistributionRange float32    `yaml:"WeightDistributionRange"`
	DisableOnMissingFile    bool       `yaml:"DisableOnMissingFile"`
	Fallbacks               []fallback `yaml:"Fallbacks"`

	RedisSentinelMasterName string      `yaml:"RedisSentinelMasterName"`
	RedisSentinels          []sentinels `yaml:"RedisSentinels"`
}

type fallback struct {
	Url           string `yaml:"URL"`
	CountryCode   string `yaml:"CountryCode"`
	ContinentCode string `yaml:"ContinentCode"`
}

type sentinels struct {
	Host string `yaml:"Host"`
}

type hashing struct {
	SHA1   bool `yaml:"SHA1"`
	SHA256 bool `yaml:"SHA256"`
	MD5    bool `yaml:"MD5"`
}

// LoadConfig loads the configuration file if it has not yet been loaded
func LoadConfig() {
	if config != nil {
		return
	}
	err := ReloadConfig()
	if err != nil {
		log.Fatal(err)
	}
}

// ReloadConfig reloads the configuration file and update it globally
func ReloadConfig() error {
	if core.ConfigFile == "" {
		if fileExists("./mirrorbits.conf") {
			core.ConfigFile = "./mirrorbits.conf"
		} else if fileExists("/etc/mirrorbits.conf") {
			core.ConfigFile = "/etc/mirrorbits.conf"
		}
	}

	content, err := ioutil.ReadFile(core.ConfigFile)
	if err != nil {
		fmt.Println("Configuration could not be found.\n\tUse -config <path>")
		os.Exit(-1)
	}

	if os.Getenv("DEBUG") != "" {
		fmt.Println("Reading configuration from", core.ConfigFile)
	}

	c := defaultConfig

	// Overload the default configuration with the user's one
	err = yaml.Unmarshal(content, &c)
	if err != nil {
		return fmt.Errorf("%s in %s", err, core.ConfigFile)
	}

	// Sanitize
	if c.WeightDistributionRange <= 0 {
		return fmt.Errorf("WeightDistributionRange must be > 0")
	}
	if !isInSlice(c.OutputMode, []string{"auto", "json", "redirect"}) {
		return fmt.Errorf("Config: outputMode can only be set to 'auto', 'json' or 'redirect'")
	}
	c.Repository = strings.TrimRight(c.Repository, "/")
	if c.RepositoryScanInterval < 0 {
		c.RepositoryScanInterval = 0
	}

	if config != nil &&
		(c.RedisAddress != config.RedisAddress ||
			c.RedisPassword != config.RedisPassword ||
			!testSentinelsEq(c.RedisSentinels, config.RedisSentinels)) {
		// TODO reload redis connections
		// Currently established connections will be updated only in case of disconnection
	}

	// Lock the pointer during the swap
	configMutex.Lock()
	config = &c
	configMutex.Unlock()

	// Notify all subscribers that the configuration has been reloaded
	notifySubscribers()

	return nil
}

// GetConfig returns a pointer to a configuration object
// FIXME reading from the pointer could cause a race!
func GetConfig() *configuration {
	configMutex.RLock()
	defer configMutex.RUnlock()

	if config == nil {
		panic("Configuration not loaded")
	}

	return config
}

func SubscribeConfig(subscriber chan bool) {
	subscribersLock.Lock()
	defer subscribersLock.Unlock()

	subscribers = append(subscribers, subscriber)
}

func notifySubscribers() {
	subscribersLock.RLock()
	defer subscribersLock.RUnlock()

	for _, subscriber := range subscribers {
		select {
		case subscriber <- true:
		default:
			// Don't block if the subscriber is unavailable
			// and discard the message.
		}
	}
}

func fileExists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func testSentinelsEq(a, b []sentinels) bool {
	if len(a) != len(b) {
		return false
	}

	for i := range a {
		if a[i].Host != b[i].Host {
			return false
		}
	}

	return true
}

//DUPLICATE
func isInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package config

import (
	"io/ioutil"

	"github.com/hashicorp/hcl"
)

func ReadConfig(filename string) (ServerConfig, error) {
	cfg := ServerConfig{}
	return cfg, cfg.Read(filename)
}

type ServerConfig struct {
	Bind []string                `hcl:"bind" json:"bind"`
	Cert []string                `hcl:"cert" json:"cert"`
	GRPC bool                    `hcl:"grpc" json:"grpc"`
	AppM []map[string]*AppConfig `hcl:"app" json:"app"`
	App  []*AppConfig            `hcl:"-" json:"-"`
}

func (this *ServerConfig) Read(filename string) error {
	in, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	if err := hcl.Unmarshal(in, this); err != nil {
		return err
	}

	this.link()

	return nil
}

func (this *ServerConfig) link() {
	for _, m := range this.AppM {
		for name, app := range m {
			app.link()
			app.server = this
			app.Name = name
			if app.Host == "" {
				app.Host = name
			}

			this.App = append(this.App, app)
		}
	}
}

type AppConfig struct {
	server *ServerConfig

	Name   string                    `hcl:"-" json:"-"`
	Host   string                    `hcl:"host" json:"host"`
	Bind   []string                  `hcl:"bind" json:"bind"`
	GRPC   *bool                     `hcl:"grpc" json:"grpc"`
	ProxyM []map[string]*ProxyConfig `hcl:"proxy" json:"proxy"`
	Proxy  []*ProxyConfig            `hcl:"-" json:"-"`
}

func (this *AppConfig) GetBind() []string {
	if len(this.Bind) == 0 {
		return this.server.Bind
	}

	return this.Bind
}

func (this *AppConfig) GetGRPC() bool {
	if this.GRPC == nil {
		return this.server.GRPC
	}

	return *this.GRPC
}

func (this *AppConfig) link() {
	for _, m := range this.ProxyM {
		for name, proxy := range m {
			proxy.link()
			proxy.app = this
			proxy.Name = name
			if proxy.URI == "" {
				proxy.URI = name
			}

			this.Proxy = append(this.Proxy, proxy)
		}
	}
}

type ProxyConfig struct {
	app *AppConfig

	Name               string `hcl:"-" json:"-"`
	URI                string `hcl:"uri" json:"uri"`
	Host               string `hcl:"host" json:"host"`
	GRPC               *bool  `hcl:"grpc" json:"grpc"`
	Backend            string `hcl:"backend" json:"backend"`
	Policy             string `hcl:"policy" json:"policy"`
	TLS                bool   `hcl:"tls" json:"tls"`
	InsecureSkipVerify bool   `hcl:"insecure_skip_verify" json:"insecure_skip_verify"`
}

func (this *ProxyConfig) GetGRPC() bool {
	if this.GRPC == nil {
		return this.app.GetGRPC()
	}

	return *this.GRPC
}

func (this *ProxyConfig) link() {

}
<commit_msg>config omitempty<commit_after>package config

import (
	"io/ioutil"

	"github.com/hashicorp/hcl"
)

func ReadConfig(filename string) (ServerConfig, error) {
	cfg := ServerConfig{}
	return cfg, cfg.Read(filename)
}

type ServerConfig struct {
	Bind []string                `hcl:"bind,omitempty" json:"bind,omitempty"`
	Cert []string                `hcl:"cert,omitempty" json:"cert,omitempty"`
	GRPC bool                    `hcl:"grpc,omitempty" json:"grpc,omitempty"`
	AppM []map[string]*AppConfig `hcl:"app,omitempty" json:"app,omitempty"`
	App  []*AppConfig            `hcl:"-" json:"-"`
}

func (this *ServerConfig) Read(filename string) error {
	in, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	if err := hcl.Unmarshal(in, this); err != nil {
		return err
	}

	this.link()

	return nil
}

func (this *ServerConfig) link() {
	for _, m := range this.AppM {
		for name, app := range m {
			app.link()
			app.server = this
			app.Name = name
			if app.Host == "" {
				app.Host = name
			}

			this.App = append(this.App, app)
		}
	}
}

type AppConfig struct {
	server *ServerConfig

	Name   string                    `hcl:"-" json:"-"`
	Host   string                    `hcl:"host,omitempty" json:"host,omitempty"`
	Bind   []string                  `hcl:"bind,omitempty" json:"bind,omitempty"`
	GRPC   *bool                     `hcl:"grpc,omitempty" json:"grpc,omitempty"`
	ProxyM []map[string]*ProxyConfig `hcl:"proxy,omitempty" json:"proxy,omitempty"`
	Proxy  []*ProxyConfig            `hcl:"-" json:"-"`
}

func (this *AppConfig) GetBind() []string {
	if len(this.Bind) == 0 {
		return this.server.Bind
	}

	return this.Bind
}

func (this *AppConfig) GetGRPC() bool {
	if this.GRPC == nil {
		return this.server.GRPC
	}

	return *this.GRPC
}

func (this *AppConfig) link() {
	for _, m := range this.ProxyM {
		for name, proxy := range m {
			proxy.link()
			proxy.app = this
			proxy.Name = name
			if proxy.URI == "" {
				proxy.URI = name
			}

			this.Proxy = append(this.Proxy, proxy)
		}
	}
}

type ProxyConfig struct {
	app *AppConfig

	Name               string `hcl:"-" json:"-"`
	URI                string `hcl:"uri,omitempty" json:"uri,omitempty"`
	Host               string `hcl:"host,omitempty" json:"host,omitempty"`
	GRPC               *bool  `hcl:"grpc,omitempty" json:"grpc,omitempty"`
	Backend            string `hcl:"backend,omitempty" json:"backend,omitempty"`
	Policy             string `hcl:"policy,omitempty" json:"policy,omitempty"`
	TLS                bool   `hcl:"tls,omitempty" json:"tls,omitempty"`
	InsecureSkipVerify bool   `hcl:"insecure_skip_verify,omitempty" json:"insecure_skip_verify,omitempty"`
}

func (this *ProxyConfig) GetGRPC() bool {
	if this.GRPC == nil {
		return this.app.GetGRPC()
	}

	return *this.GRPC
}

func (this *ProxyConfig) link() {

}
<|endoftext|>