<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"

	"gopkg.in/mflag.v1"
)

var (
	packageName string
	outputPath  string
	goPath      string
	silent      bool
)

func init() {
	mflag.BoolVar(&silent, []string{"s", "-silent"}, false, "Be silent, no fancy stuff")
	mflag.StringVar(&outputPath, []string{"o", "-output"}, ".", "Specify the path for resulting exectuable")
	mflag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s <package>\n", os.Args[0])
		mflag.PrintDefaults()
	}
	mflag.Parse()
	packageName = mflag.Arg(0)
	if len(packageName) == 0 {
		mflag.Usage()
		os.Exit(1)
	}
	if goPath = os.Getenv("GOPATH"); len(goPath) == 0 {
		fatal("$GOPATH env variable is not defined")
	}
}

func main() {
	dockerPath, err := getDocker()
	if err != nil {
		fatal("error finding docker executable:", err)
	} else if !silent {
		fmt.Println("found Docker:", dockerPath)
	}

	ver, err := getDockerVersion(dockerPath)
	if err != nil {
		fatal("Docker cannot run:", err)
	} else if !silent {
		fmt.Println(ver)
	}

	dir, name, err := getDirAndFile(outputPath)
	if err != nil {
		fatal("error while checking output path:", err)
	} else if err := os.MkdirAll(dir, 0755); err != nil {
		fatal("error creating output dir:", err)
	}
	if len(name) == 0 {
		name = filepath.Base(packageName)
	}
	if !silent {
		fmt.Println("compiling into:", filepath.Join(dir, name))
	}

	if err := runDockerGoBuild(&buildSpec{
		dockerPath:  dockerPath,
		goPath:      goPath,
		outDir:      dir,
		execName:    name,
		packageName: packageName,
	}, os.Stdout, os.Stderr); err != nil {
		fatal("error running go build using Docker:", err)
	}
}

func getDirAndFile(path string) (dir string, name string, err error) {
	abs, err := filepath.Abs(path)
	if err != nil {
		return "", "", err
	} else if outputPath == "." {
		return abs, "", nil
	} else if info, err := os.Stat(name); err == nil && info.IsDir() {
		return abs, "", errors.New("the path should specify a file, not dir")
	}
	dir = filepath.Dir(abs)
	name = filepath.Base(abs)
	return
}

type buildSpec struct {
	dockerPath  string
	goPath      string
	outDir      string
	execName    string
	packageName string
}

func runDockerGoBuild(spec *buildSpec, stdout, stderr io.Writer) error {
	// docker run -it --rm -v $GOPATH:/go -v $OUT/bin:/data golang go build -o /data/$NAME -i $PACKAGE
	build := exec.Cmd{
		Path: spec.dockerPath,
		Args: []string{
			spec.dockerPath,
			"run", "-it", "--rm",
			"-v", fmt.Sprintf("%s:/go", spec.goPath),
			"-v", fmt.Sprintf("%s:/data", spec.outDir),
			"golang", "go", "build",
			"-o", filepath.Join("/data", spec.execName),
			"-i", spec.packageName,
		},
		Stdout: stdout,
		Stderr: stderr,
	}
	return build.Run()
}

func getDocker() (string, error) {
	out, err := exec.Command("which", "docker").Output()
	if err != nil {
		return "", errors.New("executable file not found in $PATH")
	}
	return string(bytes.TrimSpace(out)), nil
}

func getDockerVersion(path string) (string, error) {
	out, err := exec.Command(path, "-v").Output()
	if err != nil {
		return "", err
	}
	return string(bytes.TrimSpace(out)), nil
}

func fatal(v ...interface{}) {
	fmt.Println(v...)
	os.Exit(1)
}
<commit_msg>Log build errors in stdout.<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"

	"gopkg.in/mflag.v1"
)

var (
	packageName string
	outputPath  string
	goPath      string
	silent      bool
)

func init() {
	mflag.BoolVar(&silent, []string{"s", "-silent"}, false, "Be silent, no fancy stuff")
	mflag.StringVar(&outputPath, []string{"o", "-output"}, ".", "Specify the path for resulting exectuable")
	mflag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s <package>\n", os.Args[0])
		mflag.PrintDefaults()
	}
	mflag.Parse()
	packageName = mflag.Arg(0)
	if len(packageName) == 0 {
		mflag.Usage()
		os.Exit(1)
	}
	if goPath = os.Getenv("GOPATH"); len(goPath) == 0 {
		fatal("$GOPATH env variable is not defined")
	}
}

func main() {
	dockerPath, err := getDocker()
	if err != nil {
		fatal("error finding docker executable:", err)
	} else if !silent {
		fmt.Println("found Docker:", dockerPath)
	}

	ver, err := getDockerVersion(dockerPath)
	if err != nil {
		fatal("Docker cannot run:", err)
	} else if !silent {
		fmt.Println(ver)
	}

	dir, name, err := getDirAndFile(outputPath)
	if err != nil {
		fatal("error while checking output path:", err)
	} else if err := os.MkdirAll(dir, 0755); err != nil {
		fatal("error creating output dir:", err)
	}
	if len(name) == 0 {
		name = filepath.Base(packageName)
	}
	if !silent {
		fmt.Println("compiling into:", filepath.Join(dir, name))
	}

	if err := runDockerGoBuild(&buildSpec{
		dockerPath:  dockerPath,
		goPath:      goPath,
		outDir:      dir,
		execName:    name,
		packageName: packageName,
	}, os.Stdout, os.Stderr); err != nil {
		fatal("error running go build using Docker:", err)
	}
}

func getDirAndFile(path string) (dir string, name string, err error) {
	abs, err := filepath.Abs(path)
	if err != nil {
		return "", "", err
	} else if outputPath == "." {
		return abs, "", nil
	} else if info, err := os.Stat(name); err == nil && info.IsDir() {
		return abs, "", errors.New("the path should specify a file, not dir")
	}
	dir = filepath.Dir(abs)
	name = filepath.Base(abs)
	return
}

type buildSpec struct {
	dockerPath  string
	goPath      string
	outDir      string
	execName    string
	packageName string
}

func runDockerGoBuild(spec *buildSpec, stdout, stderr io.Writer) error {
	// docker run -it --rm -v $GOPATH:/go -v $OUT/bin:/data golang go build -o /data/$NAME -i $PACKAGE
	build := exec.Cmd{
		Path: spec.dockerPath,
		Args: []string{
			spec.dockerPath,
			"run", "-a", "stdout", "-a", "stderr", "--rm",
			"-v", fmt.Sprintf("%s:/go", spec.goPath),
			"-v", fmt.Sprintf("%s:/data", spec.outDir),
			"golang", "go", "build",
			"-o", filepath.Join("/data", spec.execName),
			"-i", spec.packageName,
		},
		Stdout: stdout,
		Stderr: stderr,
	}
	return build.Run()
}

func getDocker() (string, error) {
	out, err := exec.Command("which", "docker").Output()
	if err != nil {
		return "", errors.New("executable file not found in $PATH")
	}
	return string(bytes.TrimSpace(out)), nil
}

func getDockerVersion(path string) (string, error) {
	out, err := exec.Command(path, "-v").Output()
	if err != nil {
		return "", err
	}
	return string(bytes.TrimSpace(out)), nil
}

func fatal(v ...interface{}) {
	fmt.Println(v...)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package core_types

import (
	"github.com/tendermint/tendermint/account"
	sm "github.com/tendermint/tendermint/state"
	"github.com/tendermint/tendermint/types"
)

type ResponseGenPrivAccount struct {
	PrivAccount *account.PrivAccount `json:"priv_account"`
}

type ResponseGetAccount struct {
	Account *account.Account `json:"account"`
}

type ResponseGetStorage struct {
	Key   []byte `json:"key"`
	Value []byte `json:"value"`
}

type ResponseCall struct {
	Return  []byte `json:"return"`
	GasUsed uint64 `json:"gas_used"`
	// TODO ...
}

type ResponseListAccounts struct {
	BlockHeight uint               `json:"block_height"`
	Accounts    []*account.Account `json:"accounts"`
}

type StorageItem struct {
	Key   []byte `json:"key"`
	Value []byte `json:"value"`
}

type ResponseDumpStorage struct {
	StorageRoot  []byte        `json:"storage_root"`
	StorageItems []StorageItem `json:"storage_items"`
}

type ResponseBlockchainInfo struct {
	LastHeight uint               `json:"last_height"`
	BlockMetas []*types.BlockMeta `json:"block_metas"`
}

type ResponseGetBlock struct {
	BlockMeta *types.BlockMeta `json:"block_meta"`
	Block     *types.Block     `json:"block"`
}

type ResponseBroadcastTx struct {
	Receipt Receipt `json:"receipt"`
}

type ResponseListUnconfirmedTxs struct {
	Txs []types.Tx `json:"txs"`
}

type Receipt struct {
	TxHash          []byte `json:"tx_hash"`
	CreatesContract uint8  `json:"creates_contract"`
	ContractAddr    []byte `json:"contract_addr"`
}

type ResponseStatus struct {
	GenesisHash       []byte `json:"genesis_hash"`
	Network           string `json:"network"`
	LatestBlockHash   []byte `json:"lastest_block_hash"`
	LatestBlockHeight uint   `json:"lastest_block_height"`
	LatestBlockTime   int64  `json:"latest_bloick_time"` // nano
}

type ResponseNetInfo struct {
	Moniker   string   `json:"moniker"`
	Network   string   `json:"network"`
	Listening bool     `json:"listening"`
	Listeners []string `json:"listeners"`
	Peers     []Peer   `json:"peers"`
}

type Peer struct {
	types.NodeInfo `json:"net_info"`
	IsOutbound     bool `json:"is_outbound"`
}

type ResponseSignTx struct {
	Tx types.Tx `json:"tx"`
}

type ResponseListValidators struct {
	BlockHeight         uint            `json:"block_height"`
	BondedValidators    []*sm.Validator `json:"bonded_validators"`
	UnbondingValidators []*sm.Validator `json:"unbonding_validators"`
}

type ResponseDumpConsensusState struct {
	RoundState      string   `json:"round_state"`
	PeerRoundStates []string `json:"peer_round_states"`
}
<commit_msg>json field fixes<commit_after>package core_types

import (
	"github.com/tendermint/tendermint/account"
	sm "github.com/tendermint/tendermint/state"
	"github.com/tendermint/tendermint/types"
)

type ResponseGenPrivAccount struct {
	PrivAccount *account.PrivAccount `json:"priv_account"`
}

type ResponseGetAccount struct {
	Account *account.Account `json:"account"`
}

type ResponseGetStorage struct {
	Key   []byte `json:"key"`
	Value []byte `json:"value"`
}

type ResponseCall struct {
	Return  []byte `json:"return"`
	GasUsed uint64 `json:"gas_used"`
	// TODO ...
}

type ResponseListAccounts struct {
	BlockHeight uint               `json:"block_height"`
	Accounts    []*account.Account `json:"accounts"`
}

type StorageItem struct {
	Key   []byte `json:"key"`
	Value []byte `json:"value"`
}

type ResponseDumpStorage struct {
	StorageRoot  []byte        `json:"storage_root"`
	StorageItems []StorageItem `json:"storage_items"`
}

type ResponseBlockchainInfo struct {
	LastHeight uint               `json:"last_height"`
	BlockMetas []*types.BlockMeta `json:"block_metas"`
}

type ResponseGetBlock struct {
	BlockMeta *types.BlockMeta `json:"block_meta"`
	Block     *types.Block     `json:"block"`
}

type ResponseBroadcastTx struct {
	Receipt Receipt `json:"receipt"`
}

type ResponseListUnconfirmedTxs struct {
	Txs []types.Tx `json:"txs"`
}

type Receipt struct {
	TxHash          []byte `json:"tx_hash"`
	CreatesContract uint8  `json:"creates_contract"`
	ContractAddr    []byte `json:"contract_addr"`
}

type ResponseStatus struct {
	GenesisHash       []byte `json:"genesis_hash"`
	Network           string `json:"network"`
	LatestBlockHash   []byte `json:"latest_block_hash"`
	LatestBlockHeight uint   `json:"latest_block_height"`
	LatestBlockTime   int64  `json:"latest_block_time"` // nano
}

type ResponseNetInfo struct {
	Moniker   string   `json:"moniker"`
	Network   string   `json:"network"`
	Listening bool     `json:"listening"`
	Listeners []string `json:"listeners"`
	Peers     []Peer   `json:"peers"`
}

type Peer struct {
	types.NodeInfo `json:"node_info"`
	IsOutbound     bool `json:"is_outbound"`
}

type ResponseSignTx struct {
	Tx types.Tx `json:"tx"`
}

type ResponseListValidators struct {
	BlockHeight         uint            `json:"block_height"`
	BondedValidators    []*sm.Validator `json:"bonded_validators"`
	UnbondingValidators []*sm.Validator `json:"unbonding_validators"`
}

type ResponseDumpConsensusState struct {
	RoundState      string   `json:"round_state"`
	PeerRoundStates []string `json:"peer_round_states"`
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"

	yaml "gopkg.in/yaml.v2"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchevents"
)

func main() {
	var (
		apply, dryrun   bool
		file, awsRegion string
	)

	flag.BoolVar(&apply, "apply", false, "apply to CloudWatch Events")
	flag.BoolVar(&dryrun, "dry-run", false, "dry-run")
	flag.StringVar(&file, "file", "config.yml", "file path to setting yaml")
	flag.StringVar(&file, "f", "config.yml", "file path to setting yaml (shorthand)")
	flag.StringVar(&awsRegion, "region", "", "aws region")
	flag.Parse()

	sess, errS := session.NewSession(
		&aws.Config{
			Region: aws.String(awsRegion),
		},
	)
	if errS != nil {
		fmt.Printf("Session error %v\n", errS)
		os.Exit(1)
	}

	cweRulesOutput, errR := cloudwatchevents.New(sess).ListRules(nil)
	if errR != nil {
		fmt.Printf("API error %v\n", errR)
		os.Exit(1)
	}

	describedRules := Rules{}
	errY := loadYaml(file, &describedRules)
	if errY != nil {
		fmt.Printf("File error %v\n", errY)
		os.Exit(1)
	}

	describedRules.Rules = AssociateRules(cweRulesOutput.Rules, describedRules.Rules)
	for i, rule := range describedRules.Rules {
		t, _ := fetchActualTargetsByRule(cloudwatchevents.New(sess), rule)
		describedRules.Rules[i].Targets = AssociateTargets(t, describedRules.Rules[i].Targets)
	}
	CheckIsNeedUpdateOrDelete(describedRules.Rules)
	displayWhatWillChange(describedRules.Rules)

	if apply && !dryrun {
		errU := updateCloudWatchEvents(cloudwatchevents.New(sess), describedRules.Rules)
		if errU != nil {
			fmt.Printf("API error %v\n", errU)
			os.Exit(1)
		}
	}
}

func loadYaml(file string, r *Rules) error {

	buf, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(buf, &r)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Version 0.1.0<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"

	yaml "gopkg.in/yaml.v2"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchevents"
)

var Version = "0.1.0"

func main() {
	var (
		apply, dryrun   bool
		file, awsRegion string
	)

	flag.BoolVar(&apply, "apply", false, "apply to CloudWatch Events")
	flag.BoolVar(&dryrun, "dry-run", false, "dry-run")
	flag.StringVar(&file, "file", "config.yml", "file path to setting yaml")
	flag.StringVar(&file, "f", "config.yml", "file path to setting yaml (shorthand)")
	flag.StringVar(&awsRegion, "region", "", "aws region")
	flag.Parse()

	sess, errS := session.NewSession(
		&aws.Config{
			Region: aws.String(awsRegion),
		},
	)
	if errS != nil {
		fmt.Printf("Session error %v\n", errS)
		os.Exit(1)
	}

	cweRulesOutput, errR := cloudwatchevents.New(sess).ListRules(nil)
	if errR != nil {
		fmt.Printf("API error %v\n", errR)
		os.Exit(1)
	}

	describedRules := Rules{}
	errY := loadYaml(file, &describedRules)
	if errY != nil {
		fmt.Printf("File error %v\n", errY)
		os.Exit(1)
	}

	describedRules.Rules = AssociateRules(cweRulesOutput.Rules, describedRules.Rules)
	for i, rule := range describedRules.Rules {
		t, _ := fetchActualTargetsByRule(cloudwatchevents.New(sess), rule)
		describedRules.Rules[i].Targets = AssociateTargets(t, describedRules.Rules[i].Targets)
	}
	CheckIsNeedUpdateOrDelete(describedRules.Rules)
	displayWhatWillChange(describedRules.Rules)

	if apply && !dryrun {
		errU := updateCloudWatchEvents(cloudwatchevents.New(sess), describedRules.Rules)
		if errU != nil {
			fmt.Printf("API error %v\n", errU)
			os.Exit(1)
		}
	}
}

func loadYaml(file string, r *Rules) error {

	buf, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(buf, &r)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Kochava
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/Shopify/sarama"
)

func main() {

	config := InitConfig()

	config.GetConfig()

	logFile, err := os.OpenFile(config.firehoseLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer logFile.Close()

	log.SetOutput(logFile)

	metricsFile, err := os.OpenFile(config.metricsLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer metricsFile.Close()

	log.Println("Getting the Kafka consumer")
	consumer, err := GetKafkaConsumer(config, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create consumer", err)
	}

	log.Println("Getting the Kafka producer")
	producer, err := GetKafkaProducer(config, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create producer", err)
	}

	defer CloseConsumer(&consumer)
	defer CloseProducer(&producer)

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)

	var wg sync.WaitGroup

	configClient := NewClient(config)
	defer configClient.Close()

	transferChan := make(chan sarama.ProducerMessage, 100000)

	log.Println(configClient.GetNumPartitions())
	for partition := 0; partition < configClient.GetNumPartitions(); partition++ {

		syncChan := make(chan int64, 1)
		offset := sarama.OffsetNewest
		finalOffset := int64(-1)

		if config.historical {
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastFourHours)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}

		}

		log.Println("Using offset ", offset, " for partition ", partition)
		partitionConsumer, err := consumer.ConsumePartition(config.topic, int32(partition), offset)
		if err != nil {
			log.Println("Unable to create partition consumer", err)
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastFourHours)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}
			partitionConsumer, err = consumer.ConsumePartition(config.topic, int32(partition), offset)
			if err != nil {
				log.Println("Unable to create partition consumer", err)
				continue
			}
		}

		wg.Add(1)

		go PullFromTopic(partitionConsumer, transferChan, signals, finalOffset, syncChan, &wg)
		log.Println("Started consumer for partition ", partition)

		wg.Add(2)
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
	}

	go MonitorChan(transferChan, signals, &wg)

	wg.Wait()

	log.Println("All threads done, closing clients and ending")
}
<commit_msg>Use last one day by default for historical mode<commit_after>// Copyright 2016 Kochava
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/Shopify/sarama"
)

func main() {

	config := InitConfig()

	config.GetConfig()

	logFile, err := os.OpenFile(config.firehoseLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer logFile.Close()

	log.SetOutput(logFile)

	metricsFile, err := os.OpenFile(config.metricsLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer metricsFile.Close()

	log.Println("Getting the Kafka consumer")
	consumer, err := GetKafkaConsumer(config, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create consumer", err)
	}

	log.Println("Getting the Kafka producer")
	producer, err := GetKafkaProducer(config, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create producer", err)
	}

	defer CloseConsumer(&consumer)
	defer CloseProducer(&producer)

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)

	var wg sync.WaitGroup

	configClient := NewClient(config)
	defer configClient.Close()

	transferChan := make(chan sarama.ProducerMessage, 100000)

	log.Println(configClient.GetNumPartitions())
	for partition := 0; partition < configClient.GetNumPartitions(); partition++ {

		syncChan := make(chan int64, 1)
		offset := sarama.OffsetNewest
		finalOffset := int64(-1)

		if config.historical {
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastFourHours)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}

		}

		log.Println("Using offset ", offset, " for partition ", partition)
		partitionConsumer, err := consumer.ConsumePartition(config.topic, int32(partition), offset)
		if err != nil {
			log.Println("Unable to create partition consumer", err)
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastOneDays)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}
			partitionConsumer, err = consumer.ConsumePartition(config.topic, int32(partition), offset)
			if err != nil {
				log.Println("Unable to create partition consumer", err)
				continue
			}
		}

		wg.Add(1)

		go PullFromTopic(partitionConsumer, transferChan, signals, finalOffset, syncChan, &wg)
		log.Println("Started consumer for partition ", partition)

		wg.Add(2)
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
	}

	go MonitorChan(transferChan, signals, &wg)

	wg.Wait()

	log.Println("All threads done, closing clients and ending")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/speedboat/js"
	"github.com/loadimpact/speedboat/lib"
	"github.com/loadimpact/speedboat/simple"
	"github.com/loadimpact/speedboat/stats"
	"github.com/loadimpact/speedboat/stats/accumulate"
	"github.com/loadimpact/speedboat/stats/influxdb"
	"github.com/loadimpact/speedboat/stats/writer"
	"github.com/urfave/cli"
	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"
)

const (
	typeURL = "url"
	typeJS  = "js"
)

// Help text template
const helpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .VisibleFlags}}[options] {{end}}filename|url{{end}}
   {{if .Version}}{{if not .HideVersion}}
VERSION:
   {{.Version}}
   {{end}}{{end}}{{if .VisibleFlags}}
OPTIONS:
   {{range .VisibleFlags}}{{.}}
   {{end}}{{end}}
`

var mVUs = stats.Stat{Name: "vus", Type: stats.GaugeType}

func pollVURamping(ctx context.Context, t lib.Test) <-chan int {
	ch := make(chan int)
	startTime := time.Now()

	go func() {
		defer close(ch)

		ticker := time.NewTicker(1 * time.Second)
		for {
			select {
			case <-ticker.C:
				ch <- t.VUsAt(time.Since(startTime))
			case <-ctx.Done():
				return
			}
		}
	}()

	return ch
}

func parseBackend(out string) (stats.Backend, error) {
	switch {
	case strings.HasPrefix(out, "influxdb+"):
		url := strings.TrimPrefix(out, "influxdb+")
		return influxdb.NewFromURL(url)
	default:
		return nil, errors.New("Unknown output destination")
	}
}

func parseStages(vus []string, total time.Duration) (stages []lib.TestStage, err error) {
	if len(vus) == 0 {
		return []lib.TestStage{
			lib.TestStage{Duration: total, StartVUs: 10, EndVUs: 10},
		}, nil
	}

	accountedTime := time.Duration(0)
	fluidStages := []int{}
	for i, spec := range vus {
		parts := strings.SplitN(spec, ":", 2)
		countParts := strings.SplitN(parts[0], "-", 2)

		stage := lib.TestStage{}

		// An absent first part means keep going from the last stage's end
		// If it's the first stage, just start with 0
		if countParts[0] == "" {
			if i > 0 {
				stage.StartVUs = stages[i-1].EndVUs
			}
		} else {
			start, err := strconv.ParseInt(countParts[0], 10, 64)
			if err != nil {
				return stages, err
			}
			stage.StartVUs = int(start)
		}

		// If an end is specified, use that, otherwise keep the VU level constant
		if len(countParts) > 1 && countParts[1] != "" {
			end, err := strconv.ParseInt(countParts[1], 10, 64)
			if err != nil {
				return stages, err
			}
			stage.EndVUs = int(end)
		} else {
			stage.EndVUs = stage.StartVUs
		}

		// If a time is specified, use that, otherwise mark the stage as "fluid", allotting it an
		// even slice of what time remains after all fixed stages are accounted for (may be 0)
		if len(parts) > 1 {
			duration, err := time.ParseDuration(parts[1])
			if err != nil {
				return stages, err
			}
			stage.Duration = duration
			accountedTime += duration
		} else {
			fluidStages = append(fluidStages, i)
		}

		stages = append(stages, stage)
	}

	// We're ignoring fluid stages if the fixed stages already take up all the allotted time
	// Otherwise, evenly divide the test's remaining time between all fluid stages
	if len(fluidStages) > 0 && accountedTime < total {
		fluidDuration := (total - accountedTime) / time.Duration(len(fluidStages))
		for _, i := range fluidStages {
			stage := stages[i]
			stage.Duration = fluidDuration
			stages[i] = stage
		}
	}

	return stages, nil
}

func guessType(arg string) string {
	switch {
	case strings.Contains(arg, "://"):
		return typeURL
	case strings.HasSuffix(arg, ".js"):
		return typeJS
	}
	return ""
}

func readAll(filename string) ([]byte, error) {
	if filename == "-" {
		return ioutil.ReadAll(os.Stdin)
	}

	return ioutil.ReadFile(filename)
}

func makeRunner(t lib.Test, filename, typ string) (lib.Runner, error) {
	if typ == typeURL {
		return simple.New(filename), nil
	}

	bytes, err := readAll(filename)
	if err != nil {
		return nil, err
	}

	switch typ {
	case typeJS:
		return js.New(filename, string(bytes)), nil
	default:
		return nil, errors.New("Type ambiguous, please specify -t/--type")
	}
}

func action(cc *cli.Context) error {
	if cc.IsSet("verbose") {
		log.SetLevel(log.DebugLevel)
	}

	for _, out := range cc.StringSlice("out") {
		backend, err := parseBackend(out)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		stats.DefaultRegistry.Backends = append(stats.DefaultRegistry.Backends, backend)
	}

	var formatter writer.Formatter
	switch cc.String("format") {
	case "":
	case "json":
		formatter = writer.JSONFormatter{}
	case "prettyjson":
		formatter = writer.PrettyJSONFormatter{}
	case "yaml":
		formatter = writer.YAMLFormatter{}
	default:
		return cli.NewExitError("Unknown output format", 1)
	}

	var summarizer *Summarizer
	if formatter != nil {
		filter := stats.MakeFilter(cc.StringSlice("exclude"), cc.StringSlice("select"))
		if cc.Bool("raw") {
			backend := &writer.Backend{
				Writer:    os.Stdout,
				Formatter: formatter,
			}
			backend.Filter = filter
			stats.DefaultRegistry.Backends = append(stats.DefaultRegistry.Backends, backend)
		} else {
			accumulator := accumulate.New()
			accumulator.Filter = filter
			accumulator.GroupBy = cc.StringSlice("group-by")
			stats.DefaultRegistry.Backends = append(stats.DefaultRegistry.Backends, accumulator)

			summarizer = &Summarizer{
				Accumulator: accumulator,
				Formatter:   formatter,
			}
		}
	}

	stages, err := parseStages(cc.StringSlice("vus"), cc.Duration("duration"))
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	t := lib.Test{Stages: stages}

	var r lib.Runner
	switch len(cc.Args()) {
	case 0:
		cli.ShowAppHelp(cc)
		return nil
	case 1:
		filename := cc.Args()[0]
		typ := cc.String("type")
		if typ == "" {
			typ = guessType(filename)
		}

		if filename == "-" && typ == "" {
			typ = typeJS
		}

		runner, err := makeRunner(t, filename, typ)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		r = runner
	default:
		return cli.NewExitError("Too many arguments!", 1)
	}

	if cc.Bool("plan") {
		data, err := yaml.Marshal(map[string]interface{}{
			"stages": stages,
		})
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		os.Stdout.Write(data)
		return nil
	}

	vus := lib.VUGroup{
		Pool: lib.VUPool{
			New: r.NewVU,
		},
		RunOnce: func(ctx context.Context, vu lib.VU) {
			if err := vu.RunOnce(ctx); err != nil {
				log.WithError(err).Error("Uncaught Error")
			}
		},
	}

	for i := 0; i < t.MaxVUs(); i++ {
		vu, err := vus.Pool.New()
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		vus.Pool.Put(vu)
	}

	ctx, cancel := context.WithTimeout(context.Background(), t.TotalDuration())

	go func() {
		ticker := time.NewTicker(1 * time.Second)
		for {
			select {
			case <-ticker.C:
				if err := stats.Submit(); err != nil {
					log.WithError(err).Error("[Couldn't submit stats]")
				}
			case <-ctx.Done():
				return
			}
		}
	}()

	interval := cc.Duration("interval")
	if interval > 0 && summarizer != nil {
		go func() {
			ticker := time.NewTicker(interval)
			for {
				select {
				case <-ticker.C:
					if err := summarizer.Print(os.Stdout); err != nil {
						log.WithError(err).Error("Couldn't print statistics!")
					}
				case <-ctx.Done():
					return
				}
			}
		}()
	}

	go func() {
		quit := make(chan os.Signal)
		signal.Notify(quit)

		select {
		case <-quit:
			cancel()
		case <-ctx.Done():
		}
	}()

	if !cc.Bool("quiet") {
		log.WithFields(log.Fields{
			"at":     time.Now(),
			"length": t.TotalDuration(),
		}).Info("Starting test...")
	}

	vus.Start(ctx)
	scaleTo := pollVURamping(ctx, t)
mainLoop:
	for {
		select {
		case num := <-scaleTo:
			vus.Scale(num)
			stats.Add(stats.Sample{
				Stat:   &mVUs,
				Values: stats.Value(float64(num)),
			})
		case <-ctx.Done():
			break mainLoop
		}
	}

	vus.Stop()

	stats.Add(stats.Sample{Stat: &mVUs, Values: stats.Value(0)})
	stats.Submit()

	if summarizer != nil {
		if err := summarizer.Print(os.Stdout); err != nil {
			log.WithError(err).Error("Couldn't print statistics!")
		}
	}

	return nil
}

func main() {
	// Submit usage statistics for the closed beta
	invocation := Invocation{}
	invocationError := make(chan error, 1)

	go func() {
		// Set SUBMIT=false to prevent stat collection
		submitURL := os.Getenv("SUBMIT")
		switch submitURL {
		case "false", "no":
			return
		case "":
			submitURL = "http://127.0.0.1:8080"
		}

		// Wait at most 2s for an invocation error to be reported
		select {
		case err := <-invocationError:
			invocation.Error = err.Error()
		case <-time.After(2 * time.Second):
		}

		// Submit stats to a specified server
		if err := invocation.Submit(submitURL); err != nil {
			log.WithError(err).Fatal("Couldn't submit statistics")
		}
	}()

	// Free up -v and -h for our own flags
	cli.VersionFlag.Name = "version"
	cli.HelpFlag.Name = "help, ?"
	cli.AppHelpTemplate = helpTemplate

	// Bootstrap the app from commandline flags
	app := cli.NewApp()
	app.Name = "speedboat"
	app.Usage = "A next-generation load generator"
	app.Version = "1.0.0-mvp1"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "plan",
			Usage: "Don't run anything, just show the test plan",
		},
		cli.StringFlag{
			Name:  "type, t",
			Usage: "Input file type, if not evident (url or js)",
		},
		cli.StringSliceFlag{
			Name:  "vus, u",
			Usage: "Number of VUs to simulate",
		},
		cli.DurationFlag{
			Name:  "duration, d",
			Usage: "Test duration",
			Value: time.Duration(10) * time.Second,
		},
		cli.BoolFlag{
			Name:  "verbose, v",
			Usage: "More verbose output",
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "Suppress the summary at the end of a test",
		},
		cli.StringFlag{
			Name:  "format, f",
			Usage: "Format for printed metrics (yaml, json, prettyjson)",
			Value: "yaml",
		},
		cli.DurationFlag{
			Name:  "interval, i",
			Usage: "Write periodic summaries",
		},
		cli.StringSliceFlag{
			Name:  "out, o",
			Usage: "Write metrics to a database",
		},
		cli.BoolFlag{
			Name:  "raw",
			Usage: "Instead of summaries, dump raw samples to stdout",
		},
		cli.StringSliceFlag{
			Name:  "select, s",
			Usage: "Include only named metrics",
		},
		cli.StringSliceFlag{
			Name:  "exclude, e",
			Usage: "Exclude named metrics",
		},
		cli.StringSliceFlag{
			Name:  "group-by, g",
			Usage: "Group metrics by tags",
		},
	}
	app.Before = func(cc *cli.Context) error {
		invocation.PopulateWithContext(cc)
		return nil
	}
	app.Action = action
	invocationError <- app.Run(os.Args)
}
<commit_msg>[fix] This is a debug message if I ever saw one<commit_after>package main

import (
	"errors"
	log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/speedboat/js"
	"github.com/loadimpact/speedboat/lib"
	"github.com/loadimpact/speedboat/simple"
	"github.com/loadimpact/speedboat/stats"
	"github.com/loadimpact/speedboat/stats/accumulate"
	"github.com/loadimpact/speedboat/stats/influxdb"
	"github.com/loadimpact/speedboat/stats/writer"
	"github.com/urfave/cli"
	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"
)

const (
	typeURL = "url"
	typeJS  = "js"
)

// Help text template
const helpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .VisibleFlags}}[options] {{end}}filename|url{{end}}
   {{if .Version}}{{if not .HideVersion}}
VERSION:
   {{.Version}}
   {{end}}{{end}}{{if .VisibleFlags}}
OPTIONS:
   {{range .VisibleFlags}}{{.}}
   {{end}}{{end}}
`

var mVUs = stats.Stat{Name: "vus", Type: stats.GaugeType}

func pollVURamping(ctx context.Context, t lib.Test) <-chan int {
	ch := make(chan int)
	startTime := time.Now()

	go func() {
		defer close(ch)

		ticker := time.NewTicker(1 * time.Second)
		for {
			select {
			case <-ticker.C:
				ch <- t.VUsAt(time.Since(startTime))
			case <-ctx.Done():
				return
			}
		}
	}()

	return ch
}

func parseBackend(out string) (stats.Backend, error) {
	switch {
	case strings.HasPrefix(out, "influxdb+"):
		url := strings.TrimPrefix(out, "influxdb+")
		return influxdb.NewFromURL(url)
	default:
		return nil, errors.New("Unknown output destination")
	}
}

func parseStages(vus []string, total time.Duration) (stages []lib.TestStage, err error) {
	if len(vus) == 0 {
		return []lib.TestStage{
			lib.TestStage{Duration: total, StartVUs: 10, EndVUs: 10},
		}, nil
	}

	accountedTime := time.Duration(0)
	fluidStages := []int{}
	for i, spec := range vus {
		parts := strings.SplitN(spec, ":", 2)
		countParts := strings.SplitN(parts[0], "-", 2)

		stage := lib.TestStage{}

		// An absent first part means keep going from the last stage's end
		// If it's the first stage, just start with 0
		if countParts[0] == "" {
			if i > 0 {
				stage.StartVUs = stages[i-1].EndVUs
			}
		} else {
			start, err := strconv.ParseInt(countParts[0], 10, 64)
			if err != nil {
				return stages, err
			}
			stage.StartVUs = int(start)
		}

		// If an end is specified, use that, otherwise keep the VU level constant
		if len(countParts) > 1 && countParts[1] != "" {
			end, err := strconv.ParseInt(countParts[1], 10, 64)
			if err != nil {
				return stages, err
			}
			stage.EndVUs = int(end)
		} else {
			stage.EndVUs = stage.StartVUs
		}

		// If a time is specified, use that, otherwise mark the stage as "fluid", allotting it an
		// even slice of what time remains after all fixed stages are accounted for (may be 0)
		if len(parts) > 1 {
			duration, err := time.ParseDuration(parts[1])
			if err != nil {
				return stages, err
			}
			stage.Duration = duration
			accountedTime += duration
		} else {
			fluidStages = append(fluidStages, i)
		}

		stages = append(stages, stage)
	}

	// We're ignoring fluid stages if the fixed stages already take up all the allotted time
	// Otherwise, evenly divide the test's remaining time between all fluid stages
	if len(fluidStages) > 0 && accountedTime < total {
		fluidDuration := (total - accountedTime) / time.Duration(len(fluidStages))
		for _, i := range fluidStages {
			stage := stages[i]
			stage.Duration = fluidDuration
			stages[i] = stage
		}
	}

	return stages, nil
}

func guessType(arg string) string {
	switch {
	case strings.Contains(arg, "://"):
		return typeURL
	case strings.HasSuffix(arg, ".js"):
		return typeJS
	}
	return ""
}

func readAll(filename string) ([]byte, error) {
	if filename == "-" {
		return ioutil.ReadAll(os.Stdin)
	}

	return ioutil.ReadFile(filename)
}

func makeRunner(t lib.Test, filename, typ string) (lib.Runner, error) {
	if typ == typeURL {
		return simple.New(filename), nil
	}

	bytes, err := readAll(filename)
	if err != nil {
		return nil, err
	}

	switch typ {
	case typeJS:
		return js.New(filename, string(bytes)), nil
	default:
		return nil, errors.New("Type ambiguous, please specify -t/--type")
	}
}

func action(cc *cli.Context) error {
	if cc.IsSet("verbose") {
		log.SetLevel(log.DebugLevel)
	}

	for _, out := range cc.StringSlice("out") {
		backend, err := parseBackend(out)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		stats.DefaultRegistry.Backends = append(stats.DefaultRegistry.Backends, backend)
	}

	var formatter writer.Formatter
	switch cc.String("format") {
	case "":
	case "json":
		formatter = writer.JSONFormatter{}
	case "prettyjson":
		formatter = writer.PrettyJSONFormatter{}
	case "yaml":
		formatter = writer.YAMLFormatter{}
	default:
		return cli.NewExitError("Unknown output format", 1)
	}

	var summarizer *Summarizer
	if formatter != nil {
		filter := stats.MakeFilter(cc.StringSlice("exclude"), cc.StringSlice("select"))
		if cc.Bool("raw") {
			backend := &writer.Backend{
				Writer:    os.Stdout,
				Formatter: formatter,
			}
			backend.Filter = filter
			stats.DefaultRegistry.Backends = append(stats.DefaultRegistry.Backends, backend)
		} else {
			accumulator := accumulate.New()
			accumulator.Filter = filter
			accumulator.GroupBy = cc.StringSlice("group-by")
			stats.DefaultRegistry.Backends = append(stats.DefaultRegistry.Backends, accumulator)

			summarizer = &Summarizer{
				Accumulator: accumulator,
				Formatter:   formatter,
			}
		}
	}

	stages, err := parseStages(cc.StringSlice("vus"), cc.Duration("duration"))
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	t := lib.Test{Stages: stages}

	var r lib.Runner
	switch len(cc.Args()) {
	case 0:
		cli.ShowAppHelp(cc)
		return nil
	case 1:
		filename := cc.Args()[0]
		typ := cc.String("type")
		if typ == "" {
			typ = guessType(filename)
		}

		if filename == "-" && typ == "" {
			typ = typeJS
		}

		runner, err := makeRunner(t, filename, typ)
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		r = runner
	default:
		return cli.NewExitError("Too many arguments!", 1)
	}

	if cc.Bool("plan") {
		data, err := yaml.Marshal(map[string]interface{}{
			"stages": stages,
		})
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		os.Stdout.Write(data)
		return nil
	}

	vus := lib.VUGroup{
		Pool: lib.VUPool{
			New: r.NewVU,
		},
		RunOnce: func(ctx context.Context, vu lib.VU) {
			if err := vu.RunOnce(ctx); err != nil {
				log.WithError(err).Error("Uncaught Error")
			}
		},
	}

	for i := 0; i < t.MaxVUs(); i++ {
		vu, err := vus.Pool.New()
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		vus.Pool.Put(vu)
	}

	ctx, cancel := context.WithTimeout(context.Background(), t.TotalDuration())

	go func() {
		ticker := time.NewTicker(1 * time.Second)
		for {
			select {
			case <-ticker.C:
				if err := stats.Submit(); err != nil {
					log.WithError(err).Error("[Couldn't submit stats]")
				}
			case <-ctx.Done():
				return
			}
		}
	}()

	interval := cc.Duration("interval")
	if interval > 0 && summarizer != nil {
		go func() {
			ticker := time.NewTicker(interval)
			for {
				select {
				case <-ticker.C:
					if err := summarizer.Print(os.Stdout); err != nil {
						log.WithError(err).Error("Couldn't print statistics!")
					}
				case <-ctx.Done():
					return
				}
			}
		}()
	}

	go func() {
		quit := make(chan os.Signal)
		signal.Notify(quit)

		select {
		case <-quit:
			cancel()
		case <-ctx.Done():
		}
	}()

	if !cc.Bool("quiet") {
		log.WithFields(log.Fields{
			"at":     time.Now(),
			"length": t.TotalDuration(),
		}).Info("Starting test...")
	}

	vus.Start(ctx)
	scaleTo := pollVURamping(ctx, t)
mainLoop:
	for {
		select {
		case num := <-scaleTo:
			vus.Scale(num)
			stats.Add(stats.Sample{
				Stat:   &mVUs,
				Values: stats.Value(float64(num)),
			})
		case <-ctx.Done():
			break mainLoop
		}
	}

	vus.Stop()

	stats.Add(stats.Sample{Stat: &mVUs, Values: stats.Value(0)})
	stats.Submit()

	if summarizer != nil {
		if err := summarizer.Print(os.Stdout); err != nil {
			log.WithError(err).Error("Couldn't print statistics!")
		}
	}

	return nil
}

func main() {
	// Submit usage statistics for the closed beta
	invocation := Invocation{}
	invocationError := make(chan error, 1)

	go func() {
		// Set SUBMIT=false to prevent stat collection
		submitURL := os.Getenv("SUBMIT")
		switch submitURL {
		case "false", "no":
			return
		case "":
			submitURL = "http://127.0.0.1:8080"
		}

		// Wait at most 2s for an invocation error to be reported
		select {
		case err := <-invocationError:
			invocation.Error = err.Error()
		case <-time.After(2 * time.Second):
		}

		// Submit stats to a specified server
		if err := invocation.Submit(submitURL); err != nil {
			log.WithError(err).Debug("Couldn't submit statistics")
		}
	}()

	// Free up -v and -h for our own flags
	cli.VersionFlag.Name = "version"
	cli.HelpFlag.Name = "help, ?"
	cli.AppHelpTemplate = helpTemplate

	// Bootstrap the app from commandline flags
	app := cli.NewApp()
	app.Name = "speedboat"
	app.Usage = "A next-generation load generator"
	app.Version = "1.0.0-mvp1"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "plan",
			Usage: "Don't run anything, just show the test plan",
		},
		cli.StringFlag{
			Name:  "type, t",
			Usage: "Input file type, if not evident (url or js)",
		},
		cli.StringSliceFlag{
			Name:  "vus, u",
			Usage: "Number of VUs to simulate",
		},
		cli.DurationFlag{
			Name:  "duration, d",
			Usage: "Test duration",
			Value: time.Duration(10) * time.Second,
		},
		cli.BoolFlag{
			Name:  "verbose, v",
			Usage: "More verbose output",
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "Suppress the summary at the end of a test",
		},
		cli.StringFlag{
			Name:  "format, f",
			Usage: "Format for printed metrics (yaml, json, prettyjson)",
			Value: "yaml",
		},
		cli.DurationFlag{
			Name:  "interval, i",
			Usage: "Write periodic summaries",
		},
		cli.StringSliceFlag{
			Name:  "out, o",
			Usage: "Write metrics to a database",
		},
		cli.BoolFlag{
			Name:  "raw",
			Usage: "Instead of summaries, dump raw samples to stdout",
		},
		cli.StringSliceFlag{
			Name:  "select, s",
			Usage: "Include only named metrics",
		},
		cli.StringSliceFlag{
			Name:  "exclude, e",
			Usage: "Exclude named metrics",
		},
		cli.StringSliceFlag{
			Name:  "group-by, g",
			Usage: "Group metrics by tags",
		},
	}
	app.Before = func(cc *cli.Context) error {
		invocation.PopulateWithContext(cc)
		return nil
	}
	app.Action = action
	invocationError <- app.Run(os.Args)
}
<|endoftext|>


<commit_before>package gopcap

// EthernetFrame represents a single ethernet frame. Valid only when the LinkType is ETHERNET.
type EthernetFrame struct {
	MACSource      [6]byte
	MACDestination [6]byte
	VLANTag        uint32
	Length         uint16
	EtherType      uint16
	data           []byte
}

func (e EthernetFrame) Data() []byte {
	return data
}
<commit_msg>Fixup clearly broken method.<commit_after>package gopcap

// EthernetFrame represents a single ethernet frame. Valid only when the LinkType is ETHERNET.
type EthernetFrame struct {
	MACSource      [6]byte
	MACDestination [6]byte
	VLANTag        uint32
	Length         uint16
	EtherType      uint16
	data           []byte
}

func (e *EthernetFrame) Data() []byte {
	return e.data
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net"
	"time"

	"github.com/dasfoo/i2c"
	"github.com/dasfoo/rover/bb"
	"github.com/dasfoo/rover/mc"
	"golang.org/x/net/context"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"

	pb "github.com/dasfoo/rover/proto"
)

// server is used to implement roverserver.RoverServiceServer.
type server struct{}

var (
	board  *bb.BB
	motors *mc.MC

	tls      = flag.Bool("tls", false, "Connection uses TLS if true, else plain TCP")
	laddr    = flag.String("laddr", "", "laddr")
	test     = flag.Bool("test", false, "Flag for startup script")
	certFile = flag.String("cert_file", "", "The TLS cert file")
	keyFile  = flag.String("key_file", "", "The TLS key file")
)

func getError(e error) error {
	errf := grpc.Errorf // Confuse `go vet' to not check this `Errorf' call. :(
	// See https://github.com/grpc/grpc-go/issues/90
	return errf(codes.Unavailable, "%s", e.Error())
}

// MoveRover implements
func (s *server) MoveRover(ctx context.Context,
	in *pb.RoverWheelRequest) (*pb.RoverWheelResponse, error) {
	_ = motors.Left(int8(in.Left)) //TODO error check
	_ = motors.Right(int8(in.Right))
	time.Sleep(1 * time.Second)

	_ = motors.Left(0)
	_ = motors.Right(0)
	return &pb.RoverWheelResponse{}, nil
}

func (s *server) GetBatteryPercentage(ctx context.Context,
	in *pb.BatteryPercentageRequest) (*pb.BatteryPercentageResponse, error) {
	var batteryPercentage byte
	var e error
	if batteryPercentage, e = board.GetBatteryPercentage(); e != nil {
		return nil, getError(e)
	}
	return &pb.BatteryPercentageResponse{
		Battery: int32(batteryPercentage),
	}, nil
}

func (s *server) GetAmbientLight(ctx context.Context,
	in *pb.AmbientLightRequest) (*pb.AmbientLightResponse, error) {
	var light uint16
	var e error
	if light, e = board.GetAmbientLight(); e != nil {
		return nil, getError(e)
	}
	return &pb.AmbientLightResponse{
		Light: int32(light),
	}, nil
}

func (s *server) GetTemperatureAndHumidity(ctx context.Context,
	in *pb.TemperatureAndHumidityRequest) (*pb.TemperatureAndHumidityResponse, error) {
	var t, h byte
	var e error
	if t, h, e = board.GetTemperatureAndHumidity(); e != nil {
		return nil, getError(e)
	}
	return &pb.TemperatureAndHumidityResponse{
		Temperature: int32(t),
		Humidity:    int32(h),
	}, nil
}

func (s *server) ReadEncoders(ctx context.Context,
	in *pb.ReadEncodersRequest) (*pb.ReadEncodersResponse, error) {
	var leftFront, leftBack, rightFront, rightBack int32
	var e error
	if leftFront, e = motors.ReadEncoder(mc.EncoderLeftFront); e != nil {
		return nil, getError(e)
	}
	if leftBack, e = motors.ReadEncoder(mc.EncoderLeftBack); e != nil {
		return nil, getError(e)
	}
	if rightFront, e = motors.ReadEncoder(mc.EncoderRightFront); e != nil {
		return nil, getError(e)
	}
	if rightBack, e = motors.ReadEncoder(mc.EncoderRightBack); e != nil {
		return nil, getError(e)
	}
	return &pb.ReadEncodersResponse{
		LeftFront:  leftFront,
		LeftBack:   leftBack,
		RightFront: rightFront,
		RightBack:  rightBack,
	}, nil
}

func setServerOptions() ([]grpc.ServerOption, error) {
	var opts []grpc.ServerOption
	if *tls {
		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
		if err != nil {
			return opts, err
		}
		return []grpc.ServerOption{grpc.Creds(creds)}, nil
	}
	return opts, nil
}

func startServer() error {
	lis, err := net.Listen("tcp", *laddr)
	if err != nil {
		log.Println("Failed to listen:", err)
	}
	opts, err := setServerOptions()
	if err != nil {
		log.Println("Failed to setup Server Options:", err)
	}
	s := grpc.NewServer(opts...)
	pb.RegisterRoverServiceServer(s, &server{})
	return s.Serve(lis)
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile | log.Lmicroseconds)
	flag.Parse()
	log.Println("Properties from command line:", *laddr)
	log.Println("Flag for startup script", *test)
	if bus, err := i2c.NewBus(1); err != nil {
		log.Fatal(err)
	} else {
		// Silence i2c bus log
		//bus.SetLogger(func(string, ...interface{}) {})

		board = bb.NewBB(bus, bb.Address)
		motors = mc.NewMC(bus, mc.Address)
	}
	if err := startServer(); err != nil {
		log.Println("Failed to start server :", err)
	}
}
<commit_msg>Add return in exeptional behavior<commit_after>package main

import (
	"flag"
	"log"
	"net"
	"time"

	"github.com/dasfoo/i2c"
	"github.com/dasfoo/rover/bb"
	"github.com/dasfoo/rover/mc"
	"golang.org/x/net/context"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"

	pb "github.com/dasfoo/rover/proto"
)

// server is used to implement roverserver.RoverServiceServer.
type server struct{}

var (
	board  *bb.BB
	motors *mc.MC

	tls      = flag.Bool("tls", false, "Connection uses TLS if true, else plain TCP")
	laddr    = flag.String("laddr", "", "laddr")
	test     = flag.Bool("test", false, "Flag for startup script")
	certFile = flag.String("cert_file", "", "The TLS cert file")
	keyFile  = flag.String("key_file", "", "The TLS key file")
)

func getError(e error) error {
	errf := grpc.Errorf // Confuse `go vet' to not check this `Errorf' call. :(
	// See https://github.com/grpc/grpc-go/issues/90
	return errf(codes.Unavailable, "%s", e.Error())
}

// MoveRover implements
func (s *server) MoveRover(ctx context.Context,
	in *pb.RoverWheelRequest) (*pb.RoverWheelResponse, error) {
	_ = motors.Left(int8(in.Left)) //TODO error check
	_ = motors.Right(int8(in.Right))
	time.Sleep(1 * time.Second)

	_ = motors.Left(0)
	_ = motors.Right(0)
	return &pb.RoverWheelResponse{}, nil
}

func (s *server) GetBatteryPercentage(ctx context.Context,
	in *pb.BatteryPercentageRequest) (*pb.BatteryPercentageResponse, error) {
	var batteryPercentage byte
	var e error
	if batteryPercentage, e = board.GetBatteryPercentage(); e != nil {
		return nil, getError(e)
	}
	return &pb.BatteryPercentageResponse{
		Battery: int32(batteryPercentage),
	}, nil
}

func (s *server) GetAmbientLight(ctx context.Context,
	in *pb.AmbientLightRequest) (*pb.AmbientLightResponse, error) {
	var light uint16
	var e error
	if light, e = board.GetAmbientLight(); e != nil {
		return nil, getError(e)
	}
	return &pb.AmbientLightResponse{
		Light: int32(light),
	}, nil
}

func (s *server) GetTemperatureAndHumidity(ctx context.Context,
	in *pb.TemperatureAndHumidityRequest) (*pb.TemperatureAndHumidityResponse, error) {
	var t, h byte
	var e error
	if t, h, e = board.GetTemperatureAndHumidity(); e != nil {
		return nil, getError(e)
	}
	return &pb.TemperatureAndHumidityResponse{
		Temperature: int32(t),
		Humidity:    int32(h),
	}, nil
}

func (s *server) ReadEncoders(ctx context.Context,
	in *pb.ReadEncodersRequest) (*pb.ReadEncodersResponse, error) {
	var leftFront, leftBack, rightFront, rightBack int32
	var e error
	if leftFront, e = motors.ReadEncoder(mc.EncoderLeftFront); e != nil {
		return nil, getError(e)
	}
	if leftBack, e = motors.ReadEncoder(mc.EncoderLeftBack); e != nil {
		return nil, getError(e)
	}
	if rightFront, e = motors.ReadEncoder(mc.EncoderRightFront); e != nil {
		return nil, getError(e)
	}
	if rightBack, e = motors.ReadEncoder(mc.EncoderRightBack); e != nil {
		return nil, getError(e)
	}
	return &pb.ReadEncodersResponse{
		LeftFront:  leftFront,
		LeftBack:   leftBack,
		RightFront: rightFront,
		RightBack:  rightBack,
	}, nil
}

func setServerOptions() ([]grpc.ServerOption, error) {
	var opts []grpc.ServerOption
	if *tls {
		creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
		if err != nil {
			return opts, err
		}
		return []grpc.ServerOption{grpc.Creds(creds)}, nil
	}
	return opts, nil
}

func startServer() error {
	lis, err := net.Listen("tcp", *laddr)
	if err != nil {
		log.Println("Failed to listen:", err)
		return err
	}
	opts, err := setServerOptions()
	if err != nil {
		log.Println("Failed to setup Server Options:", err)
		return err
	}
	s := grpc.NewServer(opts...)
	pb.RegisterRoverServiceServer(s, &server{})
	return s.Serve(lis)
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile | log.Lmicroseconds)
	flag.Parse()
	log.Println("Properties from command line:", *laddr)
	log.Println("Flag for startup script", *test)
	if bus, err := i2c.NewBus(1); err != nil {
		log.Fatal(err)
	} else {
		// Silence i2c bus log
		//bus.SetLogger(func(string, ...interface{}) {})

		board = bb.NewBB(bus, bb.Address)
		motors = mc.NewMC(bus, mc.Address)
	}
	if err := startServer(); err != nil {
		log.Println("Failed to start server :", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"time"

	"crypto/sha256"
	"math/big"

	"encoding/binary"
	"encoding/json"
)

var (
	BuiltAt string
	Version = "Unknown"

	printVersion = flag.Bool("v", false, "Display the version and exit")
)

type HashCash struct {
	Data      []byte `json:"data"`
	Timestamp int64  `json:"timestamp"`
	Nonce     int64  `json:"nonce"`
}

func (hc *HashCash) Check(zeroes int) bool {
	digest := sha256.Sum256(hc.ContentHash())

	digestHex := new(big.Int).SetBytes(digest[:])
	if digestHex.BitLen() == 256-zeroes {
		return true
	}

	return false
}

func (hc *HashCash) ContentHash() []byte {
	var buf bytes.Buffer

	buf.Write(hc.Data)
	binary.Write(&buf, binary.BigEndian, hc.Timestamp)
	binary.Write(&buf, binary.BigEndian, hc.Nonce)

	return buf.Bytes()
}

func (hc *HashCash) FindProof(zeroes int) {
	if hc.Check(zeroes) {
		return
	}

	hc.Nonce = 0
	hc.Timestamp = time.Now().Unix()

	for {
		if hc.Check(zeroes) {
			return
		}

		hc.Nonce++
	}
}

func main() {
	flag.Parse()

	if *printVersion {
		fmt.Printf("HashCash version %s built on %s\n", Version, BuiltAt)
		os.Exit(0)
	}

	hc := HashCash{
		Data: []byte("Just some test data in the string"),
	}
	hc.FindProof(20)

	json, _ := json.Marshal(hc)
	fmt.Println(string(json))
}
<commit_msg>Added a salt for stronger policies<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"time"

	"crypto/rand"
	"crypto/sha256"
	"math/big"

	"encoding/binary"
	"encoding/json"
)

var (
	BuiltAt string
	Version = "Unknown"

	printVersion = flag.Bool("v", false, "Display the version and exit")
)

type HashCash struct {
	Data      []byte    `json:"data"`
	Nonce     int64     `json:"nonce"`
	Salt      []byte    `json:"salt"`
	Timestamp time.Time `json:"timestamp"`
}

func (hc *HashCash) Check(zeroes int) bool {
	if hc.ZeroCount() >= zeroes {
		return true
	}

	return false
}

func (hc *HashCash) ContentHash() []byte {
	var buf bytes.Buffer

	buf.Write(hc.Data)
	binary.Write(&buf, binary.BigEndian, hc.Timestamp.Unix())
	buf.Write(hc.Salt)
	binary.Write(&buf, binary.BigEndian, hc.Nonce)

	return buf.Bytes()
}

func (hc *HashCash) FindProof(zeroes int) {
	if hc.Check(zeroes) {
		return
	}

	hc.Nonce = 0
	hc.Timestamp = time.Now()

	hc.Salt = make([]byte, 16)
	rand.Read(hc.Salt)

	for {
		if hc.Check(zeroes) {
			return
		}

		hc.Nonce++
	}
}

func (hc *HashCash) ZeroCount() int {
	digest := sha256.Sum256(hc.ContentHash())
	digestHex := new(big.Int).SetBytes(digest[:])
	return (256 - digestHex.BitLen())
}

func main() {
	flag.Parse()

	if *printVersion {
		fmt.Printf("HashCash version %s built on %s\n", Version, BuiltAt)
		os.Exit(0)
	}

	hc := HashCash{
		Data: []byte("Just some test data in the string"),
	}

	hc.FindProof(20)

	json, _ := json.Marshal(hc)
	fmt.Println(string(json))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
)

func usage() {
	os.Stderr.WriteString(`
Usage: circlecrossgame [OPTION]...
Play circle cross game on terminal.

Options:
	--help       show this help message
	--version    print the version
`[1:])
}

func version() {
	os.Stderr.WriteString(`
0.1.1
`[1:])
}

func _main() error {
	f := flag.NewFlagSet("circlecrossgame:", flag.ContinueOnError)
	f.SetOutput(ioutil.Discard)

	isHelp := f.Bool("help", false, "")
	isVersion := f.Bool("version", false, "")
	if err := f.Parse(os.Args[1:]); err != nil {
		return err
	}
	switch {
	case *isHelp:
		usage()
		return nil
	case *isVersion:
		version()
		return nil
	}

	g := NewGame()
	if err := g.Init(); err != nil {
		return err
	}
	return g.Main()
}

func main() {
	if err := _main(); err != nil {
		fmt.Fprintln(os.Stderr, "circlecrossgame:", err)
		os.Exit(1)
	}
}
<commit_msg>Upgrade to change quit key to q and ctrl-c<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
)

func usage() {
	os.Stderr.WriteString(`
Usage: circlecrossgame [OPTION]...
Play circle cross game on terminal.

Options:
	--help       show this help message
	--version    print the version
`[1:])
}

func version() {
	os.Stderr.WriteString(`
0.2.0
`[1:])
}

func _main() error {
	f := flag.NewFlagSet("circlecrossgame:", flag.ContinueOnError)
	f.SetOutput(ioutil.Discard)

	isHelp := f.Bool("help", false, "")
	isVersion := f.Bool("version", false, "")
	if err := f.Parse(os.Args[1:]); err != nil {
		return err
	}
	switch {
	case *isHelp:
		usage()
		return nil
	case *isVersion:
		version()
		return nil
	}

	g := NewGame()
	if err := g.Init(); err != nil {
		return err
	}
	return g.Main()
}

func main() {
	if err := _main(); err != nil {
		fmt.Fprintln(os.Stderr, "circlecrossgame:", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"time"
	"encoding/json"
	"github.com/julienschmidt/httprouter"
	"net/http"
	"log"
	"os"
	"fmt"
	"database/sql"
	_ "github.com/lib/pq"
)

var conn *sql.DB

type BlogPost struct {
	Id int64 `json:"id"`
	Titel string `json:"titel"`
	Text string `json:"text"`
	Auteur string `json:"auteur"`
	Img_url string `json:"img_url"`
	Ctime time.Time `json:"ctime"`
	Image string `json:"image"`
}

type SpinData struct {
	Id int64 `json:"id"`
	Tijd time.Time `json:"tijd"`
	Mode string `json:"mode"`
	Hellingsgraad int64 `json:"hellingsgraad"`
	Snelheid int64 `json:"snelheid"`
	Batterij int64 `json:"batterij"`
	BallonCount int64 `json:"ballonCount"`
}

type ServoData struct {
	Id int64 `json:"id"`
	ServoId int64 `json:"servo_id"`
	Tijd time.Time `json:"tijd"`
	Voltage int64 `json:"voltage"`
	Positie int64 `json:"positie"`
	Load int64 `json:"load"`
	Temperatuur int64 `json:"temperatuur"`
}

func GetBlog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM blog")
	data := []BlogPost{}
	for rows.Next() {
		post := BlogPost{}
		rows.Scan(&post.Id, &post.Titel, &post.Text, &post.Auteur, &post.Img_url, &post.Ctime, &post.Image)
		data = append(data, post)
	}
	buf,_ := json.Marshal(data)
	w.Write(buf)
}

func GetSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM spindata")
	data := []SpinData{}
	for rows.Next() {
		spin := SpinData{}
		rows.Scan(&spin.Id, &spin.Tijd, &spin.Mode, &spin.Hellingsgraad, &spin.Snelheid, &spin.Batterij, &spin.BallonCount)
		data = append(data, spin)
	}
	buf,_ := json.Marshal(data)
	w.Write(buf)
}

func GetServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	data,_ := json.Marshal("blah")
	w.Write(data)
}

func PostBlog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	data,_ := json.Marshal("blah")
	w.Write(data)
}

func PostSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	data,_ := json.Marshal("blah")
	w.Write(data)
}

func PostServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	data,_ := json.Marshal("blah")
	w.Write(data)
}

func main() {
	conn,_ = sql.Open("postgres", os.Getenv("DATABASE_URL"))
	defer conn.Close()
	router := httprouter.New()
	router.GET("/blog", GetBlog)
	router.GET("/spin", GetSpinData)
	router.GET("/servo", GetServoData)
	router.POST("/blog", PostBlog)
	router.POST("/spin", PostSpinData)
	router.POST("/servo", PostServoData)
	port := os.Getenv("PORT")
	if port == "" {
		port = "3000"
	}
	fmt.Printf("Starting server at localhost:%s...", port)
	log.Fatal(http.ListenAndServe(":"+port, router))
}<commit_msg>teapot<commit_after>package main

import (
	"time"
	"encoding/json"
	"github.com/julienschmidt/httprouter"
	"net/http"
	"log"
	"os"
	"fmt"
	"database/sql"
	_ "github.com/lib/pq"
)

var conn *sql.DB

type BlogPost struct {
	Id int64 `json:"id"`
	Titel string `json:"titel"`
	Text string `json:"text"`
	Auteur string `json:"auteur"`
	Img_url string `json:"img_url"`
	Ctime time.Time `json:"ctime"`
	Image string `json:"image"`
}

type SpinData struct {
	Id int64 `json:"id"`
	Tijd time.Time `json:"tijd"`
	Mode string `json:"mode"`
	Hellingsgraad int64 `json:"hellingsgraad"`
	Snelheid int64 `json:"snelheid"`
	Batterij int64 `json:"batterij"`
	BallonCount int64 `json:"ballonCount"`
}

type ServoData struct {
	Id int64 `json:"id"`
	ServoId int64 `json:"servo_id"`
	Tijd time.Time `json:"tijd"`
	Voltage int64 `json:"voltage"`
	Positie int64 `json:"positie"`
	Load int64 `json:"load"`
	Temperatuur int64 `json:"temperatuur"`
}

func GetBlog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM blog")
	data := []BlogPost{}
	for rows.Next() {
		post := BlogPost{}
		rows.Scan(&post.Id, &post.Titel, &post.Text, &post.Auteur, &post.Img_url, &post.Ctime, &post.Image)
		data = append(data, post)
	}
	buf,_ := json.Marshal(data)
	w.Write(buf)
}

func GetLatestSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM spindata ORDER BY tijd DESC LIMIT 1")
	spin := SpinData{}
	rows.Scan(&spin.Id, &spin.Tijd, &spin.Mode, &spin.Hellingsgraad, &spin.Snelheid, &spin.Batterij, &spin.BallonCount)
	buf,_ := json.Marshal(spin)
	w.Write(buf)
}

func GetArchivedSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM spindata")
	data := []SpinData{}
	for rows.Next() {
		spin := SpinData{}
		rows.Scan(&spin.Id, &spin.Tijd, &spin.Mode, &spin.Hellingsgraad, &spin.Snelheid, &spin.Batterij, &spin.BallonCount)
		data = append(data, spin)
	}
	buf,_ := json.Marshal(data)
	w.Write(buf)
}

func GetLatestServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM servodata ORDER BY tijd DESC LIMIT 1")
	servo := ServoData{}
	rows.Scan(&servo.Id, &servo.ServoId, &servo.Tijd, &servo.Voltage, &servo.Positie, &servo.Load, &servo.Temperatuur)
	buf,_ := json.Marshal(servo)
	w.Write(buf)
}

func GetArchivedServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM servodata")
	data := []ServoData{}
	for rows.Next() {
		servo := ServoData{}
		rows.Scan(&servo.Id, &servo.ServoId, &servo.Tijd, &servo.Voltage, &servo.Positie, &servo.Load, &servo.Temperatuur)
		data = append(data, servo)
	}
	buf,_ := json.Marshal(data)
	w.Write(buf)
}

func PostBlog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {

	/*_,err := db.Query("INSERT INTO blog (titel, text, auteur, ctime, image) VALUES ($1, $2, $3, $4, $5)", )
	if err != nil {
		w.WriteHead(500)
		w.Write([]byte("error posting"))
		return
	}
	w.WriteHead(201)
	w.Write([]byte("successful"))*/
}

func PostSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	data,_ := json.Marshal("blah")
	w.Write(data)
}

func PostServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	data,_ := json.Marshal("blah")
	w.Write(data)
}

func TeaPot(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	w.WriteHead(418)
}

func main() {
	conn,_ = sql.Open("postgres", os.Getenv("DATABASE_URL"))
	defer conn.Close()
	router := httprouter.New()
	router.GET("/blog", GetBlog)
	router.GET("/spin/latest", GetLatestSpinData)
	router.GET("/spin/archive", GetArchivedSpinData)
	router.GET("/servo/latest", GetLatestServoData)
	router.GET("/servo/archive", GetArchivedServoData)
	router.GET("/teapot", TeaPot)
	router.POST("/blog", PostBlog)
	router.POST("/spin", PostSpinData)
	router.POST("/servo", PostServoData)
	port := os.Getenv("PORT")
	if port == "" {
		port = "3000"
	}
	fmt.Printf("Starting server at localhost:%s...", port)
	log.Fatal(http.ListenAndServe(":"+port, router))
}<|endoftext|>


<commit_before>package watchdog

import "github.com/efritz/backoff"

// BlockUntilSuccess creates a watcher that fires until the given retry
// function returns a success, then disables the watcher. This function
// is synchronous.
func BlockUntilSuccess(retry Retry, backoff backoff.Backoff) {
	watcher := NewWatcher(retry, backoff)
	<-watcher.Start()
	watcher.Stop()
}
<commit_msg>(Closes #2) Add BlockUntilSuccessTimeout.<commit_after>package watchdog

import (
	"fmt"
	"time"

	"github.com/efritz/backoff"
)

// BlockUntilSuccess creates a transient watcher that fires the given
// retry function until success.
func BlockUntilSuccess(retry Retry, backoff backoff.Backoff) {
	watcher := NewWatcher(retry, backoff)
	defer watcher.Stop()

	<-watcher.Start()
}

// BlockUntilSuccessTimeout creates a transient watcher that fires the
// given retry function until success or until the specified timeout
// elapses. An error is returned in the later case.
func BlockUntilSuccessTimeout(retry Retry, backoff backoff.Backoff, timeout time.Duration) error {
	watcher := NewWatcher(retry, backoff)
	defer watcher.Stop()

	select {
	case <-watcher.Start():
		return nil
	case <-time.After(timeout):
		return fmt.Errorf("timeout elapsed")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"net"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/miekg/dns"
)

const (
	Version = "0.0.3"
)

var (
	root  string
	debug bool
)

func rrsearch(ns string, q string, t uint16) *dns.Msg {
	c := new(dns.Client)
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(q), t)

	if debug {
		log.Printf("NS %v,Question %v\n", ns, q)
	}
	r, _, err := c.Exchange(m, net.JoinHostPort(ns, "53"))
	if debug {
		log.Printf("NS %v,Response %v\n", ns, r.String())
	}
	if err != nil {
		fmt.Println(err.Error())
		panic("rrsearch Error")
	}
	return r
}

func splitRR(rr dns.RR) []string {
	if debug {
		log.Printf("RR %v\n", rr.String())
	}
	rrary := strings.SplitN(rr.String(), "\t", 5)
	return rrary
}

func setNS(rrs []string, r *dns.Msg, qns string) (string, string) {
	if debug {
		log.Printf("RRS %v,DNS Message %v Qns %v\n", rrs, r.String(), qns)
	}

	var ns string
	typ := "A"

	if strings.Contains(rrs[4], rrs[0]) || strings.Compare(qns, root) == 0 {
		for _, rr := range r.Extra {
			rrss := splitRR(rr)
			if strings.Compare(rrss[3], "AAAA") == 0 {
				break
			}
			if strings.Compare(rrss[0], rrs[4]) == 0 {
				ns = rrss[4]
				typ = rrss[3]
			}
		}
	} else {
		ns = rrs[4]
	}

	return ns, typ
}

func noRecRsolve(dst, ns string) string {
	rand.Seed(time.Now().UnixNano())
	for {
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)
			for strings.Compare(typ, "AAAA") == 0 {
				if debug {
					log.Printf("Type %v\n", typ)
				}
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
			}
			fmt.Println(ns + "=>")
			fmt.Printf("\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			if strings.Compare(rrs[0], ".") == 0 {
				fmt.Println("Domain Name Not found")
				os.Exit(1)
			}
			ns = nss
		} else {
			fmt.Println("Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf("\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func recRsolve(dst, ns string, n int) string {
	var tab string
	for i := 0; i < n; i++ {
		tab = tab + "\t"
	}
	rand.Seed(time.Now().UnixNano())
	for {
		checkNS, _ := regexp.MatchString("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", ns)
		if checkNS == false {
			if debug {
				log.Printf("NS %v,Root %v\n", ns, root)
			}
			ns = recRsolve(ns, root, n+1)
			if debug {
				log.Printf("NS %v,Root %v\n", ns, root)
			}
		}
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)

			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
				if debug {
					log.Printf("Type %v\n", typ)
				}
			}
			fmt.Println(tab + ns + "=>")
			fmt.Printf(tab+"\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			if strings.Compare(rrs[0], ".") == 0 {
				fmt.Println("Domain Name Not found")
				os.Exit(1)
			}

			ns = nss
		} else {
			fmt.Println(tab + "Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf(tab+"\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func main() {
	app := cli.NewApp()
	app.Usage = "It is a tool to see recursive search of the DNS is how to."
	app.Version = Version

	app.Commands = []cli.Command{
		{
			Name:    "rec",
			Aliases: []string{"r"},
			Usage:   "Iterate Search",
			Action: func(c *cli.Context) {
				if debug {
					log.SetFlags(log.Llongfile)
				}
				recRsolve(c.Args().First(), root, 0)
			},
		},
		{
			Name:    "norec",
			Aliases: []string{"n"},
			Usage:   "No Iterate Search",
			Action: func(c *cli.Context) {
				if debug {
					log.SetFlags(log.Llongfile)
				}
				noRecRsolve(c.Args().First(), root)
			},
		},
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "root",
			Value:       "202.12.27.33",
			Usage:       "Root DNS Server's IP address",
			Destination: &root,
		},
		cli.BoolFlag{
			Name:        "debug",
			Usage:       "Root DNS Server's IP address",
			EnvVar:      "false",
			Destination: &debug,
		},
	}
	app.Run(os.Args)
}
<commit_msg>エラー確認の追加<commit_after>package main

import (
	"fmt"
	"log"
	"math/rand"
	"net"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/miekg/dns"
)

const (
	Version = "0.0.3"
)

var (
	root  string
	debug bool
)

func rrsearch(ns string, q string, t uint16) *dns.Msg {
	c := new(dns.Client)
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(q), t)

	if debug {
		log.Printf("NS %v,Question %v\n", ns, q)
	}
	r, _, err := c.Exchange(m, net.JoinHostPort(ns, "53"))
	if debug {
		log.Printf("NS %v,Response %v\n", ns, r.String())
	}
	if err != nil {
		fmt.Println(err.Error())
		panic("rrsearch Error")
	}
	return r
}

func splitRR(rr dns.RR) []string {
	if debug {
		log.Printf("RR %v\n", rr.String())
	}
	rrary := strings.SplitN(rr.String(), "\t", 5)
	return rrary
}

func setNS(rrs []string, r *dns.Msg, qns string) (string, string) {
	if debug {
		log.Printf("RRS %v,DNS Message %v Qns %v\n", rrs, r.String(), qns)
	}

	var ns string
	typ := "A"

	if strings.Contains(rrs[4], rrs[0]) || strings.Compare(qns, root) == 0 {
		for _, rr := range r.Extra {
			rrss := splitRR(rr)
			if strings.Compare(rrss[3], "AAAA") == 0 {
				break
			}
			if strings.Compare(rrss[0], rrs[4]) == 0 {
				ns = rrss[4]
				typ = rrss[3]
			}
		}
	} else {
		ns = rrs[4]
	}

	return ns, typ
}

func noRecRsolve(dst, ns string) string {
	rand.Seed(time.Now().UnixNano())
	for {
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)
			for strings.Compare(typ, "AAAA") == 0 {
				if debug {
					log.Printf("Type %v\n", typ)
				}
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
			}
			fmt.Println(ns + "=>")
			fmt.Printf("\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			if strings.Compare(rrs[0], ".") == 0 {
				fmt.Println("Domain Name Not found")
				os.Exit(1)
			}
			ns = nss
		} else {
			fmt.Println("Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf("\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func recRsolve(dst, ns string, n int) string {
	var tab string
	for i := 0; i < n; i++ {
		tab = tab + "\t"
	}
	rand.Seed(time.Now().UnixNano())
	for {
		checkNS, _ := regexp.MatchString("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", ns)
		if checkNS == false {
			if debug {
				log.Printf("NS %v,Root %v\n", ns, root)
			}
			ns = recRsolve(ns, root, n+1)
			if debug {
				log.Printf("NS %v,Root %v\n", ns, root)
			}
		}
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r, ns)

			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r, ns)
				if debug {
					log.Printf("Type %v\n", typ)
				}
			}
			fmt.Println(tab + ns + "=>")
			fmt.Printf(tab+"\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			if strings.Compare(rrs[0], ".") == 0 {
				fmt.Println("Domain Name Not found")
				os.Exit(1)
			}

			ns = nss
		} else {
			fmt.Println(tab + "Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf(tab+"\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func main() {
	app := cli.NewApp()
	app.Usage = "It is a tool to see recursive search of the DNS is how to."
	app.Version = Version

	app.Commands = []cli.Command{
		{
			Name:    "rec",
			Aliases: []string{"r"},
			Usage:   "Iterate Search",
			Action: func(c *cli.Context) {
				if debug {
					log.SetFlags(log.Llongfile)
				}
				recRsolve(c.Args().First(), root, 0)
			},
		},
		{
			Name:    "norec",
			Aliases: []string{"n"},
			Usage:   "No Iterate Search",
			Action: func(c *cli.Context) {
				if debug {
					log.SetFlags(log.Llongfile)
				}
				noRecRsolve(c.Args().First(), root)
			},
		},
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "root",
			Value:       "202.12.27.33",
			Usage:       "Root DNS Server's IP address",
			Destination: &root,
		},
		cli.BoolFlag{
			Name:        "debug",
			Usage:       "Debug Option for Developer",
			EnvVar:      "false",
			Destination: &debug,
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"kubernetes-cloudwatch-exporter/util"

	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	_settingsFile = flag.String("settings-file", "./settings.json", "Path to load as the settings file")
	_elbMetrics   = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "k8s_cw_metric",
			Help: "Kubernetes ELB metrics",
		},
		[]string{"elb", "app", "namespace", "metric", "statistic"},
	)
)

func init() {
	prometheus.MustRegister(_elbMetrics)
}

func main() {
	settings, err := util.NewSettings(*_settingsFile)

	if err != nil {
		log.Fatalf("settings.NewSettings %v", err)
	}

	sess := session.Must(session.NewSession(&aws.Config{
		Region: aws.String(settings.AWSRegion),
	}))

	getELBNamesFunc := util.MakeELBNamesFunc(settings.TagName, settings.TagValue, settings.AppTagName, sess)
	getMetricsFunc := util.MakeMetricsFunc(sess)

	go func() {
		for {
			elbDescriptions, err := getELBNamesFunc()

			if err != nil {
				log.Fatalf("elbFunc %v", err)
			}

			fmt.Printf("Found %d load balancers\n", len(elbDescriptions))

			for _, elbDesc := range elbDescriptions {
				for _, elbMetric := range settings.Metrics {

					log.Printf("Requesting Metrics %v", elbMetric)
					log.Printf("Requesting for ELB %v", *elbDesc.Name)

					datapoints, err := getMetricsFunc(elbDesc.Name, &elbMetric, settings)

					if err != nil {
						log.Fatalf("metricsFunc %v", err)
					}

					log.Printf("Datapoints %v", datapoints)

					observeDatapoints(datapoints, elbMetric, elbDesc)
				}
			}

			time.Sleep(60 * time.Second)
		}
	}()

	// Expose the registered metrics via HTTP.
	http.Handle("/metrics", promhttp.Handler())
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func observeDatapoints(datapoints []*cloudwatch.Datapoint, elbMetric util.ELBMetric, elbDesc *util.ELBDescription) {

	for _, dp := range datapoints {
		for n, v := range getMetrics(dp) {
			_elbMetrics.WithLabelValues(*elbDesc.Name, *elbDesc.AppName, *elbDesc.AppNamespace, elbMetric.Name, n).Set(v)
		}
	}
}

func getMetrics(dp *cloudwatch.Datapoint) map[string]float64 {

	metrics := make(map[string]float64)

	if dp.Average != nil {
		metrics["Average"] = *dp.Average
	}

	if dp.Maximum != nil {
		metrics["Maximum"] = *dp.Maximum
	}

	if dp.Minimum != nil {
		metrics["Minimum"] = *dp.Minimum
	}

	if dp.SampleCount != nil {
		metrics["SampleCount"] = *dp.SampleCount
	}

	if dp.Sum != nil {
		metrics["Sum"] = *dp.Sum
	}

	if dp.ExtendedStatistics != nil {
		for p, v := range dp.ExtendedStatistics {
			metrics[p] = *v
		}
	}

	return metrics
}
<commit_msg>Better naming<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"kubernetes-cloudwatch-exporter/util"

	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	_settingsFile = flag.String("settings-file", "./settings.json", "Path to load as the settings file")
	_elbMetrics   = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "k8scw",
			Subsystem: "elb",
			Name:      "metric",
			Help:      "Kubernetes ELB metrics",
		},
		[]string{"elb", "app", "namespace", "metric", "statistic"},
	)
)

func init() {
	prometheus.MustRegister(_elbMetrics)
}

func main() {
	settings, err := util.NewSettings(*_settingsFile)

	if err != nil {
		log.Fatalf("settings.NewSettings %v", err)
	}

	sess := session.Must(session.NewSession(&aws.Config{
		Region: aws.String(settings.AWSRegion),
	}))

	getELBNamesFunc := util.MakeELBNamesFunc(settings.TagName, settings.TagValue, settings.AppTagName, sess)
	getMetricsFunc := util.MakeMetricsFunc(sess)

	go func() {
		for {
			elbDescriptions, err := getELBNamesFunc()

			if err != nil {
				log.Fatalf("elbFunc %v", err)
			}

			fmt.Printf("Found %d load balancers\n", len(elbDescriptions))

			for _, elbDesc := range elbDescriptions {
				for _, elbMetric := range settings.Metrics {

					log.Printf("Requesting Metrics %v", elbMetric)
					log.Printf("Requesting for ELB %v", *elbDesc.Name)

					datapoints, err := getMetricsFunc(elbDesc.Name, &elbMetric, settings)

					if err != nil {
						log.Fatalf("metricsFunc %v", err)
					}

					log.Printf("Datapoints %v", datapoints)

					observeDatapoints(datapoints, elbMetric, elbDesc)
				}
			}

			time.Sleep(60 * time.Second)
		}
	}()

	// Expose the registered metrics via HTTP.
	http.Handle("/metrics", promhttp.Handler())
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func observeDatapoints(datapoints []*cloudwatch.Datapoint, elbMetric util.ELBMetric, elbDesc *util.ELBDescription) {

	for _, dp := range datapoints {
		for n, v := range getMetrics(dp) {
			_elbMetrics.WithLabelValues(*elbDesc.Name, *elbDesc.AppName, *elbDesc.AppNamespace, elbMetric.Name, n).Set(v)
		}
	}
}

func getMetrics(dp *cloudwatch.Datapoint) map[string]float64 {

	metrics := make(map[string]float64)

	if dp.Average != nil {
		metrics["Average"] = *dp.Average
	}

	if dp.Maximum != nil {
		metrics["Maximum"] = *dp.Maximum
	}

	if dp.Minimum != nil {
		metrics["Minimum"] = *dp.Minimum
	}

	if dp.SampleCount != nil {
		metrics["SampleCount"] = *dp.SampleCount
	}

	if dp.Sum != nil {
		metrics["Sum"] = *dp.Sum
	}

	if dp.ExtendedStatistics != nil {
		for p, v := range dp.ExtendedStatistics {
			metrics[p] = *v
		}
	}

	return metrics
}
<|endoftext|>


<commit_before>// Copyright 2014 James McGuire. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"

	irc "github.com/fluffle/goirc/client"
	_ "github.com/go-sql-driver/mysql"
	"golang.org/x/net/html/charset"
)

var (
	config   Config
	urlRegex = regexp.MustCompile(`(?i)\b((?:https?://|www\d{0,3}[.]|[` +
		`a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+` +
		`\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[` +
		`\]{};:'".,<>?«»“”‘’]))`)
	httpRegex      = regexp.MustCompile(`https?://.*`)
	findWhiteSpace = regexp.MustCompile(`\s+`)
	db             *sql.DB
	badWords       = make(map[string]*regexp.Regexp)
	titleStart     = []byte("<title>")
	titleEnd       = []byte("</title>")
)

const FREENODE = "irc.freenode.net"

type Config struct {
	Channels      []string
	DBConn        string
	Nick          string
	Ident         string
	FullName      string
	FlickrAPIKey  string
	WolframAPIKey string
	IRCPass       string
	RebuildWords  bool
	Commands      []struct {
		Name string
		Text string
	}
	BadWords []struct {
		Word  string
		Query string
	}
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn) {
	if !httpRegex.MatchString(unparsedURL) {
		unparsedURL = `http://` + unparsedURL
	}
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	utf8Body, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
	if err != nil {
		log.Println("error converting page to utf8")
		return
	}
	restofbody, err := ioutil.ReadAll(io.LimitReader(utf8Body, 50000))
	if err != nil {
		log.Println("error reading posted link")
		return
	}
	respbody = append(respbody, restofbody...)
	titlestart := bytes.Index(respbody, titleStart)
	titleend := bytes.Index(respbody, titleEnd)
	if titlestart == -1 || titleend == -1 {
		return
	}
	title := respbody[titlestart+7 : titleend]
	title = bytes.TrimSpace(title)
	if len(title) == 0 || !utf8.Valid(title) {
		return
	}
	// Example:
	// Title: sadbox . org (at sadbox.org)
	formattedTitle := html.UnescapeString(string(title))
	formattedTitle = findWhiteSpace.ReplaceAllString(formattedTitle, " ")
	formattedTitle = fmt.Sprintf("Title: %s (at %s)", formattedTitle, postedUrl.Host)
	log.Println(formattedTitle)
	conn.Privmsg(channel, formattedTitle)
}

func logMessage(line *irc.Line, channel, message string) {
	_, err := db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel,"+
		" Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident,
		line.Host, line.Src, line.Cmd, channel, message, line.Time)
	if err != nil {
		log.Println(err)
	}
	err = updateWords(line.Nick, message)
	if err != nil {
		log.Println(err)
	}
}

func checkForUrl(channel string, splitmessage []string, conn *irc.Conn) {
	urllist := []string{}
	numlinks := 0
NextWord:
	for _, word := range splitmessage {
		word = strings.TrimSpace(word)
		if urlRegex.MatchString(word) {
			for _, subUrl := range urllist {
				if subUrl == word {
					continue NextWord
				}
			}
			numlinks++
			if numlinks > 3 {
				break
			}
			urllist = append(urllist, word)
			go sendUrl(channel, word, conn)
		}
	}
}

// This function does all the dispatching for various commands
// as well as logging each message to the database
func handleMessage(conn *irc.Conn, line *irc.Line) {
	// This is so that the bot can properly respond to pm's
	channel := line.Target()
	message := line.Args[1]
	splitmessage := strings.Split(message, " ")
	cmd := strings.TrimSpace(splitmessage[0])

	// sadbox-only commands!
	if line.Nick == "sadbox" {
		switch cmd {
		case "!dance":
			go dance(channel, conn)
		case "!audio":
			go conn.Privmsg(channel, "https://sadbox.org/static/stuff/audiophile.html")
		case "!cst":
			go conn.Privmsg(channel, "\u00039,13#CSTMASTERRACE")
		case "!chatter":
			go markov(channel, conn)
		}
	}

	// Special commands
	switch cmd {
	case "!haata":
		go haata(channel, conn)
	case "!search":
		go googSearch(channel, message, conn)
	case "!ask":
		go wolfram(channel, message, line.Nick, conn)
	case "!meebcast":
		var command string
		if len(splitmessage) >= 2 {
			command = strings.TrimSpace(splitmessage[1])
		}
		go meeba(channel, line.Nick, command, conn)
	}

	// Commands that are read in from the config file
	for _, command := range config.Commands {
		if cmd == command.Name {
			var response string
			if len(splitmessage) >= 2 {
				response = fmt.Sprintf("%s: %s", splitmessage[1], command.Text)
			} else {
				response = command.Text
			}
			go conn.Privmsg(channel, response)
		}
	}

	// This is what looks at each word and tries to figure out if it's a URL
	go checkForUrl(channel, splitmessage, conn)

	// Shove that shit in the database!
	go logMessage(line, channel, message)
}

func init() {
	log.Println("Starting sadbot")

	runtime.GOMAXPROCS(runtime.NumCPU())

	rand.Seed(time.Now().UTC().UnixNano())

	configfile, err := os.Open("config.json")
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(configfile).Decode(&config)
	if err != nil {
		log.Fatal(err)
	}

	for _, word := range config.BadWords {
		badWords[word.Word] = regexp.MustCompile(word.Query)
	}

	log.Println("Loaded config file!")
	log.Printf("Joining: %s", config.Channels)
	log.Printf("Nick: %s", config.Nick)
	log.Printf("Ident: %s", config.Ident)
	log.Printf("FullName: %s", config.FullName)

	log.Printf("Found %d commands", len(config.Commands))
	for index, command := range config.Commands {
		log.Printf("%d %s: %s", index+1, command.Name, command.Text)
	}
}

func main() {
	var err error
	db, err = sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(200)

	go makeMarkov()

	buildchan := make(chan os.Signal, 1)
	signal.Notify(buildchan, syscall.SIGUSR1)
	go func() {
		for _ = range buildchan {
			genTables()
		}
	}()

	ircConfig := irc.NewConfig(config.Nick, config.Ident, config.FullName)
	ircConfig.SSL = true
	ircConfig.Server = FREENODE
	ircConfig.Pass = config.Nick + ":" + config.IRCPass

	c := irc.Client(ircConfig)

	c.HandleFunc(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			for _, channel := range config.Channels {
				log.Printf("Joining %s", channel)
				conn.Join(channel)
			}
			log.Println("Connected!")
		})
	quit := make(chan bool)

	c.HandleFunc(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	c.HandleFunc(irc.PRIVMSG, handleMessage)
	c.HandleFunc(irc.ACTION, handleMessage)

	if err := c.Connect(); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
}
<commit_msg>updated url regex to use external library, saner deduping, etc<commit_after>// Copyright 2014 James McGuire. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"

	irc "github.com/fluffle/goirc/client"
	_ "github.com/go-sql-driver/mysql"
	"github.com/mvdan/xurls"
	"golang.org/x/net/html/charset"
)

var (
	config   Config
	urlRegex = regexp.MustCompile(`(?i)\b((?:https?://|www\d{0,3}[.]|[` +
		`a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+` +
		`\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[` +
		`\]{};:'".,<>?«»“”‘’]))`)
	httpRegex      = regexp.MustCompile(`https?://.*`)
	findWhiteSpace = regexp.MustCompile(`\s+`)
	db             *sql.DB
	badWords       = make(map[string]*regexp.Regexp)
	titleStart     = []byte("<title>")
	titleEnd       = []byte("</title>")
)

const FREENODE = "irc.freenode.net"

type Config struct {
	Channels      []string
	DBConn        string
	Nick          string
	Ident         string
	FullName      string
	FlickrAPIKey  string
	WolframAPIKey string
	IRCPass       string
	RebuildWords  bool
	Commands      []struct {
		Name string
		Text string
	}
	BadWords []struct {
		Word  string
		Query string
	}
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn, nick string) {
	if !httpRegex.MatchString(unparsedURL) {
		unparsedURL = `http://` + unparsedURL
	}
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	utf8Body, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
	if err != nil {
		log.Println("error converting page to utf8")
		return
	}
	restofbody, err := ioutil.ReadAll(io.LimitReader(utf8Body, 50000))
	if err != nil {
		log.Println("error reading posted link")
		return
	}
	respbody = append(respbody, restofbody...)
	titlestart := bytes.Index(respbody, titleStart)
	titleend := bytes.Index(respbody, titleEnd)
	if titlestart == -1 || titleend == -1 {
		return
	}
	title := respbody[titlestart+7 : titleend]
	title = bytes.TrimSpace(title)
	if len(title) == 0 || !utf8.Valid(title) {
		return
	}
	// Example:
	// Title: sadbox . org (at sadbox.org)
	formattedTitle := html.UnescapeString(string(title))
	formattedTitle = findWhiteSpace.ReplaceAllString(formattedTitle, " ")
	formattedTitle = fmt.Sprintf("(%s) %s (at %s)", nick, formattedTitle, postedUrl.Host)
	log.Println(formattedTitle)
	conn.Privmsg(channel, formattedTitle)
}

func logMessage(line *irc.Line, channel, message string) {
	_, err := db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel,"+
		" Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident,
		line.Host, line.Src, line.Cmd, channel, message, line.Time)
	if err != nil {
		log.Println(err)
	}
	err = updateWords(line.Nick, message)
	if err != nil {
		log.Println(err)
	}
}

func checkForUrl(channel, message, nick string, conn *irc.Conn) {
	urllist := make(map[string]struct{})
	for _, item := range xurls.Relaxed.FindAllString(message, -1) {
		urllist[item] = struct{}{}
	}
	numlinks := 0
	for item, _ := range urllist {
		numlinks++
		if numlinks > 3 {
			break
		}
		go sendUrl(channel, item, conn, nick)
	}
}

// This function does all the dispatching for various commands
// as well as logging each message to the database
func handleMessage(conn *irc.Conn, line *irc.Line) {
	// This is so that the bot can properly respond to pm's
	channel := line.Target()
	message := line.Args[1]
	splitmessage := strings.Split(message, " ")
	cmd := strings.TrimSpace(splitmessage[0])

	// sadbox-only commands!
	if line.Nick == "sadbox" {
		switch cmd {
		case "!dance":
			go dance(channel, conn)
		case "!audio":
			go conn.Privmsg(channel, "https://sadbox.org/static/stuff/audiophile.html")
		case "!cst":
			go conn.Privmsg(channel, "\u00039,13#CSTMASTERRACE")
		case "!chatter":
			go markov(channel, conn)
		}
	}

	// Special commands
	switch cmd {
	case "!haata":
		go haata(channel, conn)
	case "!search":
		go googSearch(channel, message, conn)
	case "!ask":
		go wolfram(channel, message, line.Nick, conn)
	case "!meebcast":
		var command string
		if len(splitmessage) >= 2 {
			command = strings.TrimSpace(splitmessage[1])
		}
		go meeba(channel, line.Nick, command, conn)
	}

	// Commands that are read in from the config file
	for _, command := range config.Commands {
		if cmd == command.Name {
			var response string
			if len(splitmessage) >= 2 {
				response = fmt.Sprintf("%s: %s", splitmessage[1], command.Text)
			} else {
				response = command.Text
			}
			go conn.Privmsg(channel, response)
		}
	}

	// This is what looks at each word and tries to figure out if it's a URL
	go checkForUrl(channel, message, line.Nick, conn)

	// Shove that shit in the database!
	go logMessage(line, channel, message)
}

func init() {
	log.Println("Starting sadbot")

	runtime.GOMAXPROCS(runtime.NumCPU())

	rand.Seed(time.Now().UTC().UnixNano())

	configfile, err := os.Open("config.json")
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(configfile).Decode(&config)
	if err != nil {
		log.Fatal(err)
	}

	for _, word := range config.BadWords {
		badWords[word.Word] = regexp.MustCompile(word.Query)
	}

	log.Println("Loaded config file!")
	log.Printf("Joining: %s", config.Channels)
	log.Printf("Nick: %s", config.Nick)
	log.Printf("Ident: %s", config.Ident)
	log.Printf("FullName: %s", config.FullName)

	log.Printf("Found %d commands", len(config.Commands))
	for index, command := range config.Commands {
		log.Printf("%d %s: %s", index+1, command.Name, command.Text)
	}
}

func main() {
	var err error
	db, err = sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(200)

	go makeMarkov()

	buildchan := make(chan os.Signal, 1)
	signal.Notify(buildchan, syscall.SIGUSR1)
	go func() {
		for _ = range buildchan {
			genTables()
		}
	}()

	ircConfig := irc.NewConfig(config.Nick, config.Ident, config.FullName)
	ircConfig.SSL = true
	ircConfig.Server = FREENODE
	ircConfig.Pass = config.Nick + ":" + config.IRCPass

	c := irc.Client(ircConfig)

	c.HandleFunc(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			for _, channel := range config.Channels {
				log.Printf("Joining %s", channel)
				conn.Join(channel)
			}
			log.Println("Connected!")
		})
	quit := make(chan bool)

	c.HandleFunc(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	c.HandleFunc(irc.PRIVMSG, handleMessage)
	c.HandleFunc(irc.ACTION, handleMessage)

	if err := c.Connect(); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
}
<|endoftext|>