<commit_before>package command

import (
	"flag"
	"fmt"
	"strconv"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
)

type Clusters struct {
	Ui  cli.Ui
	Cmd string

	verbose           bool
	registeredBrokers bool
	publicOnly        bool
}

// TODO cluster info will contain desciption,owner,etc.
func (this *Clusters) Run(args []string) (exitCode int) {
	var (
		addCluster     string
		setMode        bool
		verifyMode     bool
		public         int
		clusterName    string
		clusterPath    string
		zone           string
		priority       int
		retentionHours int
		delCluster     string
		replicas       int
		addBroker      string
		nickname       string
		delBroker      int
	)
	cmdFlags := flag.NewFlagSet("clusters", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	cmdFlags.StringVar(&addCluster, "add", "", "")
	cmdFlags.BoolVar(&this.publicOnly, "po", false, "")
	cmdFlags.BoolVar(&setMode, "s", false, "")
	cmdFlags.StringVar(&clusterName, "c", "", "")
	cmdFlags.StringVar(&clusterPath, "p", "", "")
	cmdFlags.BoolVar(&this.verbose, "l", false, "")
	cmdFlags.IntVar(&replicas, "replicas", -1, "")
	cmdFlags.StringVar(&delCluster, "del", "", "")
	cmdFlags.IntVar(&retentionHours, "retention", -1, "")
	cmdFlags.IntVar(&priority, "priority", -1, "")
	cmdFlags.IntVar(&public, "public", -1, "")
	cmdFlags.StringVar(&addBroker, "addbroker", "", "")
	cmdFlags.StringVar(&nickname, "nickname", "", "")
	cmdFlags.IntVar(&delBroker, "delbroker", -1, "")
	cmdFlags.BoolVar(&this.registeredBrokers, "registered", false, "")
	cmdFlags.BoolVar(&verifyMode, "verify", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		on("-add", "-z", "-p").
		on("-s", "-z", "-c").
		requireAdminRights("-s", "-add").
		invalid(args) {
		return 2
	}

	if this.publicOnly {
		this.verbose = true
	}

	if delCluster != "" {
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		zkcluster := zkzone.NewCluster(delCluster)

		kfkCluseterPath := zkzone.ClusterPath(delCluster)
		gafkaClusterInfoPath := zkcluster.ClusterInfoPath()
		gafkaClusterPath := zk.ClusterPath(delCluster)

		this.Ui.Info("shutdown brokers first: kill java/rm pkg/chkconfig --del")
		this.Ui.Info(fmt.Sprintf("zk rm -z %s -R -p %s",
			zone, kfkCluseterPath))
		this.Ui.Info(fmt.Sprintf("zk rm -z %s -p %s",
			zone, gafkaClusterPath))
		this.Ui.Info(fmt.Sprintf("zk rm -z %s -p %s",
			zone, gafkaClusterInfoPath))

		return
	}

	if addCluster != "" {
		// add cluster
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		defer printSwallowedErrors(this.Ui, zkzone)

		if err := zkzone.RegisterCluster(addCluster, clusterPath); err != nil {
			this.Ui.Error(err.Error())
			return 1
		}

		this.Ui.Info(fmt.Sprintf("%s: %s created", addCluster, clusterPath))

		return
	}

	if setMode {
		// setup a cluser meta info
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		zkcluster := zkzone.NewCluster(clusterName)
		if priority != -1 {
			zkcluster.SetPriority(priority)
		}
		if public != -1 {
			switch public {
			case 0:
				zkcluster.SetPublic(false)

			case 1:
				zkcluster.SetPublic(true)
			}
		}
		if retentionHours != -1 {
			zkcluster.SetRetention(retentionHours)
		}
		if replicas != -1 {
			zkcluster.SetReplicas(replicas)
		}
		if nickname != "" {
			zkcluster.SetNickname(nickname)
		}

		switch {
		case addBroker != "":
			parts := strings.Split(addBroker, ":")
			if len(parts) != 3 {
				this.Ui.Output(this.Help())
				return
			}

			brokerId, err := strconv.Atoi(parts[0])
			swallow(err)
			port, err := strconv.Atoi(parts[2])
			swallow(err)
			err = zkcluster.RegisterBroker(brokerId, parts[1], port)
			swallow(err)

		case delBroker != -1:
			this.Ui.Error("not implemented yet")

		default:
			return
		}

	}

	if verifyMode {
		if zone != "" {
			zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
			this.verifyBrokers(zkzone)

			printSwallowedErrors(this.Ui, zkzone)
		} else {
			// print all zones all clusters
			forSortedZones(func(zkzone *zk.ZkZone) {
				this.verifyBrokers(zkzone)

				printSwallowedErrors(this.Ui, zkzone)
			})
		}
		return
	}

	// display mode
	if zone != "" {
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		this.printClusters(zkzone)

		printSwallowedErrors(this.Ui, zkzone)
	} else {
		// print all zones all clusters
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.printClusters(zkzone)

			printSwallowedErrors(this.Ui, zkzone)
		})
	}

	return
}

func (this *Clusters) printRegisteredBrokers(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		info := zkcluster.RegisteredInfo()
		this.Ui.Output(fmt.Sprintf("    %s(%s)", info.Name(), info.Nickname))
		registeredBrokers := info.Roster
		if len(registeredBrokers) == 0 {
			this.Ui.Warn("        brokers not defined")
		} else {
			for _, b := range registeredBrokers {
				this.Ui.Output(fmt.Sprintf("        %2d %s", b.Id, b.Addr()))
			}
		}

	})
}

func (this *Clusters) verifyBrokers(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		zkcluster := zkzone.NewCluster(cluster)
		registeredBrokers := zkcluster.RegisteredInfo().Roster

		// find diff between registeredBrokers and liveBrokers
		// loop1 find liveBrokers>registeredBrokers
		for _, broker := range liveBrokers {
			foundInRoster := false
			for _, b := range registeredBrokers {
				bid := strconv.Itoa(b.Id)
				if bid == broker.Id && broker.Addr() == b.Addr() {
					foundInRoster = true
					break
				}
			}

			if !foundInRoster {
				// should manually register the broker
				this.Ui.Output(strings.Repeat(" ", 4) +
					color.Green("+ gk clusters -z %s -s -c %s -addbroker %s:%s",
						zkzone.Name(), cluster, broker.Id, broker.Addr()))
			}
		}

		// loop2 find liveBrokers<registeredBrokers
		for _, b := range registeredBrokers {
			foundInLive := false
			for _, broker := range liveBrokers {
				bid := strconv.Itoa(b.Id)
				if bid == broker.Id && broker.Addr() == b.Addr() {
					foundInLive = true
					break
				}
			}

			if !foundInLive {
				// the broker is dead
				this.Ui.Output(strings.Repeat(" ", 4) +
					color.Red("broker %d %s is dead", b.Id, b.Addr()))
			}
		}
	})
}

func (this *Clusters) printClusters(zkzone *zk.ZkZone) {
	if this.registeredBrokers {
		this.printRegisteredBrokers(zkzone)
		return
	}

	type clusterInfo struct {
		name, path         string
		nickname           string
		topicN, partitionN int
		err                string
		priority           int
		public             bool
		retention          int
		replicas           int
		brokerInfos        []zk.BrokerInfo
	}
	clusters := make([]clusterInfo, 0)
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		ci := clusterInfo{
			name: zkcluster.Name(),
			path: zkcluster.Chroot(),
		}
		if !this.verbose {
			clusters = append(clusters, ci)
			return
		}

		// verbose mode, will calculate topics and partition count
		brokerList := zkcluster.BrokerList()
		if len(brokerList) == 0 {
			ci.err = "no live brokers"
			clusters = append(clusters, ci)
			return
		}

		kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
		if err != nil {
			ci.err = err.Error()
			clusters = append(clusters, ci)
			return
		}
		topics, err := kfk.Topics()
		if err != nil {
			ci.err = err.Error()
			clusters = append(clusters, ci)
			return
		}
		partitionN := 0
		for _, topic := range topics {
			partitions, err := kfk.Partitions(topic)
			if err != nil {
				ci.err = err.Error()
				clusters = append(clusters, ci)
				continue
			}

			partitionN += len(partitions)
		}

		info := zkcluster.RegisteredInfo()
		if this.publicOnly && !info.Public {
			return
		}
		clusters = append(clusters, clusterInfo{
			name:        zkcluster.Name(),
			nickname:    info.Nickname,
			path:        zkcluster.Chroot(),
			topicN:      len(topics),
			partitionN:  partitionN,
			retention:   info.Retention,
			public:      info.Public,
			replicas:    info.Replicas,
			priority:    info.Priority,
			brokerInfos: info.Roster,
		})
	})

	this.Ui.Output(fmt.Sprintf("%s: %d", zkzone.Name(), len(clusters)))
	if this.verbose {
		// 2 loop: 1. print the err clusters 2. print the good clusters
		for _, c := range clusters {
			if c.err == "" {
				continue
			}

			this.Ui.Output(fmt.Sprintf("%30s: %s %s", c.name, c.path,
				color.Red(c.err)))
		}

		// loop2
		for _, c := range clusters {
			if c.err != "" {
				continue
			}

			this.Ui.Output(fmt.Sprintf("%30s: %s",
				c.name, c.path))
			this.Ui.Output(strings.Repeat(" ", 4) +
				color.Blue("nick:%s public:%v topics:%d partitions:%d replicas:%d retention:%dh brokers:%+v",
					c.nickname, c.public,
					c.topicN, c.partitionN, c.replicas, c.retention, c.brokerInfos))
		}

		return
	}

	// not verbose mode
	for _, c := range clusters {
		this.Ui.Output(fmt.Sprintf("%30s: %s", c.name, c.path))
	}

}

func (*Clusters) Synopsis() string {
	return "Register or display kafka clusters"
}

func (this *Clusters) Help() string {
	help := fmt.Sprintf(`
Usage: %s clusters [options]

    Register or display kafka clusters

Options:

    -z zone
      Only print kafka clusters within this zone.

    -c cluster name
      The new kafka cluster name.

    -l
      Use a long listing format.

    -po
      Display only public clusters.

    -add cluster name
      Add a new kafka cluster into a zone.

    -del cluster name
      Help to delete a cluster.

    -p cluster zk path
      The new kafka cluster chroot path in Zookeeper.
      e,g. gk clusters -z prod -add foo -p /kafka/services/trade

    -s
      Setup a cluster info.

    -priority n
      Set the priority of a cluster.

    -public <0|1>
      Export the cluster for PubSub system or not.
      e,g. gk cluster -z prod -c foo -s -public 1

    -retention n hours
      log.retention.hours of kafka.

    -nickname name
      Set nickname of a cluster.
      e,g. gk clusters -z prod -c foo -s -nickname bar

    -replicas n
      Set the default replicas of a cluster.

    -addbroker id:host:port
      Register a permanent broker to a cluster.

    -delbroker id TODO
      Delete a broker from a cluster.

    -registered
      Display registered permanent brokers info.

    -verify
      Verify that the online brokers are consistent with the registered brokers.

`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>exit after add broker into cluster<commit_after>package command

import (
	"flag"
	"fmt"
	"strconv"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
)

type Clusters struct {
	Ui  cli.Ui
	Cmd string

	verbose           bool
	registeredBrokers bool
	publicOnly        bool
}

// TODO cluster info will contain desciption,owner,etc.
func (this *Clusters) Run(args []string) (exitCode int) {
	var (
		addCluster     string
		setMode        bool
		verifyMode     bool
		public         int
		clusterName    string
		clusterPath    string
		zone           string
		priority       int
		retentionHours int
		delCluster     string
		replicas       int
		addBroker      string
		nickname       string
		delBroker      int
	)
	cmdFlags := flag.NewFlagSet("clusters", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	cmdFlags.StringVar(&addCluster, "add", "", "")
	cmdFlags.BoolVar(&this.publicOnly, "po", false, "")
	cmdFlags.BoolVar(&setMode, "s", false, "")
	cmdFlags.StringVar(&clusterName, "c", "", "")
	cmdFlags.StringVar(&clusterPath, "p", "", "")
	cmdFlags.BoolVar(&this.verbose, "l", false, "")
	cmdFlags.IntVar(&replicas, "replicas", -1, "")
	cmdFlags.StringVar(&delCluster, "del", "", "")
	cmdFlags.IntVar(&retentionHours, "retention", -1, "")
	cmdFlags.IntVar(&priority, "priority", -1, "")
	cmdFlags.IntVar(&public, "public", -1, "")
	cmdFlags.StringVar(&addBroker, "addbroker", "", "")
	cmdFlags.StringVar(&nickname, "nickname", "", "")
	cmdFlags.IntVar(&delBroker, "delbroker", -1, "")
	cmdFlags.BoolVar(&this.registeredBrokers, "registered", false, "")
	cmdFlags.BoolVar(&verifyMode, "verify", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		on("-add", "-z", "-p").
		on("-s", "-z", "-c").
		requireAdminRights("-s", "-add").
		invalid(args) {
		return 2
	}

	if this.publicOnly {
		this.verbose = true
	}

	if delCluster != "" {
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		zkcluster := zkzone.NewCluster(delCluster)

		kfkCluseterPath := zkzone.ClusterPath(delCluster)
		gafkaClusterInfoPath := zkcluster.ClusterInfoPath()
		gafkaClusterPath := zk.ClusterPath(delCluster)

		this.Ui.Info("shutdown brokers first: kill java/rm pkg/chkconfig --del")
		this.Ui.Info(fmt.Sprintf("zk rm -z %s -R -p %s",
			zone, kfkCluseterPath))
		this.Ui.Info(fmt.Sprintf("zk rm -z %s -p %s",
			zone, gafkaClusterPath))
		this.Ui.Info(fmt.Sprintf("zk rm -z %s -p %s",
			zone, gafkaClusterInfoPath))

		return
	}

	if addCluster != "" {
		// add cluster
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		defer printSwallowedErrors(this.Ui, zkzone)

		if err := zkzone.RegisterCluster(addCluster, clusterPath); err != nil {
			this.Ui.Error(err.Error())
			return 1
		}

		this.Ui.Info(fmt.Sprintf("%s: %s created", addCluster, clusterPath))

		return
	}

	if setMode {
		// setup a cluser meta info
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		zkcluster := zkzone.NewCluster(clusterName)
		if priority != -1 {
			zkcluster.SetPriority(priority)
		}
		if public != -1 {
			switch public {
			case 0:
				zkcluster.SetPublic(false)

			case 1:
				zkcluster.SetPublic(true)
			}
		}
		if retentionHours != -1 {
			zkcluster.SetRetention(retentionHours)
		}
		if replicas != -1 {
			zkcluster.SetReplicas(replicas)
		}
		if nickname != "" {
			zkcluster.SetNickname(nickname)
		}

		switch {
		case addBroker != "":
			parts := strings.Split(addBroker, ":")
			if len(parts) != 3 {
				this.Ui.Output(this.Help())
				return
			}

			brokerId, err := strconv.Atoi(parts[0])
			swallow(err)
			port, err := strconv.Atoi(parts[2])
			swallow(err)
			err = zkcluster.RegisterBroker(brokerId, parts[1], port)
			swallow(err)
			return

		case delBroker != -1:
			this.Ui.Error("not implemented yet")

		default:
			return
		}

	}

	if verifyMode {
		if zone != "" {
			zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
			this.verifyBrokers(zkzone)

			printSwallowedErrors(this.Ui, zkzone)
		} else {
			// print all zones all clusters
			forSortedZones(func(zkzone *zk.ZkZone) {
				this.verifyBrokers(zkzone)

				printSwallowedErrors(this.Ui, zkzone)
			})
		}

		return
	}

	// display mode
	if zone != "" {
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		this.printClusters(zkzone)

		printSwallowedErrors(this.Ui, zkzone)
	} else {
		// print all zones all clusters
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.printClusters(zkzone)

			printSwallowedErrors(this.Ui, zkzone)
		})
	}

	return
}

func (this *Clusters) printRegisteredBrokers(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		info := zkcluster.RegisteredInfo()
		this.Ui.Output(fmt.Sprintf("    %s(%s)", info.Name(), info.Nickname))
		registeredBrokers := info.Roster
		if len(registeredBrokers) == 0 {
			this.Ui.Warn("        brokers not defined")
		} else {
			for _, b := range registeredBrokers {
				this.Ui.Output(fmt.Sprintf("        %2d %s", b.Id, b.Addr()))
			}
		}

	})
}

func (this *Clusters) verifyBrokers(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		zkcluster := zkzone.NewCluster(cluster)
		registeredBrokers := zkcluster.RegisteredInfo().Roster

		// find diff between registeredBrokers and liveBrokers
		// loop1 find liveBrokers>registeredBrokers
		for _, broker := range liveBrokers {
			foundInRoster := false
			for _, b := range registeredBrokers {
				bid := strconv.Itoa(b.Id)
				if bid == broker.Id && broker.Addr() == b.Addr() {
					foundInRoster = true
					break
				}
			}

			if !foundInRoster {
				// should manually register the broker
				this.Ui.Output(strings.Repeat(" ", 4) +
					color.Green("+ gk clusters -z %s -s -c %s -addbroker %s:%s",
						zkzone.Name(), cluster, broker.Id, broker.Addr()))
			}
		}

		// loop2 find liveBrokers<registeredBrokers
		for _, b := range registeredBrokers {
			foundInLive := false
			for _, broker := range liveBrokers {
				bid := strconv.Itoa(b.Id)
				if bid == broker.Id && broker.Addr() == b.Addr() {
					foundInLive = true
					break
				}
			}

			if !foundInLive {
				// the broker is dead
				this.Ui.Output(strings.Repeat(" ", 4) +
					color.Red("broker %d %s is dead", b.Id, b.Addr()))
			}
		}
	})
}

func (this *Clusters) printClusters(zkzone *zk.ZkZone) {
	if this.registeredBrokers {
		this.printRegisteredBrokers(zkzone)
		return
	}

	type clusterInfo struct {
		name, path         string
		nickname           string
		topicN, partitionN int
		err                string
		priority           int
		public             bool
		retention          int
		replicas           int
		brokerInfos        []zk.BrokerInfo
	}
	clusters := make([]clusterInfo, 0)
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		ci := clusterInfo{
			name: zkcluster.Name(),
			path: zkcluster.Chroot(),
		}
		if !this.verbose {
			clusters = append(clusters, ci)
			return
		}

		// verbose mode, will calculate topics and partition count
		brokerList := zkcluster.BrokerList()
		if len(brokerList) == 0 {
			ci.err = "no live brokers"
			clusters = append(clusters, ci)
			return
		}

		kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
		if err != nil {
			ci.err = err.Error()
			clusters = append(clusters, ci)
			return
		}
		topics, err := kfk.Topics()
		if err != nil {
			ci.err = err.Error()
			clusters = append(clusters, ci)
			return
		}
		partitionN := 0
		for _, topic := range topics {
			partitions, err := kfk.Partitions(topic)
			if err != nil {
				ci.err = err.Error()
				clusters = append(clusters, ci)
				continue
			}

			partitionN += len(partitions)
		}

		info := zkcluster.RegisteredInfo()
		if this.publicOnly && !info.Public {
			return
		}
		clusters = append(clusters, clusterInfo{
			name:        zkcluster.Name(),
			nickname:    info.Nickname,
			path:        zkcluster.Chroot(),
			topicN:      len(topics),
			partitionN:  partitionN,
			retention:   info.Retention,
			public:      info.Public,
			replicas:    info.Replicas,
			priority:    info.Priority,
			brokerInfos: info.Roster,
		})
	})

	this.Ui.Output(fmt.Sprintf("%s: %d", zkzone.Name(), len(clusters)))
	if this.verbose {
		// 2 loop: 1. print the err clusters 2. print the good clusters
		for _, c := range clusters {
			if c.err == "" {
				continue
			}

			this.Ui.Output(fmt.Sprintf("%30s: %s %s", c.name, c.path,
				color.Red(c.err)))
		}

		// loop2
		for _, c := range clusters {
			if c.err != "" {
				continue
			}

			this.Ui.Output(fmt.Sprintf("%30s: %s",
				c.name, c.path))
			this.Ui.Output(strings.Repeat(" ", 4) +
				color.Blue("nick:%s public:%v topics:%d partitions:%d replicas:%d retention:%dh brokers:%+v",
					c.nickname, c.public,
					c.topicN, c.partitionN, c.replicas, c.retention, c.brokerInfos))
		}

		return
	}

	// not verbose mode
	for _, c := range clusters {
		this.Ui.Output(fmt.Sprintf("%30s: %s", c.name, c.path))
	}

}

func (*Clusters) Synopsis() string {
	return "Register or display kafka clusters"
}

func (this *Clusters) Help() string {
	help := fmt.Sprintf(`
Usage: %s clusters [options]

    Register or display kafka clusters

Options:

    -z zone
      Only print kafka clusters within this zone.

    -c cluster name
      The new kafka cluster name.

    -l
      Use a long listing format.

    -po
      Display only public clusters.

    -add cluster name
      Add a new kafka cluster into a zone.

    -del cluster name
      Help to delete a cluster.

    -p cluster zk path
      The new kafka cluster chroot path in Zookeeper.
      e,g. gk clusters -z prod -add foo -p /kafka/services/trade

    -s
      Setup a cluster info.

    -priority n
      Set the priority of a cluster.

    -public <0|1>
      Export the cluster for PubSub system or not.
      e,g. gk cluster -z prod -c foo -s -public 1

    -retention n hours
      log.retention.hours of kafka.

    -nickname name
      Set nickname of a cluster.
      e,g. gk clusters -z prod -c foo -s -nickname bar

    -replicas n
      Set the default replicas of a cluster.

    -addbroker id:host:port
      Register a permanent broker to a cluster.

    -delbroker id TODO
      Delete a broker from a cluster.

    -registered
      Display registered permanent brokers info.

    -verify
      Verify that the online brokers are consistent with the registered brokers.

`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>//go:generate go build

/*
Goexports generates wrappers of package exported symbols

Output files are written in the current directory, and prefixed with the go version.

Usage:

    goexports package...

Example:

    goexports github.com/containous/yaegi/interp

The same goexport program is used for all target operating systems and architectures.
The GOOS and GOARCH environment variables set the desired target.
*/
package main

import (
	"bytes"
	"fmt"
	"go/constant"
	"go/format"
	"go/importer"
	"go/types"
	"io/ioutil"
	"log"
	"os"
	"path"
	"runtime"
	"strconv"
	"strings"
	"text/template"
)

const model = `// +build {{.BuildTags}}

package {{.Dest}}

// Code generated by 'goexports {{.PkgName}}'. DO NOT EDIT.

import (
{{- range $key, $value := .Imports }}
	{{- if $value}}
	"{{$key}}"
	{{- end}}
{{- end}}
	"{{.PkgName}}"
	"reflect"
)

func init() {
	Symbols["{{.PkgName}}"] = map[string]reflect.Value{
		// function, constant and variable definitions
		{{range $key, $value := .Val -}}
			{{- if $value.Addr -}}
				"{{$key}}": reflect.ValueOf(&{{$value.Name}}).Elem(),
			{{else -}}
				"{{$key}}": reflect.ValueOf({{$value.Name}}),
			{{end -}}
		{{end}}

		// type definitions
		{{range $key, $value := .Typ -}}
			"{{$key}}": reflect.ValueOf((*{{$value}})(nil)),
		{{end}}

		// interface wrapper definitions
		{{range $key, $value := .Wrap -}}
			"_{{$key}}": reflect.ValueOf((*{{$value.Name}})(nil)),
		{{end}}
	}
}
{{range $key, $value := .Wrap -}}
	// {{$value.Name}} is an interface wrapper for {{$key}} type
	type {{$value.Name}} struct {
		{{range $m := $value.Method -}}
		W{{$m.Name}} func{{$m.Param}} {{$m.Result}}
		{{end}}
	}
	{{range $m := $value.Method -}}
		func (W {{$value.Name}}) {{$m.Name}}{{$m.Param}} {{$m.Result}} { {{$m.Ret}} W.W{{$m.Name}}{{$m.Arg}} }
	{{end}}
{{end}}
`

// Val store the value name and addressable status of symbols
type Val struct {
	Name string // "package.name"
	Addr bool   // true if symbol is a Var
}

// Method store information for generating interface wrapper method
type Method struct {
	Name, Param, Result, Arg, Ret string
}

// Wrap store information for generating interface wrapper
type Wrap struct {
	Name   string
	Method []Method
}

func genContent(dest, pkgName string) ([]byte, error) {
	p, err := importer.For("source", nil).Import(pkgName)
	if err != nil {
		return nil, err
	}

	prefix := "_" + pkgName + "_"
	prefix = strings.NewReplacer("/", "_", "-", "_", ".", "_").Replace(prefix)

	typ := map[string]string{}
	val := map[string]Val{}
	wrap := map[string]Wrap{}
	imports := map[string]bool{}
	sc := p.Scope()

	for _, pkg := range p.Imports() {
		imports[pkg.Path()] = false
	}
	qualify := func(pkg *types.Package) string {
		if pkg.Path() != pkgName {
			imports[pkg.Path()] = true
		}
		return pkg.Name()
	}

	for _, name := range sc.Names() {
		o := sc.Lookup(name)
		if !o.Exported() {
			continue
		}

		pname := path.Base(pkgName) + "." + name
		switch o := o.(type) {
		case *types.Const:
			val[name] = Val{fixConst(pname, o.Val()), false}
		case *types.Func:
			val[name] = Val{pname, false}
		case *types.Var:
			val[name] = Val{pname, true}
		case *types.TypeName:
			typ[name] = pname
			if t, ok := o.Type().Underlying().(*types.Interface); ok {
				var methods []Method
				for i := 0; i < t.NumMethods(); i++ {
					f := t.Method(i)
					if !f.Exported() {
						continue
					}

					sign := f.Type().(*types.Signature)
					args := make([]string, sign.Params().Len())
					params := make([]string, len(args))
					for j := range args {
						v := sign.Params().At(j)
						if args[j] = v.Name(); args[j] == "" {
							args[j] = fmt.Sprintf("a%d", j)
						}
						params[j] = args[j] + " " + types.TypeString(v.Type(), qualify)
					}
					arg := "(" + strings.Join(args, ", ") + ")"
					param := "(" + strings.Join(params, ", ") + ")"

					results := make([]string, sign.Results().Len())
					for j := range results {
						v := sign.Results().At(j)
						results[j] = v.Name() + " " + types.TypeString(v.Type(), qualify)
					}
					result := "(" + strings.Join(results, ", ") + ")"

					ret := ""
					if sign.Results().Len() > 0 {
						ret = "return"
					}

					methods = append(methods, Method{f.Name(), param, result, arg, ret})
				}
				wrap[name] = Wrap{prefix + name, methods}
			}
		}
	}

	parts := strings.Split(runtime.Version(), ".")
	currentGoVersion := parts[0] + "." + parts[1]

	minor, err := strconv.Atoi(parts[1])
	if err != nil {
		return nil, fmt.Errorf("failed to parse version: %v", err)
	}
	nextGoVersion := parts[0] + "." + strconv.Itoa(minor+1)

	base := template.New("goexports")
	parse, err := base.Parse(model)
	if err != nil {
		return nil, fmt.Errorf("template parsing error: %v", err)
	}

	buildTags := currentGoVersion + ",!" + nextGoVersion
	if pkgName == "log/syslog" {
		buildTags += ",!windows,!nacl,!plan9"
	}

	b := &bytes.Buffer{}
	data := map[string]interface{}{
		"Dest":      dest,
		"Imports":   imports,
		"PkgName":   pkgName,
		"Val":       val,
		"Typ":       typ,
		"Wrap":      wrap,
		"BuildTags": buildTags,
	}
	err = parse.Execute(b, data)
	if err != nil {
		return nil, fmt.Errorf("template error: %v", err)
	}

	// gofmt
	source, err := format.Source(b.Bytes())
	if err != nil {
		return nil, fmt.Errorf("failed to format source: %v: %s", err, b.Bytes())
	}
	return source, nil
}

// fixConst checks untyped constant value, converting it if necessary to avoid overflow
func fixConst(name string, val constant.Value) string {
	if val.Kind() == constant.Int {
		str := val.ExactString()
		i, err := strconv.ParseInt(str, 0, 64)
		if err == nil {
			switch {
			case i == int64(int32(i)):
				return name
			case i == int64(uint32(i)):
				return "uint32(" + name + ")"
			default:
				return "int64(" + name + ")"
			}
		}
		_, err = strconv.ParseUint(str, 0, 64)
		if err == nil {
			return "uint64(" + name + ")"
		}
	}
	return name
}

func main() {
	dir, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	dest := path.Base(dir)

	for _, pkg := range os.Args[1:] {
		content, err := genContent(dest, pkg)
		if err != nil {
			log.Fatal(err)
		}

		var oFile string
		if pkg == "syscall" {
			goos, arch := os.Getenv("GOOS"), os.Getenv("GOARCH")
			oFile = strings.Replace(pkg, "/", "_", -1) + "_" + goos + "_" + arch + ".go"
		} else {
			oFile = strings.Replace(pkg, "/", "_", -1) + ".go"
		}

		parts := strings.Split(runtime.Version(), ".")

		err = ioutil.WriteFile(parts[0]+"_"+parts[1]+"_"+oFile, content, 0666)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>goexports: allow to generate mapping for devel, beta, and rc.<commit_after>//go:generate go build

/*
Goexports generates wrappers of package exported symbols

Output files are written in the current directory, and prefixed with the go version.

Usage:

    goexports package...

Example:

    goexports github.com/containous/yaegi/interp

The same goexport program is used for all target operating systems and architectures.
The GOOS and GOARCH environment variables set the desired target.
*/
package main

import (
	"bytes"
	"fmt"
	"go/constant"
	"go/format"
	"go/importer"
	"go/types"
	"io/ioutil"
	"log"
	"os"
	"path"
	"runtime"
	"strconv"
	"strings"
	"text/template"
)

const model = `{{if .BuildTags}}// +build {{.BuildTags}}{{end}}

package {{.Dest}}

// Code generated by 'goexports {{.PkgName}}'. DO NOT EDIT.

import (
{{- range $key, $value := .Imports }}
	{{- if $value}}
	"{{$key}}"
	{{- end}}
{{- end}}
	"{{.PkgName}}"
	"reflect"
)

func init() {
	Symbols["{{.PkgName}}"] = map[string]reflect.Value{
		// function, constant and variable definitions
		{{range $key, $value := .Val -}}
			{{- if $value.Addr -}}
				"{{$key}}": reflect.ValueOf(&{{$value.Name}}).Elem(),
			{{else -}}
				"{{$key}}": reflect.ValueOf({{$value.Name}}),
			{{end -}}
		{{end}}

		// type definitions
		{{range $key, $value := .Typ -}}
			"{{$key}}": reflect.ValueOf((*{{$value}})(nil)),
		{{end}}

		// interface wrapper definitions
		{{range $key, $value := .Wrap -}}
			"_{{$key}}": reflect.ValueOf((*{{$value.Name}})(nil)),
		{{end}}
	}
}
{{range $key, $value := .Wrap -}}
	// {{$value.Name}} is an interface wrapper for {{$key}} type
	type {{$value.Name}} struct {
		{{range $m := $value.Method -}}
		W{{$m.Name}} func{{$m.Param}} {{$m.Result}}
		{{end}}
	}
	{{range $m := $value.Method -}}
		func (W {{$value.Name}}) {{$m.Name}}{{$m.Param}} {{$m.Result}} { {{$m.Ret}} W.W{{$m.Name}}{{$m.Arg}} }
	{{end}}
{{end}}
`

// Val store the value name and addressable status of symbols
type Val struct {
	Name string // "package.name"
	Addr bool   // true if symbol is a Var
}

// Method store information for generating interface wrapper method
type Method struct {
	Name, Param, Result, Arg, Ret string
}

// Wrap store information for generating interface wrapper
type Wrap struct {
	Name   string
	Method []Method
}

func genContent(dest, pkgName string) ([]byte, error) {
	p, err := importer.For("source", nil).Import(pkgName)
	if err != nil {
		return nil, err
	}

	prefix := "_" + pkgName + "_"
	prefix = strings.NewReplacer("/", "_", "-", "_", ".", "_").Replace(prefix)

	typ := map[string]string{}
	val := map[string]Val{}
	wrap := map[string]Wrap{}
	imports := map[string]bool{}
	sc := p.Scope()

	for _, pkg := range p.Imports() {
		imports[pkg.Path()] = false
	}
	qualify := func(pkg *types.Package) string {
		if pkg.Path() != pkgName {
			imports[pkg.Path()] = true
		}
		return pkg.Name()
	}

	for _, name := range sc.Names() {
		o := sc.Lookup(name)
		if !o.Exported() {
			continue
		}

		pname := path.Base(pkgName) + "." + name
		switch o := o.(type) {
		case *types.Const:
			val[name] = Val{fixConst(pname, o.Val()), false}
		case *types.Func:
			val[name] = Val{pname, false}
		case *types.Var:
			val[name] = Val{pname, true}
		case *types.TypeName:
			typ[name] = pname
			if t, ok := o.Type().Underlying().(*types.Interface); ok {
				var methods []Method
				for i := 0; i < t.NumMethods(); i++ {
					f := t.Method(i)
					if !f.Exported() {
						continue
					}

					sign := f.Type().(*types.Signature)
					args := make([]string, sign.Params().Len())
					params := make([]string, len(args))
					for j := range args {
						v := sign.Params().At(j)
						if args[j] = v.Name(); args[j] == "" {
							args[j] = fmt.Sprintf("a%d", j)
						}
						params[j] = args[j] + " " + types.TypeString(v.Type(), qualify)
					}
					arg := "(" + strings.Join(args, ", ") + ")"
					param := "(" + strings.Join(params, ", ") + ")"

					results := make([]string, sign.Results().Len())
					for j := range results {
						v := sign.Results().At(j)
						results[j] = v.Name() + " " + types.TypeString(v.Type(), qualify)
					}
					result := "(" + strings.Join(results, ", ") + ")"

					ret := ""
					if sign.Results().Len() > 0 {
						ret = "return"
					}

					methods = append(methods, Method{f.Name(), param, result, arg, ret})
				}
				wrap[name] = Wrap{prefix + name, methods}
			}
		}
	}

	var buildTags string
	if runtime.Version() != "devel" {
		parts := strings.Split(runtime.Version(), ".")

		minorRaw := getMinor(parts[1])

		currentGoVersion := parts[0] + "." + minorRaw

		minor, errParse := strconv.Atoi(minorRaw)
		if errParse != nil {
			return nil, fmt.Errorf("failed to parse version: %v", errParse)
		}

		nextGoVersion := parts[0] + "." + strconv.Itoa(minor+1)

		buildTags = currentGoVersion + ",!" + nextGoVersion
	}

	base := template.New("goexports")
	parse, err := base.Parse(model)
	if err != nil {
		return nil, fmt.Errorf("template parsing error: %v", err)
	}

	if pkgName == "log/syslog" {
		buildTags += ",!windows,!nacl,!plan9"
	}

	b := &bytes.Buffer{}
	data := map[string]interface{}{
		"Dest":      dest,
		"Imports":   imports,
		"PkgName":   pkgName,
		"Val":       val,
		"Typ":       typ,
		"Wrap":      wrap,
		"BuildTags": buildTags,
	}
	err = parse.Execute(b, data)
	if err != nil {
		return nil, fmt.Errorf("template error: %v", err)
	}

	// gofmt
	source, err := format.Source(b.Bytes())
	if err != nil {
		return nil, fmt.Errorf("failed to format source: %v: %s", err, b.Bytes())
	}
	return source, nil
}

// fixConst checks untyped constant value, converting it if necessary to avoid overflow
func fixConst(name string, val constant.Value) string {
	if val.Kind() == constant.Int {
		str := val.ExactString()
		i, err := strconv.ParseInt(str, 0, 64)
		if err == nil {
			switch {
			case i == int64(int32(i)):
				return name
			case i == int64(uint32(i)):
				return "uint32(" + name + ")"
			default:
				return "int64(" + name + ")"
			}
		}
		_, err = strconv.ParseUint(str, 0, 64)
		if err == nil {
			return "uint64(" + name + ")"
		}
	}
	return name
}

func main() {
	dir, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	dest := path.Base(dir)

	for _, pkg := range os.Args[1:] {
		content, err := genContent(dest, pkg)
		if err != nil {
			log.Fatal(err)
		}

		var oFile string
		if pkg == "syscall" {
			goos, arch := os.Getenv("GOOS"), os.Getenv("GOARCH")
			oFile = strings.Replace(pkg, "/", "_", -1) + "_" + goos + "_" + arch + ".go"
		} else {
			oFile = strings.Replace(pkg, "/", "_", -1) + ".go"
		}

		prefix := runtime.Version()
		if runtime.Version() != "devel" {
			parts := strings.Split(runtime.Version(), ".")

			prefix = parts[0] + "_" + getMinor(parts[1])
		}

		err = ioutil.WriteFile(prefix+"_"+oFile, content, 0666)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func getMinor(part string) string {
	minor := part
	index := strings.Index(minor, "beta")
	if index < 0 {
		index = strings.Index(minor, "rc")
	}
	if index > 0 {
		minor = minor[:index]
	}

	return minor
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gin-gonic/gin"
	"net/http"
	"gopkg.in/mgo.v2/bson"
	"github.com/gin-gonic/contrib/sessions"
	"encoding/json"
	"net/url"
	"crypto/rand"
	"encoding/hex"
	"golang.org/x/crypto/bcrypt"
	"regexp"
	"strings"
)

func generateToken1(n int) []byte {
	b := make([]byte, n)
	_, err := rand.Read(b)
	if err != nil {
		println(err.Error())
		return b
	}
	token := make([]byte, n * 2)
	hex.Encode(token, b)
	return token
}

func AdminRender(c *gin.Context) {
	render, _ := TemplateStorage[c.Request.URL.Path]

	// TODO
	db := getMongoDBInstance()
	defer db.Session.Close()
	collection := db.C(ACCOUNTS)
	count, _ := collection.Count()
	c.Set("CountAccount", count)
	collection = db.C(USERS)
	count, _ = collection.Count()
	c.Set("CountUser", count)
	collection = db.C(ADMINS)
	count, _ = collection.Count()
	c.Set("CountAdmin", count)
	collection = db.C(ADMINGROUPS)
	count, _ = collection.Count()
	c.Set("CountAdminGroup", count)
	collection = db.C(CATEGORIES)
	count, _ = collection.Count()
	c.Set("CountCategory", count)
	collection = db.C(STATUS)
	count, _ = collection.Count()
	c.Set("CountStatus", count)

	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}

func AccountVerificationRender1(c *gin.Context) {
	account, _ := getAccount(c)
	user, _ := getUser(c)
	if account.IsVerified == "yes" {
		c.Redirect(http.StatusFound, user.DefaultReturnUrl())
		return
	}
	if len(account.VerificationToken) > 0 {

	} else {
		VerifyURL := generateToken(21)
		hash, err := bcrypt.GenerateFromPassword(VerifyURL, bcrypt.DefaultCost)
		if err != nil {
			println(err.Error())
			return
		}

		db := getMongoDBInstance()
		defer db.Session.Close()
		collection := db.C(ACCOUNTS)
		account.VerificationToken = string(hash)
		collection.UpdateId(account.ID, account)// todo how to update only part?
		verifyURL := "http" +"://"+ c.Request.Host +"/account/verification/" + string(VerifyURL) + "/"
		c.Set("VerifyURL", verifyURL)

		mailConf := MailConfig{}
		mailConf.Data = c.Keys
		mailConf.From = config.SMTP.From.Name + " <" + config.SMTP.From.Address + ">"
		mailConf.To = user.Email
		mailConf.Subject = "Your " + config.ProjectName + " Account"
		mailConf.ReplyTo = user.Email
		mailConf.HtmlPath = "views/account/verification/email-html.html"

		if err := mailConf.SendMail(); err != nil {
			//todo it's not serious
		}

	}

	render, _ := TemplateStorage[c.Request.URL.Path]
	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}

func Verify1 (c *gin.Context) {

	account, _ := getAccount(c)
	user, _ := getUser(c)
	err := bcrypt.CompareHashAndPassword([]byte(account.VerificationToken), []byte(c.Param("token")))
	if err == nil {
		db := getMongoDBInstance()
		defer db.Session.Close()
		collection := db.C(ACCOUNTS)
		account.VerificationToken = ""
		account.IsVerified = "yes"
		collection.UpdateId(account.ID, account)
	}
	c.Redirect(http.StatusFound, user.DefaultReturnUrl())
}

func ResendVerification1 (c *gin.Context) {
	account, _ := getAccount(c)
	user, _ := getUser(c)
	if account.IsVerified == "yes" {
		c.HTML(http.StatusOK, user.DefaultReturnUrl(), c.Keys)
		return
	}
	response := Response{} // todo sync.Pool
	response.Errors = []string{}
	response.ErrFor = make(map[string]string)
	defer response.Recover(c)

	var body struct {
		Username    string  `json:"username"`
		Email   string  `json:"email"`
		Password string  `json:"password"`
	}
	decoder := json.NewDecoder(c.Request.Body)
	err := decoder.Decode(&body)

	email := strings.ToLower(body.Email)
	if len(email) == 0 {
		response.ErrFor["email"] = "required"
	} else {
		r, err := regexp.MatchString(`^[a-zA-Z0-9\-\_\.\+]+@[a-zA-Z0-9\-\_\.]+\.[a-zA-Z0-9\-\_]+$`, email)
		if err != nil {
			println(err.Error())
		}
		if !r {
			response.ErrFor["email"] = `invalid email format`
		}
	}
	if response.HasErrors() {
		response.Fail(c)
		return
	}
	db := getMongoDBInstance()
	defer db.Session.Close()
	collection := db.C(USERS)
	{
		user_ := User{}
		collection.Find(bson.M{"email": email, "_id": bson.M{"$ne": user.ID}}).One(&user_)
		if len(user_.Username) > 0 {
			response.ErrFor["email"] = `email already taken`
		}
	}
	if response.HasErrors() {
		response.Fail(c)
		return
	}
	user.Email = email
	collection.UpdateId(user.ID, user)

	collection = db.C(ACCOUNTS)
	VerifyURL := generateToken(21)
	hash, err := bcrypt.GenerateFromPassword(VerifyURL, bcrypt.DefaultCost)
	if err != nil {
		println(err.Error())
		return
	}
	account.VerificationToken = string(hash)
	collection.UpdateId(account.ID, account)// todo how to update only part?
	verifyURL := "http" +"://"+ c.Request.Host +"/account/verification/" + string(VerifyURL) + "/"
	c.Set("VerifyURL", verifyURL)
	mailConf := MailConfig{}
	mailConf.Data = c.Keys
	mailConf.From = config.SMTP.From.Name + " <" + config.SMTP.From.Address + ">"
	mailConf.To = user.Email
	mailConf.Subject = "Your " + config.ProjectName + " Account"
	mailConf.ReplyTo = user.Email
	mailConf.HtmlPath = "views/account/verification/email-html.html"

	if err := mailConf.SendMail(); err != nil {
		//todo it's not serious
	}

	response.Success = true
	c.JSON(http.StatusOK, response)
}

func AccountSettingsRender1(c *gin.Context) {
	sess := sessions.Default(c)

	public := sess.Get("public")
	db := getMongoDBInstance()
	defer db.Session.Close()
	collection := db.C(USERS)
	us := User{} // todo pool
	err := collection.FindId(bson.ObjectIdHex(public.(string))).One(&us)
	if err != nil {
		println(err.Error())
	}
	if len(us.Username) != 0 {
		User, _ := json.Marshal(gin.H{
			"_id": us.ID.Hex(),
			"user": us.Username,
			"email": us.Email,
		})
		c.Set("User", url.QueryEscape(string(User)))
	}
	collection = db.C(ACCOUNTS)
	ac := Account{} // todo pool
	err = collection.FindId(bson.ObjectIdHex(us.Roles.Account.Hex())).One(&ac)
	if err != nil {
		println(err.Error())
	}
	if len(ac.ID) != 0 {
		Account, _ := json.Marshal(gin.H{
			"_id": ac.ID.Hex(),
			"name": gin.H{
				"first": ac.Name.First,
				"middle": ac.Name.Middle,
				"last": ac.Name.Last,
			},
			"company": ac.Company,
			"phone": ac.Phone,
			"zip": ac.Zip,
		})
		c.Set("Account", url.QueryEscape(string(Account)))
	}
	render, _ := TemplateStorage[c.Request.URL.Path]
	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}<commit_msg>[admin] render<commit_after>package main

import (
	"github.com/gin-gonic/gin"
	"net/http"
	"gopkg.in/mgo.v2/bson"
	"github.com/gin-gonic/contrib/sessions"
	"encoding/json"
	"net/url"
	"crypto/rand"
	"encoding/hex"
	"golang.org/x/crypto/bcrypt"
	"regexp"
	"strings"
)

func generateToken1(n int) []byte {
	b := make([]byte, n)
	_, err := rand.Read(b)
	if err != nil {
		println(err.Error())
		return b
	}
	token := make([]byte, n * 2)
	hex.Encode(token, b)
	return token
}

func AdminRender(c *gin.Context) {
	render, _ := TemplateStorage[c.Request.URL.Path]

/*	// TODO
	db := getMongoDBInstance()
	defer db.Session.Close()
	collection := db.C(ACCOUNTS)
	count, _ := collection.Count()
	c.Set("CountAccount", count)
	collection = db.C(USERS)
	count, _ = collection.Count()
	c.Set("CountUser", count)
	collection = db.C(ADMINS)
	count, _ = collection.Count()
	c.Set("CountAdmin", count)
	collection = db.C(ADMINGROUPS)
	count, _ = collection.Count()
	c.Set("CountAdminGroup", count)
	collection = db.C(CATEGORIES)
	count, _ = collection.Count()
	c.Set("CountCategory", count)
	collection = db.C(STATUS)
	count, _ = collection.Count()
	c.Set("CountStatus", count)*/

	db := getMongoDBInstance()
	defer db.Session.Close()

	CountAccountChan := make(chan int)
	CountUserChan := make(chan int)
	CountAdminChan := make(chan int)
	CountAdminGroupChan := make(chan int)
	CountCategoryChan := make(chan int)
	CountStatusChan := make(chan int)

	go getCount(db.C(ACCOUNTS), CountAccountChan, bson.M{})
	go getCount(db.C(USERS), CountUserChan, bson.M{})
	go getCount(db.C(ADMINS), CountAdminChan, bson.M{})
	go getCount(db.C(ADMINGROUPS), CountAdminGroupChan, bson.M{})
	go getCount(db.C(CATEGORIES), CountCategoryChan, bson.M{})
	go getCount(db.C(STATUS), CountStatusChan, bson.M{})

	c.Set("CountAccount", <- CountAccountChan)
	c.Set("CountUser", <- CountUserChan)
	c.Set("CountAdmin", <- CountAdminChan)
	c.Set("CountAdminGroup", <- CountAdminGroupChan)
	c.Set("CountCategory", <- CountCategoryChan)
	c.Set("CountStatus", <- CountStatusChan)

	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}

func AccountVerificationRender1(c *gin.Context) {
	account, _ := getAccount(c)
	user, _ := getUser(c)
	if account.IsVerified == "yes" {
		c.Redirect(http.StatusFound, user.DefaultReturnUrl())
		return
	}
	if len(account.VerificationToken) > 0 {

	} else {
		VerifyURL := generateToken(21)
		hash, err := bcrypt.GenerateFromPassword(VerifyURL, bcrypt.DefaultCost)
		if err != nil {
			println(err.Error())
			return
		}

		db := getMongoDBInstance()
		defer db.Session.Close()
		collection := db.C(ACCOUNTS)
		account.VerificationToken = string(hash)
		collection.UpdateId(account.ID, account)// todo how to update only part?
		verifyURL := "http" +"://"+ c.Request.Host +"/account/verification/" + string(VerifyURL) + "/"
		c.Set("VerifyURL", verifyURL)

		mailConf := MailConfig{}
		mailConf.Data = c.Keys
		mailConf.From = config.SMTP.From.Name + " <" + config.SMTP.From.Address + ">"
		mailConf.To = user.Email
		mailConf.Subject = "Your " + config.ProjectName + " Account"
		mailConf.ReplyTo = user.Email
		mailConf.HtmlPath = "views/account/verification/email-html.html"

		if err := mailConf.SendMail(); err != nil {
			//todo it's not serious
		}

	}

	render, _ := TemplateStorage[c.Request.URL.Path]
	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}

func Verify1 (c *gin.Context) {

	account, _ := getAccount(c)
	user, _ := getUser(c)
	err := bcrypt.CompareHashAndPassword([]byte(account.VerificationToken), []byte(c.Param("token")))
	if err == nil {
		db := getMongoDBInstance()
		defer db.Session.Close()
		collection := db.C(ACCOUNTS)
		account.VerificationToken = ""
		account.IsVerified = "yes"
		collection.UpdateId(account.ID, account)
	}
	c.Redirect(http.StatusFound, user.DefaultReturnUrl())
}

func ResendVerification1 (c *gin.Context) {
	account, _ := getAccount(c)
	user, _ := getUser(c)
	if account.IsVerified == "yes" {
		c.HTML(http.StatusOK, user.DefaultReturnUrl(), c.Keys)
		return
	}
	response := Response{} // todo sync.Pool
	response.Errors = []string{}
	response.ErrFor = make(map[string]string)
	defer response.Recover(c)

	var body struct {
		Username    string  `json:"username"`
		Email   string  `json:"email"`
		Password string  `json:"password"`
	}
	decoder := json.NewDecoder(c.Request.Body)
	err := decoder.Decode(&body)

	email := strings.ToLower(body.Email)
	if len(email) == 0 {
		response.ErrFor["email"] = "required"
	} else {
		r, err := regexp.MatchString(`^[a-zA-Z0-9\-\_\.\+]+@[a-zA-Z0-9\-\_\.]+\.[a-zA-Z0-9\-\_]+$`, email)
		if err != nil {
			println(err.Error())
		}
		if !r {
			response.ErrFor["email"] = `invalid email format`
		}
	}
	if response.HasErrors() {
		response.Fail(c)
		return
	}
	db := getMongoDBInstance()
	defer db.Session.Close()
	collection := db.C(USERS)
	{
		user_ := User{}
		collection.Find(bson.M{"email": email, "_id": bson.M{"$ne": user.ID}}).One(&user_)
		if len(user_.Username) > 0 {
			response.ErrFor["email"] = `email already taken`
		}
	}
	if response.HasErrors() {
		response.Fail(c)
		return
	}
	user.Email = email
	collection.UpdateId(user.ID, user)

	collection = db.C(ACCOUNTS)
	VerifyURL := generateToken(21)
	hash, err := bcrypt.GenerateFromPassword(VerifyURL, bcrypt.DefaultCost)
	if err != nil {
		println(err.Error())
		return
	}
	account.VerificationToken = string(hash)
	collection.UpdateId(account.ID, account)// todo how to update only part?
	verifyURL := "http" +"://"+ c.Request.Host +"/account/verification/" + string(VerifyURL) + "/"
	c.Set("VerifyURL", verifyURL)
	mailConf := MailConfig{}
	mailConf.Data = c.Keys
	mailConf.From = config.SMTP.From.Name + " <" + config.SMTP.From.Address + ">"
	mailConf.To = user.Email
	mailConf.Subject = "Your " + config.ProjectName + " Account"
	mailConf.ReplyTo = user.Email
	mailConf.HtmlPath = "views/account/verification/email-html.html"

	if err := mailConf.SendMail(); err != nil {
		//todo it's not serious
	}

	response.Success = true
	c.JSON(http.StatusOK, response)
}

func AccountSettingsRender1(c *gin.Context) {
	sess := sessions.Default(c)

	public := sess.Get("public")
	db := getMongoDBInstance()
	defer db.Session.Close()
	collection := db.C(USERS)
	us := User{} // todo pool
	err := collection.FindId(bson.ObjectIdHex(public.(string))).One(&us)
	if err != nil {
		println(err.Error())
	}
	if len(us.Username) != 0 {
		User, _ := json.Marshal(gin.H{
			"_id": us.ID.Hex(),
			"user": us.Username,
			"email": us.Email,
		})
		c.Set("User", url.QueryEscape(string(User)))
	}
	collection = db.C(ACCOUNTS)
	ac := Account{} // todo pool
	err = collection.FindId(bson.ObjectIdHex(us.Roles.Account.Hex())).One(&ac)
	if err != nil {
		println(err.Error())
	}
	if len(ac.ID) != 0 {
		Account, _ := json.Marshal(gin.H{
			"_id": ac.ID.Hex(),
			"name": gin.H{
				"first": ac.Name.First,
				"middle": ac.Name.Middle,
				"last": ac.Name.Last,
			},
			"company": ac.Company,
			"phone": ac.Phone,
			"zip": ac.Zip,
		})
		c.Set("Account", url.QueryEscape(string(Account)))
	}
	render, _ := TemplateStorage[c.Request.URL.Path]
	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}<|endoftext|>


<commit_before>package cli_test

import (
	"bufio"
	"bytes"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"

	"github.com/influxdb/influxdb/client"
	"github.com/influxdb/influxdb/cmd/influx/cli"
	"github.com/peterh/liner"
)

func TestParseCommand_CommandsExist(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{}
	tests := []struct {
		cmd string
	}{
		{cmd: "gopher"},
		{cmd: "connect"},
		{cmd: "help"},
		{cmd: "pretty"},
		{cmd: "use"},
	}
	for _, test := range tests {
		if !c.ParseCommand(test.cmd) {
			t.Fatalf(`Command failed for %q.`, test.cmd)
		}
	}
}

func TestParseCommand_BlankCommand(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{}
	tests := []struct {
		cmd string
	}{
		{cmd: ""}, // test that a blank command doesn't work
	}
	for _, test := range tests {
		if c.ParseCommand(test.cmd) {
			t.Fatalf(`Command failed for %q.`, test.cmd)
		}
	}
}

func TestParseCommand_TogglePretty(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{}
	if c.Pretty {
		t.Fatalf(`Pretty should be false.`)
	}
	c.ParseCommand("pretty")
	if !c.Pretty {
		t.Fatalf(`Pretty should be true.`)
	}
	c.ParseCommand("pretty")
	if c.Pretty {
		t.Fatalf(`Pretty should be false.`)
	}
}

func TestParseCommand_Exit(t *testing.T) {
	t.Parallel()
	tests := []struct {
		cmd string
	}{
		{cmd: "exit"},
		{cmd: " exit"},
		{cmd: "exit "},
		{cmd: "Exit "},
	}

	for _, test := range tests {
		c := cli.CommandLine{Quit: make(chan struct{}, 1)}
		c.ParseCommand(test.cmd)
		// channel should be closed
		if _, ok := <-c.Quit; ok {
			t.Fatalf(`Command "exit" failed for %q.`, test.cmd)
		}
	}
}

func TestParseCommand_Use(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{}
	tests := []struct {
		cmd string
	}{
		{cmd: "use db"},
		{cmd: " use db"},
		{cmd: "use db "},
		{cmd: "use db;"},
		{cmd: "use db; "},
		{cmd: "Use db"},
	}

	for _, test := range tests {
		if !c.ParseCommand(test.cmd) {
			t.Fatalf(`Command "use" failed for %q.`, test.cmd)
		}

		if c.Database != "db" {
			t.Fatalf(`Command "use" changed database to %q. Expected db`, c.Database)
		}
	}
}

func TestParseCommand_Consistency(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{}
	tests := []struct {
		cmd string
	}{
		{cmd: "consistency one"},
		{cmd: " consistency one"},
		{cmd: "consistency one "},
		{cmd: "consistency one;"},
		{cmd: "consistency one; "},
		{cmd: "Consistency one"},
	}

	for _, test := range tests {
		if !c.ParseCommand(test.cmd) {
			t.Fatalf(`Command "consistency" failed for %q.`, test.cmd)
		}

		if c.WriteConsistency != "one" {
			t.Fatalf(`Command "consistency" changed consistency to %q. Expected one`, c.WriteConsistency)
		}
	}
}

func TestParseCommand_Insert(t *testing.T) {
	t.Parallel()
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var data client.Response
		w.WriteHeader(http.StatusNoContent)
		_ = json.NewEncoder(w).Encode(data)
	}))
	defer ts.Close()

	u, _ := url.Parse(ts.URL)
	config := client.Config{URL: *u}
	c, err := client.NewClient(config)
	if err != nil {
		t.Fatalf("unexpected error.  expected %v, actual %v", nil, err)
	}
	m := cli.CommandLine{Client: c}

	tests := []struct {
		cmd string
	}{
		{cmd: "INSERT cpu,host=serverA,region=us-west value=1.0"},
		{cmd: " INSERT cpu,host=serverA,region=us-west value=1.0"},
		{cmd: "INSERT   cpu,host=serverA,region=us-west value=1.0"},
		{cmd: "insert cpu,host=serverA,region=us-west    value=1.0    "},
		{cmd: "insert"},
		{cmd: "Insert "},
		{cmd: "insert c"},
		{cmd: "insert int"},
	}

	for _, test := range tests {
		if !m.ParseCommand(test.cmd) {
			t.Fatalf(`Command "insert" failed for %q.`, test.cmd)
		}
	}
}

func TestParseCommand_InsertInto(t *testing.T) {
	t.Parallel()
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var data client.Response
		w.WriteHeader(http.StatusNoContent)
		_ = json.NewEncoder(w).Encode(data)
	}))
	defer ts.Close()

	u, _ := url.Parse(ts.URL)
	config := client.Config{URL: *u}
	c, err := client.NewClient(config)
	if err != nil {
		t.Fatalf("unexpected error.  expected %v, actual %v", nil, err)
	}
	m := cli.CommandLine{Client: c}

	tests := []struct {
		cmd, db, rp string
	}{
		{
			cmd: `INSERT INTO test cpu,host=serverA,region=us-west value=1.0`,
			db:  "",
			rp:  "test",
		},
		{
			cmd: ` INSERT INTO .test cpu,host=serverA,region=us-west value=1.0`,
			db:  "",
			rp:  "test",
		},
		{
			cmd: `INSERT INTO   "test test" cpu,host=serverA,region=us-west value=1.0`,
			db:  "",
			rp:  "test test",
		},
		{
			cmd: `Insert iNTO test.test cpu,host=serverA,region=us-west value=1.0`,
			db:  "test",
			rp:  "test",
		},
		{
			cmd: `insert into "test test" cpu,host=serverA,region=us-west value=1.0`,
			db:  "test",
			rp:  "test test",
		},
		{
			cmd: `insert into "d b"."test test" cpu,host=serverA,region=us-west value=1.0`,
			db:  "d b",
			rp:  "test test",
		},
	}

	for _, test := range tests {
		if !m.ParseCommand(test.cmd) {
			t.Fatalf(`Command "insert into" failed for %q.`, test.cmd)
		}
		if m.Database != test.db {
			t.Fatalf(`Command "insert into" db parsing failed, expected: %q, actual: %q`, test.db, m.Database)
		}
		if m.RetentionPolicy != test.rp {
			t.Fatalf(`Command "insert into" rp parsing failed, expected: %q, actual: %q`, test.rp, m.RetentionPolicy)
		}
	}
}

func TestParseCommand_History(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{Line: liner.NewLiner()}
	defer c.Line.Close()

	// append one entry to history
	c.Line.AppendHistory("abc")

	tests := []struct {
		cmd string
	}{
		{cmd: "history"},
		{cmd: " history"},
		{cmd: "history "},
		{cmd: "History "},
	}

	for _, test := range tests {
		if !c.ParseCommand(test.cmd) {
			t.Fatalf(`Command "history" failed for %q.`, test.cmd)
		}
	}

	// buf size should be at least 1
	var buf bytes.Buffer
	c.Line.WriteHistory(&buf)
	if buf.Len() < 1 {
		t.Fatal("History is borked")
	}
}

func TestParseCommand_HistoryWithBlankCommand(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{Line: liner.NewLiner()}
	defer c.Line.Close()

	// append one entry to history
	c.Line.AppendHistory("x")

	tests := []struct {
		cmd string
	}{
		{cmd: "history"},
		{cmd: " history"},
		{cmd: "history "},
		{cmd: "History "},
		{cmd: ""},  // shouldn't be persisted in history
		{cmd: " "}, // shouldn't be persisted in history
	}

	// don't validate because blank commands are never executed
	for _, test := range tests {
		c.ParseCommand(test.cmd)
	}

	// buf shall not contain empty commands
	var buf bytes.Buffer
	c.Line.WriteHistory(&buf)
	scanner := bufio.NewScanner(&buf)
	for scanner.Scan() {
		if scanner.Text() == "" || scanner.Text() == " " {
			t.Fatal("Empty commands should not be persisted in history.")
		}
	}
}
<commit_msg>Added more tests to increase coverage. Refs #2313<commit_after>package cli_test

import (
	"bufio"
	"bytes"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"testing"

	"github.com/influxdb/influxdb/client"
	"github.com/influxdb/influxdb/cmd/influx/cli"
	"github.com/peterh/liner"
)

const (
	CLIENT_VERSION = "y.y"
	SERVER_VERSION = "x.x"
)

func TestNewCLI(t *testing.T) {
	t.Parallel()
	c := cli.New(CLIENT_VERSION)

	if c == nil {
		t.Fatal("CommandLine shouldn't be nil.")
	}

	if c.ClientVersion != CLIENT_VERSION {
		t.Fatalf("CommandLine version is %s but should be %s", c.ClientVersion, CLIENT_VERSION)
	}
}

func TestRunCLI(t *testing.T) {
	t.Parallel()
	ts := emptyTestServer()
	defer ts.Close()

	u, _ := url.Parse(ts.URL)
	h, p, _ := net.SplitHostPort(u.Host)
	c := cli.New(CLIENT_VERSION)
	c.Host = h
	c.Port, _ = strconv.Atoi(p)
	c.Run()
}

func TestConnect(t *testing.T) {
	t.Parallel()
	ts := emptyTestServer()
	defer ts.Close()

	u, _ := url.Parse(ts.URL)
	cmd := "connect " + u.Host
	c := cli.CommandLine{}

	// assert connection is established
	if err := c.Connect(cmd); err != nil {
		t.Fatalf("There was an error while connecting to %s: %s", u.Path, err)
	}

	// assert server version is populated
	if c.ServerVersion != SERVER_VERSION {
		t.Fatalf("Server version is %s but should be %s.", c.ServerVersion, SERVER_VERSION)
	}
}

func TestSetAuth(t *testing.T) {
	t.Parallel()
	c := cli.New(CLIENT_VERSION)
	config := client.NewConfig()
	client, _ := client.NewClient(config)
	c.Client = client
	u := "userx"
	p := "pwdy"
	c.SetAuth("auth " + u + " " + p)

	// validate CLI configuration
	if c.Username != u {
		t.Fatalf("Username is %s but should be %s", c.Username, u)
	}
	if c.Password != p {
		t.Fatalf("Password is %s but should be %s", c.Password, p)
	}
}

func TestSetPrecision(t *testing.T) {
	t.Parallel()
	c := cli.New(CLIENT_VERSION)
	config := client.NewConfig()
	client, _ := client.NewClient(config)
	c.Client = client

	// validate set non-default precision
	p := "ns"
	c.SetPrecision("precision " + p)
	if c.Precision != p {
		t.Fatalf("Precision is %s but should be %s", c.Precision, p)
	}

	// validate set default precision which equals empty string
	p = "rfc3339"
	c.SetPrecision("precision " + p)
	if c.Precision != "" {
		t.Fatalf("Precision is %s but should be empty", c.Precision)
	}
}

func TestSetFormat(t *testing.T) {
	t.Parallel()
	c := cli.New(CLIENT_VERSION)
	config := client.NewConfig()
	client, _ := client.NewClient(config)
	c.Client = client

	// validate set non-default format
	f := "json"
	c.SetFormat("format " + f)
	if c.Format != f {
		t.Fatalf("Format is %s but should be %s", c.Format, f)
	}
}

func TestSetWriteConsistency(t *testing.T) {
	t.Parallel()
	c := cli.New(CLIENT_VERSION)
	config := client.NewConfig()
	client, _ := client.NewClient(config)
	c.Client = client

	// set valid write consistency
	consistency := "all"
	c.SetWriteConsistency("consistency " + consistency)
	if c.WriteConsistency != consistency {
		t.Fatalf("WriteConsistency is %s but should be %s", c.WriteConsistency, consistency)
	}

	// set different valid write consistency and validate change
	consistency = "quorum"
	c.SetWriteConsistency("consistency " + consistency)
	if c.WriteConsistency != consistency {
		t.Fatalf("WriteConsistency is %s but should be %s", c.WriteConsistency, consistency)
	}

	// set invalid write consistency and verify there was no change
	invalidConsistency := "invalid_consistency"
	c.SetWriteConsistency("consistency " + invalidConsistency)
	if c.WriteConsistency == invalidConsistency {
		t.Fatalf("WriteConsistency is %s but should be %s", c.WriteConsistency, consistency)
	}
}

func TestParseCommand_CommandsExist(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{}
	tests := []struct {
		cmd string
	}{
		{cmd: "gopher"},
		{cmd: "connect"},
		{cmd: "help"},
		{cmd: "pretty"},
		{cmd: "use"},
	}
	for _, test := range tests {
		if !c.ParseCommand(test.cmd) {
			t.Fatalf(`Command failed for %q.`, test.cmd)
		}
	}
}

func TestParseCommand_BlankCommand(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{}
	tests := []struct {
		cmd string
	}{
		{cmd: ""}, // test that a blank command doesn't work
	}
	for _, test := range tests {
		if c.ParseCommand(test.cmd) {
			t.Fatalf(`Command failed for %q.`, test.cmd)
		}
	}
}

func TestParseCommand_TogglePretty(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{}
	if c.Pretty {
		t.Fatalf(`Pretty should be false.`)
	}
	c.ParseCommand("pretty")
	if !c.Pretty {
		t.Fatalf(`Pretty should be true.`)
	}
	c.ParseCommand("pretty")
	if c.Pretty {
		t.Fatalf(`Pretty should be false.`)
	}
}

func TestParseCommand_Exit(t *testing.T) {
	t.Parallel()
	tests := []struct {
		cmd string
	}{
		{cmd: "exit"},
		{cmd: " exit"},
		{cmd: "exit "},
		{cmd: "Exit "},
	}

	for _, test := range tests {
		c := cli.CommandLine{Quit: make(chan struct{}, 1)}
		c.ParseCommand(test.cmd)
		// channel should be closed
		if _, ok := <-c.Quit; ok {
			t.Fatalf(`Command "exit" failed for %q.`, test.cmd)
		}
	}
}

func TestParseCommand_Use(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{}
	tests := []struct {
		cmd string
	}{
		{cmd: "use db"},
		{cmd: " use db"},
		{cmd: "use db "},
		{cmd: "use db;"},
		{cmd: "use db; "},
		{cmd: "Use db"},
	}

	for _, test := range tests {
		if !c.ParseCommand(test.cmd) {
			t.Fatalf(`Command "use" failed for %q.`, test.cmd)
		}

		if c.Database != "db" {
			t.Fatalf(`Command "use" changed database to %q. Expected db`, c.Database)
		}
	}
}

func TestParseCommand_Consistency(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{}
	tests := []struct {
		cmd string
	}{
		{cmd: "consistency one"},
		{cmd: " consistency one"},
		{cmd: "consistency one "},
		{cmd: "consistency one;"},
		{cmd: "consistency one; "},
		{cmd: "Consistency one"},
	}

	for _, test := range tests {
		if !c.ParseCommand(test.cmd) {
			t.Fatalf(`Command "consistency" failed for %q.`, test.cmd)
		}

		if c.WriteConsistency != "one" {
			t.Fatalf(`Command "consistency" changed consistency to %q. Expected one`, c.WriteConsistency)
		}
	}
}

func TestParseCommand_Insert(t *testing.T) {
	t.Parallel()
	ts := emptyTestServer()
	defer ts.Close()

	u, _ := url.Parse(ts.URL)
	config := client.Config{URL: *u}
	c, err := client.NewClient(config)
	if err != nil {
		t.Fatalf("unexpected error.  expected %v, actual %v", nil, err)
	}
	m := cli.CommandLine{Client: c}

	tests := []struct {
		cmd string
	}{
		{cmd: "INSERT cpu,host=serverA,region=us-west value=1.0"},
		{cmd: " INSERT cpu,host=serverA,region=us-west value=1.0"},
		{cmd: "INSERT   cpu,host=serverA,region=us-west value=1.0"},
		{cmd: "insert cpu,host=serverA,region=us-west    value=1.0    "},
		{cmd: "insert"},
		{cmd: "Insert "},
		{cmd: "insert c"},
		{cmd: "insert int"},
	}

	for _, test := range tests {
		if !m.ParseCommand(test.cmd) {
			t.Fatalf(`Command "insert" failed for %q.`, test.cmd)
		}
	}
}

func TestParseCommand_InsertInto(t *testing.T) {
	t.Parallel()
	ts := emptyTestServer()
	defer ts.Close()

	u, _ := url.Parse(ts.URL)
	config := client.Config{URL: *u}
	c, err := client.NewClient(config)
	if err != nil {
		t.Fatalf("unexpected error.  expected %v, actual %v", nil, err)
	}
	m := cli.CommandLine{Client: c}

	tests := []struct {
		cmd, db, rp string
	}{
		{
			cmd: `INSERT INTO test cpu,host=serverA,region=us-west value=1.0`,
			db:  "",
			rp:  "test",
		},
		{
			cmd: ` INSERT INTO .test cpu,host=serverA,region=us-west value=1.0`,
			db:  "",
			rp:  "test",
		},
		{
			cmd: `INSERT INTO   "test test" cpu,host=serverA,region=us-west value=1.0`,
			db:  "",
			rp:  "test test",
		},
		{
			cmd: `Insert iNTO test.test cpu,host=serverA,region=us-west value=1.0`,
			db:  "test",
			rp:  "test",
		},
		{
			cmd: `insert into "test test" cpu,host=serverA,region=us-west value=1.0`,
			db:  "test",
			rp:  "test test",
		},
		{
			cmd: `insert into "d b"."test test" cpu,host=serverA,region=us-west value=1.0`,
			db:  "d b",
			rp:  "test test",
		},
	}

	for _, test := range tests {
		if !m.ParseCommand(test.cmd) {
			t.Fatalf(`Command "insert into" failed for %q.`, test.cmd)
		}
		if m.Database != test.db {
			t.Fatalf(`Command "insert into" db parsing failed, expected: %q, actual: %q`, test.db, m.Database)
		}
		if m.RetentionPolicy != test.rp {
			t.Fatalf(`Command "insert into" rp parsing failed, expected: %q, actual: %q`, test.rp, m.RetentionPolicy)
		}
	}
}

func TestParseCommand_History(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{Line: liner.NewLiner()}
	defer c.Line.Close()

	// append one entry to history
	c.Line.AppendHistory("abc")

	tests := []struct {
		cmd string
	}{
		{cmd: "history"},
		{cmd: " history"},
		{cmd: "history "},
		{cmd: "History "},
	}

	for _, test := range tests {
		if !c.ParseCommand(test.cmd) {
			t.Fatalf(`Command "history" failed for %q.`, test.cmd)
		}
	}

	// buf size should be at least 1
	var buf bytes.Buffer
	c.Line.WriteHistory(&buf)
	if buf.Len() < 1 {
		t.Fatal("History is borked")
	}
}

func TestParseCommand_HistoryWithBlankCommand(t *testing.T) {
	t.Parallel()
	c := cli.CommandLine{Line: liner.NewLiner()}
	defer c.Line.Close()

	// append one entry to history
	c.Line.AppendHistory("x")

	tests := []struct {
		cmd string
	}{
		{cmd: "history"},
		{cmd: " history"},
		{cmd: "history "},
		{cmd: "History "},
		{cmd: ""},  // shouldn't be persisted in history
		{cmd: " "}, // shouldn't be persisted in history
	}

	// don't validate because blank commands are never executed
	for _, test := range tests {
		c.ParseCommand(test.cmd)
	}

	// buf shall not contain empty commands
	var buf bytes.Buffer
	c.Line.WriteHistory(&buf)
	scanner := bufio.NewScanner(&buf)
	for scanner.Scan() {
		if scanner.Text() == "" || scanner.Text() == " " {
			t.Fatal("Empty commands should not be persisted in history.")
		}
	}
}

// helper methods

func emptyTestServer() *httptest.Server {
	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("X-Influxdb-Version", SERVER_VERSION)
		return
	}))
}
<|endoftext|>


<commit_before>// +build android plan9 nacl windows

package main

func (v *View) Suspend(usePlugin bool) bool {
	messenger.Error("Suspend is only supported on Linux")

	return false
}
<commit_msg>Remove android from actions_other build tag<commit_after>// +build plan9 nacl windows

package main

func (v *View) Suspend(usePlugin bool) bool {
	messenger.Error("Suspend is only supported on Posix")

	return false
}
<|endoftext|>


<commit_before>package termo

import (
	"errors"
	"fmt"
	"syscall"
	"unicode/utf8"

	"github.com/jonvaldes/termo/terminal"
)

// NotATerminal is the error returned when running
// termo in an unsupported environment
var NotATerminal error = errors.New("not running in a terminal")

var oldTermState *terminal.State

// Control sequences documentation: http://www.xfree86.org/current/ctlseqs.html

// Init initializes termo to work with the terminal
func Init() error {
	if !terminal.IsTerminal(syscall.Stdin) {
		return NotATerminal
	}
	var err error
	oldTermState, err = terminal.MakeRaw(syscall.Stdin)
	if err != nil {
		panic(err)
	}
	fmt.Printf("\033[?25l") // Hide cursor
	return nil
}

// Stop restores the terminal to its original state
func Stop() {
	terminal.Restore(syscall.Stdin, oldTermState)
	fmt.Printf("\033[?25h")   // Unhide cursor
	fmt.Printf("\033[?1003l") // Reset mouse
}

func EnableMouseEvents() {
	fmt.Printf("\033[?1003h")
}

// Size returns the current size of the terminal
func Size() (int, int, error) {
	return terminal.GetSize(syscall.Stdin)
}

// ScanCode contains data for a terminal keypress
type ScanCode []byte

// IsEscapeCode returns true if the terminal
// considers it an escape code
func (s ScanCode) IsEscapeCode() bool {
	return len(s) > 2 && s[0] == 27 && s[1] == 91
}

// EscapeCode returns the escape code for a keypress
func (s ScanCode) EscapeCode() byte {
	return s[2]
}

func (s ScanCode) IsMouseMoveEvent() bool {
	return len(s) == 6 && s.IsEscapeCode() && s[2] == 77 && s[3] == 67
}

func (s ScanCode) IsMouseDownEvent() bool {
	return len(s) == 6 && s.IsEscapeCode() && s[2] == 77 && s[3] == 32
}

func (s ScanCode) IsMouseUpEvent() bool {
	return len(s) == 6 && s.IsEscapeCode() && s[2] == 77 && s[3] == 35
}

// Coords returned start at [0,0] for upper-left corner
func (s ScanCode) MouseCoords() (int, int) {
	return int(s[4] - 33), int(s[5] - 33)
}

// Rune returns the actual key pressed (only for
// non-escapecode keypresses)
func (s ScanCode) Rune() rune {
	r, _ := utf8.DecodeRune(s)
	return r
}

// ReadScanCode reads a keypress from stdin.
// It will block until it can read something
func ReadScanCode() (ScanCode, error) {
	s := ScanCode{0, 0, 0, 0, 0, 0}
	_, err := syscall.Read(syscall.Stdin, s)
	return s, err
}

func StartKeyReadLoop(keyChan chan<- ScanCode, errChan chan<- error) {
	go func() {
		for {
			s, err := ReadScanCode()
			if err != nil {
				errChan <- err
				return
			}
			keyChan <- s
		}
	}()
}

type Attribute int

const (
	AttrNone  Attribute = 0
	AttrBold  Attribute = 1
	AttrDim   Attribute = 2
	AttrUnder Attribute = 4
	AttrBlink Attribute = 5
	AttrRev   Attribute = 7
	AttrHid   Attribute = 8
)

type Color int

const (
	ColorBlack Color = 30 + iota
	ColorRed
	ColorGreen
	ColorYellow
	ColorBlue
	ColorMagenta
	ColorCyan
	ColorGray
	ColorDefault Color = 39
)

func (c Color) Light() Color {
	return c + 60
}

func background(c Color) Color {
	return c + 10
}

type CellState struct {
	Attrib  Attribute
	FGColor Color
	BGColor Color
}

type cell struct {
	state CellState
	r     rune
}

// Framebuffer contains the runes to draw
// in the terminal
type Framebuffer struct {
	w, h  int
	chars []cell
}

// NewFramebuffer creates a Framebuffer with the specified size
// and initializes it filling it with blank spaces
func NewFramebuffer(w, h int) *Framebuffer {
	result := &Framebuffer{w, h, make([]cell, w*h)}
	result.Clear()
	return result
}

// Get returns the rune stored in the [x,y] position.
// If coords are outside the framebuffer size, it returns ' '
func (f *Framebuffer) Get(x, y int) (rune, CellState) {
	if x < 0 || y < 0 || x >= f.w || y >= f.h {
		return ' ', CellState{AttrNone, ColorDefault, ColorDefault}
	}
	c := f.chars[x+y*f.w]
	return c.r, c.state
}

// Put sets a rune in the specified position
func (f *Framebuffer) Put(x, y int, s CellState, r rune) {
	if x < 0 || y < 0 || x >= f.w || y >= f.h {
		return
	}
	f.chars[x+y*f.w].r = r
	f.chars[x+y*f.w].state = s
}

// PutRect fills a rectangular region with a rune
func (f *Framebuffer) PutRect(x0, y0, w, h int, s CellState, r rune) {
	for y := y0; y < y0+h; y++ {
		for x := x0; x < x0+w; x++ {
			f.Put(x, y, s, r)
		}
	}
}

// PutText draws a string from left to right, starting at x0,y0
// There is no wrapping mechanism, and parts of the text outside
// the framebuffer will be ignored.
func (f *Framebuffer) PutText(x0, y0 int, s CellState, t string) {
	i := 0
	for _, runeValue := range t {
		f.Put(x0+i, y0, s, runeValue)
		i++
	}
}

// Clear fills the framebuffer with blank spaces
func (f *Framebuffer) Clear() {
	f.PutRect(0, 0, f.w, f.h, CellState{Attrib: AttrNone, FGColor: ColorDefault, BGColor: ColorDefault}, ' ')
}

// Flush pushes the current state of the framebuffer to the terminal
func (f *Framebuffer) Flush() {
	fmt.Printf("\033[0;0H")
	for y := 0; y < f.h; y++ {
		if y != 0 {
			fmt.Print("\n")
		}
		for x := 0; x < f.w; x++ {
			c := f.chars[y*f.w+x]
			if c.r < 32 {
				continue
			}
			fmt.Printf("\033[%d;%d;%dm%c\033[0m", c.state.Attrib, c.state.FGColor, background(c.state.BGColor), c.r)
		}
	}
	fmt.Printf("\033[0m")
}
<commit_msg>Improved API, added rect drawing, cursor funcs<commit_after>package termo

import (
	"errors"
	"fmt"
	"syscall"
	"unicode/utf8"

	"github.com/jonvaldes/termo/terminal"
)

// NotATerminal is the error returned when running
// termo in an unsupported environment
var NotATerminal error = errors.New("not running in a terminal")

var oldTermState *terminal.State

// Control sequences documentation: http://www.xfree86.org/current/ctlseqs.html

// Init initializes termo to work with the terminal
func Init() error {
	if !terminal.IsTerminal(syscall.Stdin) {
		return NotATerminal
	}
	var err error
	oldTermState, err = terminal.MakeRaw(syscall.Stdin)
	if err != nil {
		panic(err)
	}
	HideCursor()
	return nil
}

// Stop restores the terminal to its original state
func Stop() {
	terminal.Restore(syscall.Stdin, oldTermState)
	ShowCursor()
	fmt.Printf("\033[?1003l") // Reset mouse
}

func HideCursor() {
	fmt.Printf("\033[?25l")
}

func ShowCursor() {
	fmt.Printf("\033[?25h")
}

func EnableMouseEvents() {
	fmt.Printf("\033[?1003h")
}

// Size returns the current size of the terminal
func Size() (int, int, error) {
	return terminal.GetSize(syscall.Stdin)
}

// ScanCode contains data for a terminal keypress
type ScanCode []byte

// IsEscapeCode returns true if the terminal
// considers it an escape code
func (s ScanCode) IsEscapeCode() bool {
	return len(s) > 2 && s[0] == 27 && s[1] == 91
}

// EscapeCode returns the escape code for a keypress
func (s ScanCode) EscapeCode() byte {
	return s[2]
}

func (s ScanCode) IsMouseMoveEvent() bool {
	return len(s) == 6 && s.IsEscapeCode() && s[2] == 77 && s[3] == 67
}

func (s ScanCode) IsMouseDownEvent() bool {
	return len(s) == 6 && s.IsEscapeCode() && s[2] == 77 && s[3] == 32
}

func (s ScanCode) IsMouseUpEvent() bool {
	return len(s) == 6 && s.IsEscapeCode() && s[2] == 77 && s[3] == 35
}

// Coords returned start at [0,0] for upper-left corner
func (s ScanCode) MouseCoords() (int, int) {
	return int(s[4] - 33), int(s[5] - 33)
}

// Rune returns the actual key pressed (only for
// non-escapecode keypresses)
func (s ScanCode) Rune() rune {
	r, _ := utf8.DecodeRune(s)
	return r
}

// ReadScanCode reads a keypress from stdin.
// It will block until it can read something
func ReadScanCode() (ScanCode, error) {
	s := ScanCode{0, 0, 0, 0, 0, 0}
	_, err := syscall.Read(syscall.Stdin, s)
	return s, err
}

func StartKeyReadLoop(keyChan chan<- ScanCode, errChan chan<- error) {
	go func() {
		for {
			s, err := ReadScanCode()
			if err != nil {
				errChan <- err
				return
			}
			keyChan <- s
		}
	}()
}

type Attribute int

const (
	AttrNone  Attribute = 0
	AttrBold  Attribute = 1
	AttrDim   Attribute = 2
	AttrUnder Attribute = 4
	AttrBlink Attribute = 5
	AttrRev   Attribute = 7
	AttrHid   Attribute = 8
)

type Color int

const (
	ColorBlack Color = 30 + iota
	ColorRed
	ColorGreen
	ColorYellow
	ColorBlue
	ColorMagenta
	ColorCyan
	ColorGray
	ColorDefault Color = 39
)

func (c Color) Light() Color {
	return c + 60
}

func background(c Color) Color {
	return c + 10
}

type CellState struct {
	Attrib  Attribute
	FGColor Color
	BGColor Color
}

var (
	StateDefault = CellState{Attrib: AttrNone, FGColor: ColorDefault, BGColor: ColorDefault}
)

type cell struct {
	state CellState
	r     rune
}

// Framebuffer contains the runes to draw
// in the terminal
type Framebuffer struct {
	w, h  int
	chars []cell
}

// NewFramebuffer creates a Framebuffer with the specified size
// and initializes it filling it with blank spaces
func NewFramebuffer(w, h int) *Framebuffer {
	result := &Framebuffer{w, h, make([]cell, w*h)}
	result.Clear()
	return result
}

// Get returns the rune stored in the [x,y] position.
// If coords are outside the framebuffer size, it returns ' '
func (f *Framebuffer) Get(x, y int) (rune, CellState) {
	if x < 0 || y < 0 || x >= f.w || y >= f.h {
		return ' ', CellState{AttrNone, ColorDefault, ColorDefault}
	}
	c := f.chars[x+y*f.w]
	return c.r, c.state
}

// Put sets a rune in the specified position
func (f *Framebuffer) Set(x, y int, s CellState, r rune) {
	if x < 0 || y < 0 || x >= f.w || y >= f.h {
		return
	}
	f.chars[x+y*f.w].r = r
	f.chars[x+y*f.w].state = s
}

func (f *Framebuffer) SetRune(x, y int, r rune) {
	if x < 0 || y < 0 || x >= f.w || y >= f.h {
		return
	}
	f.chars[x+y*f.w].r = r
}

// PutRect fills a rectangular region with a rune and state
func (f *Framebuffer) SetRect(x0, y0, w, h int, s CellState, r rune) {
	for y := y0; y < y0+h; y++ {
		for x := x0; x < x0+w; x++ {
			f.Set(x, y, s, r)
		}
	}
}

// PutRect fills a rectangular region with a rune and state
func (f *Framebuffer) AttribRect(x0, y0, w, h int, s CellState) {
	for y := y0; y < y0+h; y++ {
		for x := x0; x < x0+w; x++ {
			if x >= 0 && y >= 0 && x < f.w && y < f.h {
				f.chars[x+y*f.w].state = s
			}
		}
	}
}

var singleWidthCharset = []rune{'─', '│', '┌', '┐', '└', '┘'}
var doubleWidthCharset = []rune{'═', '║', '╔', '╗', '╚', '╝'}

func (f *Framebuffer) ASCIIRect(x0, y0, w, h int, doubleWidth bool, clearInside bool) {
	c := singleWidthCharset
	if doubleWidth {
		c = doubleWidthCharset
	}

	for y := y0; y < y0+h; y++ {
		for x := x0; x < x0+w; x++ {
			var r rune
			if x == 0 {
				if y == 0 {
					r = c[2]
				} else if y == y0+h-1 {
					r = c[4]
				} else {
					r = c[1]
				}
			} else if x == x0+w-1 {
				if y == 0 {
					r = c[3]
				} else if y == y0+h-1 {
					r = c[5]
				} else {
					r = c[1]
				}
			} else if y == 0 || y == y0+h-1 {
				r = c[0]
			} else {
				if !clearInside {
					continue
				} else {
					r = ' '
				}
			}

			f.SetRune(x, y, r)
		}
	}
}

// PutText draws a string from left to right, starting at x0,y0
// There is no wrapping mechanism, and parts of the text outside
// the framebuffer will be ignored.
func (f *Framebuffer) SetText(x0, y0 int, t string) {
	i := 0
	for _, runeValue := range t {
		f.SetRune(x0+i, y0, runeValue)
		i++
	}
}

// Clear fills the framebuffer with blank spaces
func (f *Framebuffer) Clear() {
	f.SetRect(0, 0, f.w, f.h, StateDefault, ' ')
}

// Flush pushes the current state of the framebuffer to the terminal
func (f *Framebuffer) Flush() {
	fmt.Printf("\033[0;0H")
	for y := 0; y < f.h; y++ {
		if y != 0 {
			fmt.Print("\n")
		}
		for x := 0; x < f.w; x++ {
			c := f.chars[y*f.w+x]
			if c.r < 32 {
				continue
			}
			fmt.Printf("\033[%d;%d;%dm%c\033[0m", c.state.Attrib, c.state.FGColor, background(c.state.BGColor), c.r)
		}
	}
	fmt.Printf("\033[0m")
}
<|endoftext|>


<commit_before>package cli

import (
	"context"
	"sort"

	"github.com/kopia/kopia/internal/timetrack"
	"github.com/kopia/kopia/internal/units"
	"github.com/kopia/kopia/repo/content"
	"github.com/kopia/kopia/repo/encryption"
	"github.com/kopia/kopia/repo/hashing"
)

var (
	benchmarkCryptoCommand              = benchmarkCommands.Command("crypto", "Run hash and encryption benchmarks")
	benchmarkCryptoBlockSize            = benchmarkCryptoCommand.Flag("block-size", "Size of a block to encrypt").Default("1MB").Bytes()
	benchmarkCryptoRepeat               = benchmarkCryptoCommand.Flag("repeat", "Number of repetitions").Default("100").Int()
	benchmarkCryptoDeprecatedAlgorithms = benchmarkCryptoCommand.Flag("deprecated", "Include deprecated algorithms").Bool()
	benchmarkCryptoOptionPrint          = benchmarkCryptoCommand.Flag("print-options", "Print out options usable for repository creation").Bool()
)

func runBenchmarkCryptoAction(ctx context.Context) error {
	type benchResult struct {
		hash       string
		encryption string
		throughput float64
	}

	var results []benchResult

	data := make([]byte, *benchmarkCryptoBlockSize)

	const (
		maxEncryptionOverhead = 1024
		maxHashSize           = 64
	)

	var hashOutput [maxHashSize]byte

	encryptOutput := make([]byte, len(data)+maxEncryptionOverhead)

	for _, ha := range hashing.SupportedAlgorithms() {
		for _, ea := range encryption.SupportedAlgorithms(*benchmarkCryptoDeprecatedAlgorithms) {
			h, e, err := content.CreateHashAndEncryptor(&content.FormattingOptions{
				Encryption: ea,
				Hash:       ha,
				MasterKey:  make([]byte, 32),
				HMACSecret: make([]byte, 32),
			})
			if err != nil {
				continue
			}

			log(ctx).Infof("Benchmarking hash '%v' and encryption '%v'... (%v x %v bytes)", ha, ea, *benchmarkCryptoRepeat, len(data))

			tt := timetrack.Start()

			hashCount := *benchmarkCryptoRepeat

			for i := 0; i < hashCount; i++ {
				contentID := h(hashOutput[:0], data)
				if _, encerr := e.Encrypt(encryptOutput[:0], data, contentID); encerr != nil {
					log(ctx).Errorf("encryption failed: %v", encerr)
					break
				}
			}

			_, bytesPerSecond := tt.Completed(float64(len(data)) * float64(hashCount))

			results = append(results, benchResult{hash: ha, encryption: ea, throughput: bytesPerSecond})
		}
	}

	sort.Slice(results, func(i, j int) bool {
		return results[i].throughput > results[j].throughput
	})
	printStdout("     %-20v %-20v %v\n", "Hash", "Encryption", "Throughput")
	printStdout("-----------------------------------------------------------------\n")

	for ndx, r := range results {
		printStdout("%3d. %-20v %-20v %v / second", ndx, r.hash, r.encryption, units.BytesStringBase2(int64(r.throughput)))

		if *benchmarkCryptoOptionPrint {
			printStdout(",   --block-hash=%s --encryption=%s", r.hash, r.encryption)
		}

		printStdout("\n")
	}

	printStdout("-----------------------------------------------------------------\n")
	printStdout("Fastest option for this machine is: --block-hash==%s --encryption=%s\n", results[0].hash, results[0].encryption)

	return nil
}

func init() {
	benchmarkCryptoCommand.Action(noRepositoryAction(runBenchmarkCryptoAction))
}
<commit_msg>Benchmark: Remove superfluous = in printout (#1043)<commit_after>package cli

import (
	"context"
	"sort"

	"github.com/kopia/kopia/internal/timetrack"
	"github.com/kopia/kopia/internal/units"
	"github.com/kopia/kopia/repo/content"
	"github.com/kopia/kopia/repo/encryption"
	"github.com/kopia/kopia/repo/hashing"
)

var (
	benchmarkCryptoCommand              = benchmarkCommands.Command("crypto", "Run hash and encryption benchmarks")
	benchmarkCryptoBlockSize            = benchmarkCryptoCommand.Flag("block-size", "Size of a block to encrypt").Default("1MB").Bytes()
	benchmarkCryptoRepeat               = benchmarkCryptoCommand.Flag("repeat", "Number of repetitions").Default("100").Int()
	benchmarkCryptoDeprecatedAlgorithms = benchmarkCryptoCommand.Flag("deprecated", "Include deprecated algorithms").Bool()
	benchmarkCryptoOptionPrint          = benchmarkCryptoCommand.Flag("print-options", "Print out options usable for repository creation").Bool()
)

func runBenchmarkCryptoAction(ctx context.Context) error {
	type benchResult struct {
		hash       string
		encryption string
		throughput float64
	}

	var results []benchResult

	data := make([]byte, *benchmarkCryptoBlockSize)

	const (
		maxEncryptionOverhead = 1024
		maxHashSize           = 64
	)

	var hashOutput [maxHashSize]byte

	encryptOutput := make([]byte, len(data)+maxEncryptionOverhead)

	for _, ha := range hashing.SupportedAlgorithms() {
		for _, ea := range encryption.SupportedAlgorithms(*benchmarkCryptoDeprecatedAlgorithms) {
			h, e, err := content.CreateHashAndEncryptor(&content.FormattingOptions{
				Encryption: ea,
				Hash:       ha,
				MasterKey:  make([]byte, 32),
				HMACSecret: make([]byte, 32),
			})
			if err != nil {
				continue
			}

			log(ctx).Infof("Benchmarking hash '%v' and encryption '%v'... (%v x %v bytes)", ha, ea, *benchmarkCryptoRepeat, len(data))

			tt := timetrack.Start()

			hashCount := *benchmarkCryptoRepeat

			for i := 0; i < hashCount; i++ {
				contentID := h(hashOutput[:0], data)
				if _, encerr := e.Encrypt(encryptOutput[:0], data, contentID); encerr != nil {
					log(ctx).Errorf("encryption failed: %v", encerr)
					break
				}
			}

			_, bytesPerSecond := tt.Completed(float64(len(data)) * float64(hashCount))

			results = append(results, benchResult{hash: ha, encryption: ea, throughput: bytesPerSecond})
		}
	}

	sort.Slice(results, func(i, j int) bool {
		return results[i].throughput > results[j].throughput
	})
	printStdout("     %-20v %-20v %v\n", "Hash", "Encryption", "Throughput")
	printStdout("-----------------------------------------------------------------\n")

	for ndx, r := range results {
		printStdout("%3d. %-20v %-20v %v / second", ndx, r.hash, r.encryption, units.BytesStringBase2(int64(r.throughput)))

		if *benchmarkCryptoOptionPrint {
			printStdout(",   --block-hash=%s --encryption=%s", r.hash, r.encryption)
		}

		printStdout("\n")
	}

	printStdout("-----------------------------------------------------------------\n")
	printStdout("Fastest option for this machine is: --block-hash=%s --encryption=%s\n", results[0].hash, results[0].encryption)

	return nil
}

func init() {
	benchmarkCryptoCommand.Action(noRepositoryAction(runBenchmarkCryptoAction))
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"

	"github.com/spf13/cobra"
)

var cmdRebuildIndex = &cobra.Command{
	Use:   "rebuild-index [flags]",
	Short: "Build a new index",
	Long: `
The "rebuild-index" command creates a new index based on the pack files in the
repository.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runRebuildIndex(rebuildIndexOptions, globalOptions)
	},
}

// RebuildIndexOptions collects all options for the rebuild-index command.
type RebuildIndexOptions struct {
	ReadAllPacks bool
}

var rebuildIndexOptions RebuildIndexOptions

func init() {
	cmdRoot.AddCommand(cmdRebuildIndex)
	f := cmdRebuildIndex.Flags()
	f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")

}

func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepoExclusive(gopts.ctx, repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	return rebuildIndex(opts, gopts, repo, restic.NewIDSet())
}

func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
	ctx := gopts.ctx

	var obsoleteIndexes restic.IDs
	packSizeFromList := make(map[restic.ID]int64)
	packSizeFromIndex := make(map[restic.ID]int64)
	removePacks := restic.NewIDSet()

	if opts.ReadAllPacks {
		// get list of old index files but start with empty index
		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
			obsoleteIndexes = append(obsoleteIndexes, id)
			return nil
		})
		if err != nil {
			return err
		}
	} else {
		Verbosef("loading indexes...\n")
		err := repo.LoadIndex(gopts.ctx)
		if err != nil {
			return err
		}
		packSizeFromIndex = repo.Index().PackSize(ctx, false)
	}

	Verbosef("getting pack files to read...\n")
	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
		size, ok := packSizeFromIndex[id]
		if !ok || size != packSize {
			// Pack was not referenced in index or size does not match
			packSizeFromList[id] = packSize
			removePacks.Insert(id)
		}
		if !ok {
			Warnf("adding pack file to index %v\n", id)
		} else if size != packSize {
			Warnf("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size)
		}
		delete(packSizeFromIndex, id)
		return nil
	})
	if err != nil {
		return err
	}
	for id := range packSizeFromIndex {
		// forget pack files that are referenced in the index but do not exist
		// when rebuilding the index
		removePacks.Insert(id)
		Warnf("removing not found pack file %v\n", id)
	}

	if len(packSizeFromList) > 0 {
		Verbosef("reading pack files\n")
		bar := newProgressMax(!globalOptions.Quiet, uint64(len(packSizeFromList)), "packs")
		invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
		bar.Done()
		if err != nil {
			return err
		}

		for _, id := range invalidFiles {
			Verboseff("skipped incomplete pack file: %v\n", id)
		}
	}

	err = rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes)
	if err != nil {
		return err
	}
	Verbosef("done\n")

	return nil
}
<commit_msg>rebuild-index: Ignore broken index files<commit_after>package main

import (
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"

	"github.com/spf13/cobra"
)

var cmdRebuildIndex = &cobra.Command{
	Use:   "rebuild-index [flags]",
	Short: "Build a new index",
	Long: `
The "rebuild-index" command creates a new index based on the pack files in the
repository.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runRebuildIndex(rebuildIndexOptions, globalOptions)
	},
}

// RebuildIndexOptions collects all options for the rebuild-index command.
type RebuildIndexOptions struct {
	ReadAllPacks bool
}

var rebuildIndexOptions RebuildIndexOptions

func init() {
	cmdRoot.AddCommand(cmdRebuildIndex)
	f := cmdRebuildIndex.Flags()
	f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")

}

func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepoExclusive(gopts.ctx, repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	return rebuildIndex(opts, gopts, repo, restic.NewIDSet())
}

func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
	ctx := gopts.ctx

	var obsoleteIndexes restic.IDs
	packSizeFromList := make(map[restic.ID]int64)
	packSizeFromIndex := make(map[restic.ID]int64)
	removePacks := restic.NewIDSet()

	if opts.ReadAllPacks {
		// get list of old index files but start with empty index
		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
			obsoleteIndexes = append(obsoleteIndexes, id)
			return nil
		})
		if err != nil {
			return err
		}
	} else {
		Verbosef("loading indexes...\n")
		mi := repository.NewMasterIndex()
		err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
			if err != nil {
				Warnf("removing invalid index %v: %v\n", id, err)
				obsoleteIndexes = append(obsoleteIndexes, id)
				return nil
			}

			mi.Insert(idx)
			return nil
		})
		if err != nil {
			return err
		}

		err = mi.MergeFinalIndexes()
		if err != nil {
			return err
		}

		err = repo.SetIndex(mi)
		if err != nil {
			return err
		}
		packSizeFromIndex = repo.Index().PackSize(ctx, false)
	}

	Verbosef("getting pack files to read...\n")
	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
		size, ok := packSizeFromIndex[id]
		if !ok || size != packSize {
			// Pack was not referenced in index or size does not match
			packSizeFromList[id] = packSize
			removePacks.Insert(id)
		}
		if !ok {
			Warnf("adding pack file to index %v\n", id)
		} else if size != packSize {
			Warnf("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size)
		}
		delete(packSizeFromIndex, id)
		return nil
	})
	if err != nil {
		return err
	}
	for id := range packSizeFromIndex {
		// forget pack files that are referenced in the index but do not exist
		// when rebuilding the index
		removePacks.Insert(id)
		Warnf("removing not found pack file %v\n", id)
	}

	if len(packSizeFromList) > 0 {
		Verbosef("reading pack files\n")
		bar := newProgressMax(!globalOptions.Quiet, uint64(len(packSizeFromList)), "packs")
		invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
		bar.Done()
		if err != nil {
			return err
		}

		for _, id := range invalidFiles {
			Verboseff("skipped incomplete pack file: %v\n", id)
		}
	}

	err = rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes)
	if err != nil {
		return err
	}
	Verbosef("done\n")

	return nil
}
<|endoftext|>