<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	etcdraft "github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
	hashic "github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/relab/raft"
	etcd "github.com/relab/raft/etcd"
	hraft "github.com/relab/raft/hashicorp"
	"github.com/relab/raft/raftgorums"
	"github.com/relab/rkv/rkvpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	bgorums    = "gorums"
	betcd      = "etcd"
	bhashicorp = "hashicorp"
)

var (
	bench             = flag.Bool("quiet", false, "Silence log output")
	recover           = flag.Bool("recover", false, "Recover from stable storage")
	batch             = flag.Bool("batch", true, "Enable batching")
	serverMetrics     = flag.Bool("servermetrics", true, "Enable server-side metrics")
	electionTimeout   = flag.Duration("election", time.Second, "How long servers wait before starting an election")
	heartbeatTimeout  = flag.Duration("heartbeat", 20*time.Millisecond, "How often a heartbeat should be sent")
	entriesPerMsg     = flag.Uint64("entriespermsg", 64, "Entries per Appendentries message")
	catchupMultiplier = flag.Uint64("catchupmultiplier", 1024, "How many more times entries per message allowed during catch up")
	cache             = flag.Int("cache", 1024*1024*64, "How many entries should be kept in memory") // ~1GB @ 16bytes per entry.
)

func main() {
	var (
		id      = flag.Uint64("id", 0, "server ID")
		servers = flag.String("servers", ":9201,:9202,:9203,:9204,:9205,:9206,:9207", "comma separated list of server addresses")
		cluster = flag.String("cluster", "1,2,3", "comma separated list of server ids to form cluster with, [1 >= id <= len(servers)]")
		backend = flag.String("backend", "gorums", "Raft backend to use [gorums|etcd|hashicorp]")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*servers, ",")

	if len(nodes) == 0 {
		fmt.Print("-server argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	selected := strings.Split(*cluster, ",")

	var ids []uint64

	for _, sid := range selected {
		id, err := strconv.ParseUint(sid, 10, 64)

		if err != nil {
			fmt.Print("could not parse -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		if id <= 0 || id > uint64(len(nodes)) {
			fmt.Print("invalid -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		ids = append(ids, id)
	}

	if len(ids) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if len(ids) > len(nodes) {
		fmt.Print("-cluster specifies too many servers\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *entriesPerMsg < 1 {
		fmt.Print("-entriespermsg must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *catchupMultiplier < 1 {
		fmt.Print("-catchupmultiplier must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	logger := logrus.New()

	logFile, err := os.OpenFile(
		fmt.Sprintf("%s%sraft%.2d.log", os.TempDir(), string(filepath.Separator), *id),
		os.O_CREATE|os.O_TRUNC|os.O_APPEND|os.O_WRONLY, 0600,
	)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Hooks.Add(NewLogToFileHook(logFile))

	if *bench {
		logger.Out = ioutil.Discard
		grpc.EnableTracing = false
	}

	grpclog.SetLogger(logger)

	lis, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		logger.Fatal(err)
	}

	grpcServer := grpc.NewServer(grpc.MaxMsgSize(1024 * 1024 * 1024))

	if *serverMetrics {
		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(fmt.Sprintf(":590%d", *id), nil))
		}()
	}

	lat := raft.NewLatency()
	event := raft.NewEvent()

	var once sync.Once
	writeData := func() {
		lat.Write(fmt.Sprintf("./latency-%v.csv", time.Now().UnixNano()))
		event.Write(fmt.Sprintf("./event-%v.csv", time.Now().UnixNano()))
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		event.Record(raft.EventTerminated)
		once.Do(writeData)
		os.Exit(1)
	}()
	defer func() {
		once.Do(writeData)
	}()

	switch *backend {
	case bgorums:
		rungorums(logger, lis, grpcServer, *id, ids, nodes, lat, event)
	case betcd:
		runetcd(logger, lis, grpcServer, *id, ids, nodes, lat, event)
	case bhashicorp:
		runhashicorp(logger, lis, grpcServer, *id, ids, nodes, lat, event)
	}

}

func runhashicorp(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
	lat *raft.Latency, event *raft.Event,
) {
	servers := make([]hashic.Server, len(nodes))
	for i, addr := range nodes {
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)
		addr = host + ":" + strconv.Itoa(p-100)

		suffrage := hashic.Voter

		if !contains(uint64(i+1), ids) {
			suffrage = hashic.Nonvoter
		}

		servers[i] = hashic.Server{
			Suffrage: suffrage,
			ID:       hashic.ServerID(addr),
			Address:  hashic.ServerAddress(addr),
		}
	}

	addr, err := net.ResolveTCPAddr("tcp", string(servers[id-1].Address))
	if err != nil {
		logger.Fatal(err)
	}
	trans, err := hashic.NewTCPTransport(string(servers[id-1].Address), addr, len(nodes)+1, 10*time.Second, os.Stderr)
	if err != nil {
		logger.Fatal(err)
	}

	path := fmt.Sprintf("hashicorp%.2d.bolt", id)
	overwrite := !*recover
	// Check if file already exists.
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			// We don't need to overwrite a file that doesn't exist.
			overwrite = false
		} else {
			// If we are unable to verify the existence of the file,
			// there is probably a permission problem.
			logger.Fatal(err)
		}
	}
	if overwrite {
		if err := os.Remove(path); err != nil {
			logger.Fatal(err)
		}
	}
	logs, err := raftboltdb.NewBoltStore(path)
	if err != nil {
		logger.Fatal(err)
	}

	snaps := hashic.NewInmemSnapshotStore()

	cfg := &hashic.Config{
		LocalID:            servers[id-1].ID,
		ProtocolVersion:    hashic.ProtocolVersionMax,
		HeartbeatTimeout:   *electionTimeout,
		ElectionTimeout:    *electionTimeout,
		CommitTimeout:      *heartbeatTimeout,
		MaxAppendEntries:   int(*entriesPerMsg),
		ShutdownOnRemove:   true,
		TrailingLogs:       math.MaxUint64,
		SnapshotInterval:   120 * time.Hour,
		SnapshotThreshold:  math.MaxUint64,
		LeaderLeaseTimeout: *electionTimeout / 2,
	}

	node := hraft.NewRaft(logger, NewStore(), cfg, servers, trans, logs, logs, snaps, ids, lat, event)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	logger.Fatal(grpcServer.Serve(lis))
}

func runetcd(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
	lat *raft.Latency, event *raft.Event,
) {
	peers := make([]etcdraft.Peer, len(ids))

	for i, nid := range ids {
		addr := nodes[i]
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)

		ur, err := url.Parse("http://" + addr)
		ur.Host = host + ":" + strconv.Itoa(p-100)
		if err != nil {
			logger.Fatal(err)
		}

		peers[i] = etcdraft.Peer{
			ID:      nid,
			Context: []byte(ur.String()),
		}
	}

	dir := fmt.Sprintf("etcdwal%.2d", id)

	switch {
	case wal.Exist(dir) && !*recover:
		if err := os.RemoveAll(dir); err != nil {
			logger.Fatal(err)
		}
		fallthrough
	case !wal.Exist(dir):
		if err := os.Mkdir(dir, 0750); err != nil {
			logger.Fatalf("rkvd: cannot create dir for wal (%v)", err)
		}

		w, err := wal.Create(dir, nil)
		if err != nil {
			logger.Fatalf("rkvd: create wal error (%v)", err)
		}
		w.Close()
	}

	walsnap := walpb.Snapshot{}
	w, err := wal.Open(dir, walsnap)
	if err != nil {
		logger.Fatalf("rkvd: error loading wal (%v)", err)
	}

	_, st, ents, err := w.ReadAll()
	if err != nil {
		log.Fatalf("rkvd: failed to read WAL (%v)", err)
	}

	storage := etcdraft.NewMemoryStorage()
	storage.SetHardState(st)
	storage.Append(ents)

	node := etcd.NewRaft(
		logger,
		NewStore(),
		storage,
		w,
		&etcdraft.Config{
			ID:            id,
			ElectionTick:  int(*electionTimeout / *heartbeatTimeout),
			HeartbeatTick: 1,
			Storage:       storage,
			MaxSizePerMsg: *entriesPerMsg,
			// etcdserver says: Never overflow the rafthttp buffer,
			// which is 4096. We keep the same constant.
			MaxInflightMsgs: 4096 / 8,
			CheckQuorum:     true,
			PreVote:         true,
			Logger:          logger,
		},
		peers,
		*heartbeatTimeout,
		!contains(id, ids),
		nodes,
		lat, event,
	)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	host, port, err := net.SplitHostPort(nodes[id-1])
	if err != nil {
		logger.Fatal(err)
	}
	p, _ := strconv.Atoi(port)
	selflis := host + ":" + strconv.Itoa(p-100)

	lishttp, err := net.Listen("tcp", selflis)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Fatal(http.Serve(lishttp, node.Handler()))
}

func rungorums(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
	lat *raft.Latency, event *raft.Event,
) {
	storage, err := raft.NewFileStorage(fmt.Sprintf("db%.2d.bolt", id), !*recover)

	if err != nil {
		logger.Fatal(err)
	}

	storageWithCache := raft.NewCacheStorage(storage, *cache)

	node := raftgorums.NewRaft(NewStore(), &raftgorums.Config{
		ID:                id,
		Servers:           nodes,
		InitialCluster:    ids,
		Batch:             *batch,
		Storage:           storageWithCache,
		ElectionTimeout:   *electionTimeout,
		HeartbeatTimeout:  *heartbeatTimeout,
		EntriesPerMsg:     *entriesPerMsg,
		CatchupMultiplier: *catchupMultiplier,
		Logger:            logger,
		MetricsEnabled:    true,
	}, lat, event)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	logger.Fatal(node.Run(grpcServer))
}

func contains(x uint64, xs []uint64) bool {
	for _, y := range xs {
		if y == x {
			return true
		}
	}
	return false
}
<commit_msg>rkvd/main.go: Max MaxMsgSize a flag<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	etcdraft "github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
	hashic "github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/relab/raft"
	etcd "github.com/relab/raft/etcd"
	hraft "github.com/relab/raft/hashicorp"
	"github.com/relab/raft/raftgorums"
	"github.com/relab/rkv/rkvpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	bgorums    = "gorums"
	betcd      = "etcd"
	bhashicorp = "hashicorp"
)

var (
	bench             = flag.Bool("quiet", false, "Silence log output")
	recover           = flag.Bool("recover", false, "Recover from stable storage")
	batch             = flag.Bool("batch", true, "Enable batching")
	serverMetrics     = flag.Bool("servermetrics", true, "Enable server-side metrics")
	electionTimeout   = flag.Duration("election", time.Second, "How long servers wait before starting an election")
	heartbeatTimeout  = flag.Duration("heartbeat", 20*time.Millisecond, "How often a heartbeat should be sent")
	entriesPerMsg     = flag.Uint64("entriespermsg", 64, "Entries per Appendentries message")
	catchupMultiplier = flag.Uint64("catchupmultiplier", 1024, "How many more times entries per message allowed during catch up")
	cache             = flag.Int("cache", 1024*1024*64, "How many entries should be kept in memory") // ~1GB @ 16bytes per entry.
	maxgrpc           = flag.Int("maxgrpc", 128<<20, "Max GRPC message size")                        // ~128MB.
)

func main() {
	var (
		id      = flag.Uint64("id", 0, "server ID")
		servers = flag.String("servers", ":9201,:9202,:9203,:9204,:9205,:9206,:9207", "comma separated list of server addresses")
		cluster = flag.String("cluster", "1,2,3", "comma separated list of server ids to form cluster with, [1 >= id <= len(servers)]")
		backend = flag.String("backend", "gorums", "Raft backend to use [gorums|etcd|hashicorp]")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*servers, ",")

	if len(nodes) == 0 {
		fmt.Print("-server argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	selected := strings.Split(*cluster, ",")

	var ids []uint64

	for _, sid := range selected {
		id, err := strconv.ParseUint(sid, 10, 64)

		if err != nil {
			fmt.Print("could not parse -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		if id <= 0 || id > uint64(len(nodes)) {
			fmt.Print("invalid -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		ids = append(ids, id)
	}

	if len(ids) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if len(ids) > len(nodes) {
		fmt.Print("-cluster specifies too many servers\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *entriesPerMsg < 1 {
		fmt.Print("-entriespermsg must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *catchupMultiplier < 1 {
		fmt.Print("-catchupmultiplier must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	logger := logrus.New()

	logFile, err := os.OpenFile(
		fmt.Sprintf("%s%sraft%.2d.log", os.TempDir(), string(filepath.Separator), *id),
		os.O_CREATE|os.O_TRUNC|os.O_APPEND|os.O_WRONLY, 0600,
	)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Hooks.Add(NewLogToFileHook(logFile))

	if *bench {
		logger.Out = ioutil.Discard
		grpc.EnableTracing = false
	}

	grpclog.SetLogger(logger)

	lis, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		logger.Fatal(err)
	}

	grpcServer := grpc.NewServer(grpc.MaxMsgSize(*maxgrpc))

	if *serverMetrics {
		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(fmt.Sprintf(":590%d", *id), nil))
		}()
	}

	lat := raft.NewLatency()
	event := raft.NewEvent()

	var once sync.Once
	writeData := func() {
		lat.Write(fmt.Sprintf("./latency-%v.csv", time.Now().UnixNano()))
		event.Write(fmt.Sprintf("./event-%v.csv", time.Now().UnixNano()))
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		event.Record(raft.EventTerminated)
		once.Do(writeData)
		os.Exit(1)
	}()
	defer func() {
		once.Do(writeData)
	}()

	switch *backend {
	case bgorums:
		rungorums(logger, lis, grpcServer, *id, ids, nodes, lat, event)
	case betcd:
		runetcd(logger, lis, grpcServer, *id, ids, nodes, lat, event)
	case bhashicorp:
		runhashicorp(logger, lis, grpcServer, *id, ids, nodes, lat, event)
	}

}

func runhashicorp(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
	lat *raft.Latency, event *raft.Event,
) {
	servers := make([]hashic.Server, len(nodes))
	for i, addr := range nodes {
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)
		addr = host + ":" + strconv.Itoa(p-100)

		suffrage := hashic.Voter

		if !contains(uint64(i+1), ids) {
			suffrage = hashic.Nonvoter
		}

		servers[i] = hashic.Server{
			Suffrage: suffrage,
			ID:       hashic.ServerID(addr),
			Address:  hashic.ServerAddress(addr),
		}
	}

	addr, err := net.ResolveTCPAddr("tcp", string(servers[id-1].Address))
	if err != nil {
		logger.Fatal(err)
	}
	trans, err := hashic.NewTCPTransport(string(servers[id-1].Address), addr, len(nodes)+1, 10*time.Second, os.Stderr)
	if err != nil {
		logger.Fatal(err)
	}

	path := fmt.Sprintf("hashicorp%.2d.bolt", id)
	overwrite := !*recover
	// Check if file already exists.
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			// We don't need to overwrite a file that doesn't exist.
			overwrite = false
		} else {
			// If we are unable to verify the existence of the file,
			// there is probably a permission problem.
			logger.Fatal(err)
		}
	}
	if overwrite {
		if err := os.Remove(path); err != nil {
			logger.Fatal(err)
		}
	}
	logs, err := raftboltdb.NewBoltStore(path)
	if err != nil {
		logger.Fatal(err)
	}

	snaps := hashic.NewInmemSnapshotStore()

	cfg := &hashic.Config{
		LocalID:            servers[id-1].ID,
		ProtocolVersion:    hashic.ProtocolVersionMax,
		HeartbeatTimeout:   *electionTimeout,
		ElectionTimeout:    *electionTimeout,
		CommitTimeout:      *heartbeatTimeout,
		MaxAppendEntries:   int(*entriesPerMsg),
		ShutdownOnRemove:   true,
		TrailingLogs:       math.MaxUint64,
		SnapshotInterval:   120 * time.Hour,
		SnapshotThreshold:  math.MaxUint64,
		LeaderLeaseTimeout: *electionTimeout / 2,
	}

	node := hraft.NewRaft(logger, NewStore(), cfg, servers, trans, logs, logs, snaps, ids, lat, event)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	logger.Fatal(grpcServer.Serve(lis))
}

func runetcd(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
	lat *raft.Latency, event *raft.Event,
) {
	peers := make([]etcdraft.Peer, len(ids))

	for i, nid := range ids {
		addr := nodes[i]
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)

		ur, err := url.Parse("http://" + addr)
		ur.Host = host + ":" + strconv.Itoa(p-100)
		if err != nil {
			logger.Fatal(err)
		}

		peers[i] = etcdraft.Peer{
			ID:      nid,
			Context: []byte(ur.String()),
		}
	}

	dir := fmt.Sprintf("etcdwal%.2d", id)

	switch {
	case wal.Exist(dir) && !*recover:
		if err := os.RemoveAll(dir); err != nil {
			logger.Fatal(err)
		}
		fallthrough
	case !wal.Exist(dir):
		if err := os.Mkdir(dir, 0750); err != nil {
			logger.Fatalf("rkvd: cannot create dir for wal (%v)", err)
		}

		w, err := wal.Create(dir, nil)
		if err != nil {
			logger.Fatalf("rkvd: create wal error (%v)", err)
		}
		w.Close()
	}

	walsnap := walpb.Snapshot{}
	w, err := wal.Open(dir, walsnap)
	if err != nil {
		logger.Fatalf("rkvd: error loading wal (%v)", err)
	}

	_, st, ents, err := w.ReadAll()
	if err != nil {
		log.Fatalf("rkvd: failed to read WAL (%v)", err)
	}

	storage := etcdraft.NewMemoryStorage()
	storage.SetHardState(st)
	storage.Append(ents)

	node := etcd.NewRaft(
		logger,
		NewStore(),
		storage,
		w,
		&etcdraft.Config{
			ID:            id,
			ElectionTick:  int(*electionTimeout / *heartbeatTimeout),
			HeartbeatTick: 1,
			Storage:       storage,
			MaxSizePerMsg: *entriesPerMsg,
			// etcdserver says: Never overflow the rafthttp buffer,
			// which is 4096. We keep the same constant.
			MaxInflightMsgs: 4096 / 8,
			CheckQuorum:     true,
			PreVote:         true,
			Logger:          logger,
		},
		peers,
		*heartbeatTimeout,
		!contains(id, ids),
		nodes,
		lat, event,
	)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	host, port, err := net.SplitHostPort(nodes[id-1])
	if err != nil {
		logger.Fatal(err)
	}
	p, _ := strconv.Atoi(port)
	selflis := host + ":" + strconv.Itoa(p-100)

	lishttp, err := net.Listen("tcp", selflis)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Fatal(http.Serve(lishttp, node.Handler()))
}

func rungorums(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
	lat *raft.Latency, event *raft.Event,
) {
	storage, err := raft.NewFileStorage(fmt.Sprintf("db%.2d.bolt", id), !*recover)

	if err != nil {
		logger.Fatal(err)
	}

	storageWithCache := raft.NewCacheStorage(storage, *cache)

	node := raftgorums.NewRaft(NewStore(), &raftgorums.Config{
		ID:                id,
		Servers:           nodes,
		InitialCluster:    ids,
		Batch:             *batch,
		Storage:           storageWithCache,
		ElectionTimeout:   *electionTimeout,
		HeartbeatTimeout:  *heartbeatTimeout,
		EntriesPerMsg:     *entriesPerMsg,
		CatchupMultiplier: *catchupMultiplier,
		Logger:            logger,
		MetricsEnabled:    true,
	}, lat, event)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	logger.Fatal(node.Run(grpcServer))
}

func contains(x uint64, xs []uint64) bool {
	for _, y := range xs {
		if y == x {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2017 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package signer

import (
	"bytes"
	"encoding/hex"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/minio/minio-go/v7/pkg/s3utils"
)

// Signature and API related constants.
const (
	signV4Algorithm   = "AWS4-HMAC-SHA256"
	iso8601DateFormat = "20060102T150405Z"
	yyyymmdd          = "20060102"
)

// Different service types
const (
	ServiceTypeS3  = "s3"
	ServiceTypeSTS = "sts"
)

///
/// Excerpts from @lsegal -
/// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258.
///
///  User-Agent:
///
///      This is ignored from signing because signing this causes
///      problems with generating pre-signed URLs (that are executed
///      by other agents) or when customers pass requests through
///      proxies, which may modify the user-agent.
///
///
///  Authorization:
///
///      Is skipped for obvious reasons
///
var v4IgnoredHeaders = map[string]bool{
	"Authorization": true,
	"User-Agent":    true,
}

// getSigningKey hmac seed to calculate final signature.
func getSigningKey(secret, loc string, t time.Time, serviceType string) []byte {
	date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
	location := sumHMAC(date, []byte(loc))
	service := sumHMAC(location, []byte(serviceType))
	signingKey := sumHMAC(service, []byte("aws4_request"))
	return signingKey
}

// getSignature final signature in hexadecimal form.
func getSignature(signingKey []byte, stringToSign string) string {
	return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
}

// getScope generate a string of a specific date, an AWS region, and a
// service.
func getScope(location string, t time.Time, serviceType string) string {
	scope := strings.Join([]string{
		t.Format(yyyymmdd),
		location,
		serviceType,
		"aws4_request",
	}, "/")
	return scope
}

// GetCredential generate a credential string.
func GetCredential(accessKeyID, location string, t time.Time, serviceType string) string {
	scope := getScope(location, t, serviceType)
	return accessKeyID + "/" + scope
}

// getHashedPayload get the hexadecimal value of the SHA256 hash of
// the request payload.
func getHashedPayload(req http.Request) string {
	hashedPayload := req.Header.Get("X-Amz-Content-Sha256")
	if hashedPayload == "" {
		// Presign does not have a payload, use S3 recommended value.
		hashedPayload = unsignedPayload
	}
	return hashedPayload
}

// getCanonicalHeaders generate a list of request headers for
// signature.
func getCanonicalHeaders(req http.Request, ignoredHeaders map[string]bool) string {
	var headers []string
	vals := make(map[string][]string)
	for k, vv := range req.Header {
		if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
			continue // ignored header
		}
		headers = append(headers, strings.ToLower(k))
		vals[strings.ToLower(k)] = vv
	}
	headers = append(headers, "host")
	sort.Strings(headers)

	var buf bytes.Buffer
	// Save all the headers in canonical form <header>:<value> newline
	// separated for each header.
	for _, k := range headers {
		buf.WriteString(k)
		buf.WriteByte(':')
		switch {
		case k == "host":
			buf.WriteString(getHostAddr(&req))
			fallthrough
		default:
			for idx, v := range vals[k] {
				if idx > 0 {
					buf.WriteByte(',')
				}
				buf.WriteString(signV4TrimAll(v))
			}
			buf.WriteByte('\n')
		}
	}
	return buf.String()
}

// getSignedHeaders generate all signed request headers.
// i.e lexically sorted, semicolon-separated list of lowercase
// request header names.
func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) string {
	var headers []string
	for k := range req.Header {
		if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
			continue // Ignored header found continue.
		}
		headers = append(headers, strings.ToLower(k))
	}
	headers = append(headers, "host")
	sort.Strings(headers)
	return strings.Join(headers, ";")
}

// getCanonicalRequest generate a canonical request of style.
//
// canonicalRequest =
//  <HTTPMethod>\n
//  <CanonicalURI>\n
//  <CanonicalQueryString>\n
//  <CanonicalHeaders>\n
//  <SignedHeaders>\n
//  <HashedPayload>
func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool, hashedPayload string) string {
	req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
	canonicalRequest := strings.Join([]string{
		req.Method,
		s3utils.EncodePath(req.URL.Path),
		req.URL.RawQuery,
		getCanonicalHeaders(req, ignoredHeaders),
		getSignedHeaders(req, ignoredHeaders),
		hashedPayload,
	}, "\n")
	return canonicalRequest
}

// getStringToSign a string based on selected query values.
func getStringToSignV4(t time.Time, location, canonicalRequest, serviceType string) string {
	stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n"
	stringToSign = stringToSign + getScope(location, t, serviceType) + "\n"
	stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest)))
	return stringToSign
}

// PreSignV4 presign the request, in accordance with
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html.
func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, expires int64) *http.Request {
	// Presign is not needed for anonymous credentials.
	if accessKeyID == "" || secretAccessKey == "" {
		return &req
	}

	// Initial time.
	t := time.Now().UTC()

	// Get credential string.
	credential := GetCredential(accessKeyID, location, t, ServiceTypeS3)

	// Get all signed headers.
	signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)

	// Set URL query.
	query := req.URL.Query()
	query.Set("X-Amz-Algorithm", signV4Algorithm)
	query.Set("X-Amz-Date", t.Format(iso8601DateFormat))
	query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
	query.Set("X-Amz-SignedHeaders", signedHeaders)
	query.Set("X-Amz-Credential", credential)
	// Set session token if available.
	if sessionToken != "" {
		query.Set("X-Amz-Security-Token", sessionToken)
	}
	req.URL.RawQuery = query.Encode()

	// Get canonical request.
	canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, getHashedPayload(req))

	// Get string to sign from canonical request.
	stringToSign := getStringToSignV4(t, location, canonicalRequest, ServiceTypeS3)

	// Gext hmac signing key.
	signingKey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3)

	// Calculate signature.
	signature := getSignature(signingKey, stringToSign)

	// Add signature header to RawQuery.
	req.URL.RawQuery += "&X-Amz-Signature=" + signature

	return &req
}

// PostPresignSignatureV4 - presigned signature for PostPolicy
// requests.
func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
	// Get signining key.
	signingkey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3)
	// Calculate signature.
	signature := getSignature(signingkey, policyBase64)
	return signature
}

// SignV4STS - signature v4 for STS request.
func SignV4STS(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request {
	return signV4(req, accessKeyID, secretAccessKey, "", location, ServiceTypeSTS)
}

// Internal function called for different service types.
func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string) *http.Request {
	// Signature calculation is not needed for anonymous credentials.
	if accessKeyID == "" || secretAccessKey == "" {
		return &req
	}

	// Initial time.
	t := time.Now().UTC()

	// Set x-amz-date.
	req.Header.Set("X-Amz-Date", t.Format(iso8601DateFormat))

	// Set session token if available.
	if sessionToken != "" {
		req.Header.Set("X-Amz-Security-Token", sessionToken)
	}

	hashedPayload := getHashedPayload(req)
	if serviceType == ServiceTypeSTS {
		// Content sha256 header is not sent with the request
		// but it is expected to have sha256 of payload for signature
		// in STS service type request.
		req.Header.Del("X-Amz-Content-Sha256")
	}

	// Get canonical request.
	canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, hashedPayload)

	// Get string to sign from canonical request.
	stringToSign := getStringToSignV4(t, location, canonicalRequest, serviceType)

	// Get hmac signing key.
	signingKey := getSigningKey(secretAccessKey, location, t, serviceType)

	// Get credential string.
	credential := GetCredential(accessKeyID, location, t, serviceType)

	// Get all signed headers.
	signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)

	// Calculate signature.
	signature := getSignature(signingKey, stringToSign)

	// If regular request, construct the final authorization header.
	parts := []string{
		signV4Algorithm + " Credential=" + credential,
		"SignedHeaders=" + signedHeaders,
		"Signature=" + signature,
	}

	// Set authorization header.
	auth := strings.Join(parts, ", ")
	req.Header.Set("Authorization", auth)

	return &req
}

// SignV4 sign the request before Do(), in accordance with
// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html.
func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request {
	return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3)
}
<commit_msg>add custom Host header support (#1579)<commit_after>/*
 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2017 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package signer

import (
	"bytes"
	"encoding/hex"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/minio/minio-go/v7/pkg/s3utils"
)

// Signature and API related constants.
const (
	signV4Algorithm   = "AWS4-HMAC-SHA256"
	iso8601DateFormat = "20060102T150405Z"
	yyyymmdd          = "20060102"
)

// Different service types
const (
	ServiceTypeS3  = "s3"
	ServiceTypeSTS = "sts"
)

///
/// Excerpts from @lsegal -
/// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258.
///
///  User-Agent:
///
///      This is ignored from signing because signing this causes
///      problems with generating pre-signed URLs (that are executed
///      by other agents) or when customers pass requests through
///      proxies, which may modify the user-agent.
///
///
///  Authorization:
///
///      Is skipped for obvious reasons
///
var v4IgnoredHeaders = map[string]bool{
	"Authorization": true,
	"User-Agent":    true,
}

// getSigningKey hmac seed to calculate final signature.
func getSigningKey(secret, loc string, t time.Time, serviceType string) []byte {
	date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
	location := sumHMAC(date, []byte(loc))
	service := sumHMAC(location, []byte(serviceType))
	signingKey := sumHMAC(service, []byte("aws4_request"))
	return signingKey
}

// getSignature final signature in hexadecimal form.
func getSignature(signingKey []byte, stringToSign string) string {
	return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
}

// getScope generate a string of a specific date, an AWS region, and a
// service.
func getScope(location string, t time.Time, serviceType string) string {
	scope := strings.Join([]string{
		t.Format(yyyymmdd),
		location,
		serviceType,
		"aws4_request",
	}, "/")
	return scope
}

// GetCredential generate a credential string.
func GetCredential(accessKeyID, location string, t time.Time, serviceType string) string {
	scope := getScope(location, t, serviceType)
	return accessKeyID + "/" + scope
}

// getHashedPayload get the hexadecimal value of the SHA256 hash of
// the request payload.
func getHashedPayload(req http.Request) string {
	hashedPayload := req.Header.Get("X-Amz-Content-Sha256")
	if hashedPayload == "" {
		// Presign does not have a payload, use S3 recommended value.
		hashedPayload = unsignedPayload
	}
	return hashedPayload
}

// getCanonicalHeaders generate a list of request headers for
// signature.
func getCanonicalHeaders(req http.Request, ignoredHeaders map[string]bool) string {
	var headers []string
	vals := make(map[string][]string)
	for k, vv := range req.Header {
		if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
			continue // ignored header
		}
		headers = append(headers, strings.ToLower(k))
		vals[strings.ToLower(k)] = vv
	}
	if !headerExists("host", headers) {
		headers = append(headers, "host")
	}
	sort.Strings(headers)

	var buf bytes.Buffer
	// Save all the headers in canonical form <header>:<value> newline
	// separated for each header.
	for _, k := range headers {
		buf.WriteString(k)
		buf.WriteByte(':')
		switch {
		case k == "host":
			buf.WriteString(getHostAddr(&req))
			buf.WriteByte('\n')
		default:
			for idx, v := range vals[k] {
				if idx > 0 {
					buf.WriteByte(',')
				}
				buf.WriteString(signV4TrimAll(v))
			}
			buf.WriteByte('\n')
		}
	}
	return buf.String()
}

func headerExists(key string, headers []string) bool {
	for _, k := range headers {
		if k == key {
			return true
		}
	}
	return false
}

// getSignedHeaders generate all signed request headers.
// i.e lexically sorted, semicolon-separated list of lowercase
// request header names.
func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) string {
	var headers []string
	for k := range req.Header {
		if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
			continue // Ignored header found continue.
		}
		headers = append(headers, strings.ToLower(k))
	}
	if !headerExists("host", headers) {
		headers = append(headers, "host")
	}
	sort.Strings(headers)
	return strings.Join(headers, ";")
}

// getCanonicalRequest generate a canonical request of style.
//
// canonicalRequest =
//  <HTTPMethod>\n
//  <CanonicalURI>\n
//  <CanonicalQueryString>\n
//  <CanonicalHeaders>\n
//  <SignedHeaders>\n
//  <HashedPayload>
func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool, hashedPayload string) string {
	req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
	canonicalRequest := strings.Join([]string{
		req.Method,
		s3utils.EncodePath(req.URL.Path),
		req.URL.RawQuery,
		getCanonicalHeaders(req, ignoredHeaders),
		getSignedHeaders(req, ignoredHeaders),
		hashedPayload,
	}, "\n")
	return canonicalRequest
}

// getStringToSign a string based on selected query values.
func getStringToSignV4(t time.Time, location, canonicalRequest, serviceType string) string {
	stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n"
	stringToSign = stringToSign + getScope(location, t, serviceType) + "\n"
	stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest)))
	return stringToSign
}

// PreSignV4 presign the request, in accordance with
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html.
func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, expires int64) *http.Request {
	// Presign is not needed for anonymous credentials.
	if accessKeyID == "" || secretAccessKey == "" {
		return &req
	}

	// Initial time.
	t := time.Now().UTC()

	// Get credential string.
	credential := GetCredential(accessKeyID, location, t, ServiceTypeS3)

	// Get all signed headers.
	signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)

	// Set URL query.
	query := req.URL.Query()
	query.Set("X-Amz-Algorithm", signV4Algorithm)
	query.Set("X-Amz-Date", t.Format(iso8601DateFormat))
	query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
	query.Set("X-Amz-SignedHeaders", signedHeaders)
	query.Set("X-Amz-Credential", credential)
	// Set session token if available.
	if sessionToken != "" {
		query.Set("X-Amz-Security-Token", sessionToken)
	}
	req.URL.RawQuery = query.Encode()

	// Get canonical request.
	canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, getHashedPayload(req))

	// Get string to sign from canonical request.
	stringToSign := getStringToSignV4(t, location, canonicalRequest, ServiceTypeS3)

	// Gext hmac signing key.
	signingKey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3)

	// Calculate signature.
	signature := getSignature(signingKey, stringToSign)

	// Add signature header to RawQuery.
	req.URL.RawQuery += "&X-Amz-Signature=" + signature

	return &req
}

// PostPresignSignatureV4 - presigned signature for PostPolicy
// requests.
func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
	// Get signining key.
	signingkey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3)
	// Calculate signature.
	signature := getSignature(signingkey, policyBase64)
	return signature
}

// SignV4STS - signature v4 for STS request.
func SignV4STS(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request {
	return signV4(req, accessKeyID, secretAccessKey, "", location, ServiceTypeSTS)
}

// Internal function called for different service types.
func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string) *http.Request {
	// Signature calculation is not needed for anonymous credentials.
	if accessKeyID == "" || secretAccessKey == "" {
		return &req
	}

	// Initial time.
	t := time.Now().UTC()

	// Set x-amz-date.
	req.Header.Set("X-Amz-Date", t.Format(iso8601DateFormat))

	// Set session token if available.
	if sessionToken != "" {
		req.Header.Set("X-Amz-Security-Token", sessionToken)
	}

	hashedPayload := getHashedPayload(req)
	if serviceType == ServiceTypeSTS {
		// Content sha256 header is not sent with the request
		// but it is expected to have sha256 of payload for signature
		// in STS service type request.
		req.Header.Del("X-Amz-Content-Sha256")
	}

	// Get canonical request.
	canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, hashedPayload)

	// Get string to sign from canonical request.
	stringToSign := getStringToSignV4(t, location, canonicalRequest, serviceType)

	// Get hmac signing key.
	signingKey := getSigningKey(secretAccessKey, location, t, serviceType)

	// Get credential string.
	credential := GetCredential(accessKeyID, location, t, serviceType)

	// Get all signed headers.
	signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)

	// Calculate signature.
	signature := getSignature(signingKey, stringToSign)

	// If regular request, construct the final authorization header.
	parts := []string{
		signV4Algorithm + " Credential=" + credential,
		"SignedHeaders=" + signedHeaders,
		"Signature=" + signature,
	}

	// Set authorization header.
	auth := strings.Join(parts, ", ")
	req.Header.Set("Authorization", auth)

	return &req
}

// SignV4 sign the request before Do(), in accordance with
// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html.
func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request {
	return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3)
}
<|endoftext|>


<commit_before>package opentsdb

import (
	"testing"

	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/tsdb"
	. "github.com/smartystreets/goconvey/convey"
)

func TestOpenTsdbExecutor(t *testing.T) {
	Convey("OpenTsdb query testing", t, func() {
		exec := &OpenTsdbExecutor{}

		Convey("Build metric with downsampling enabled", func() {
			query := &tsdb.Query{
				Model: simplejson.New(),
			}

			query.Model.Set("metric", "cpu.average.percent")
			query.Model.Set("aggregator", "avg")
			query.Model.Set("disableDownsampling", false)
			query.Model.Set("downsampleInterval", "")
			query.Model.Set("downsampleAggregator", "avg")
			query.Model.Set("downsampleFillPolicy", "none")

			metric := exec.buildMetric(query)

			So(len(metric), ShouldEqual, 3)
			So(metric["metric"], ShouldEqual, "cpu.average.percent")
			So(metric["aggregator"], ShouldEqual, "avg")
			So(metric["downsample"], ShouldEqual, "1m-avg")
		})

		Convey("Build metric with downsampling disabled", func() {
			query := &tsdb.Query{
				Model: simplejson.New(),
			}

			query.Model.Set("metric", "cpu.average.percent")
			query.Model.Set("aggregator", "avg")
			query.Model.Set("disableDownsampling", true)
			query.Model.Set("downsampleInterval", "")
			query.Model.Set("downsampleAggregator", "avg")
			query.Model.Set("downsampleFillPolicy", "none")

			metric := exec.buildMetric(query)

			So(len(metric), ShouldEqual, 2)
			So(metric["metric"], ShouldEqual, "cpu.average.percent")
			So(metric["aggregator"], ShouldEqual, "avg")
		})

		Convey("Build metric with downsampling enabled with params", func() {
			query := &tsdb.Query{
				Model: simplejson.New(),
			}

			query.Model.Set("metric", "cpu.average.percent")
			query.Model.Set("aggregator", "avg")
			query.Model.Set("disableDownsampling", false)
			query.Model.Set("downsampleInterval", "5m")
			query.Model.Set("downsampleAggregator", "sum")
			query.Model.Set("downsampleFillPolicy", "null")

			metric := exec.buildMetric(query)

			So(len(metric), ShouldEqual, 3)
			So(metric["metric"], ShouldEqual, "cpu.average.percent")
			So(metric["aggregator"], ShouldEqual, "avg")
			So(metric["downsample"], ShouldEqual, "5m-sum-null")
		})

		Convey("Build metric with tags with downsampling disabled", func() {
			query := &tsdb.Query{
				Model: simplejson.New(),
			}

			query.Model.Set("metric", "cpu.average.percent")
			query.Model.Set("aggregator", "avg")
			query.Model.Set("disableDownsampling", true)
			query.Model.Set("downsampleInterval", "5m")
			query.Model.Set("downsampleAggregator", "sum")
			query.Model.Set("downsampleFillPolicy", "null")

			tags := simplejson.New()
			tags.Set("env", "prod")
			tags.Set("app", "grafana")
			query.Model.Set("tags", tags.MustMap())

			metric := exec.buildMetric(query)

			So(len(metric), ShouldEqual, 3)
			So(metric["metric"], ShouldEqual, "cpu.average.percent")
			So(metric["aggregator"], ShouldEqual, "avg")
			So(metric["downsample"], ShouldEqual, nil)
			So(len(metric["tags"].(map[string]interface{})), ShouldEqual, 2)
			So(metric["tags"].(map[string]interface{})["env"], ShouldEqual, "prod")
			So(metric["tags"].(map[string]interface{})["app"], ShouldEqual, "grafana")
			So(metric["tags"].(map[string]interface{})["ip"], ShouldEqual, nil)
		})

		Convey("Build metric with rate enabled but counter disabled", func() {
			query := &tsdb.Query{
				Model: simplejson.New(),
			}

			query.Model.Set("metric", "cpu.average.percent")
			query.Model.Set("aggregator", "avg")
			query.Model.Set("disableDownsampling", true)
			query.Model.Set("shouldComputeRate", true)
			query.Model.Set("isCounter", false)

			tags := simplejson.New()
			tags.Set("env", "prod")
			tags.Set("app", "grafana")
			query.Model.Set("tags", tags.MustMap())

			metric := exec.buildMetric(query)

			So(len(metric), ShouldEqual, 5)
			So(metric["metric"], ShouldEqual, "cpu.average.percent")
			So(metric["aggregator"], ShouldEqual, "avg")
			So(len(metric["tags"].(map[string]interface{})), ShouldEqual, 2)
			So(metric["tags"].(map[string]interface{})["env"], ShouldEqual, "prod")
			So(metric["tags"].(map[string]interface{})["app"], ShouldEqual, "grafana")
			So(metric["tags"].(map[string]interface{})["ip"], ShouldEqual, nil)
			So(metric["rate"], ShouldEqual, true)
			So(metric["rateOptions"].(map[string]interface{})["counter"], ShouldEqual, false)
		})

		Convey("Build metric with rate and counter enabled", func() {
			query := &tsdb.Query{
				Model: simplejson.New(),
			}

			query.Model.Set("metric", "cpu.average.percent")
			query.Model.Set("aggregator", "avg")
			query.Model.Set("disableDownsampling", true)
			query.Model.Set("shouldComputeRate", true)
			query.Model.Set("isCounter", true)
			query.Model.Set("counterMax", 45)
			query.Model.Set("counterResetValue", 60)

			tags := simplejson.New()
			tags.Set("env", "prod")
			tags.Set("app", "grafana")
			query.Model.Set("tags", tags.MustMap())

			metric := exec.buildMetric(query)

			So(len(metric), ShouldEqual, 5)
			So(metric["metric"], ShouldEqual, "cpu.average.percent")
			So(metric["aggregator"], ShouldEqual, "avg")
			So(len(metric["tags"].(map[string]interface{})), ShouldEqual, 2)
			So(metric["tags"].(map[string]interface{})["env"], ShouldEqual, "prod")
			So(metric["tags"].(map[string]interface{})["app"], ShouldEqual, "grafana")
			So(metric["tags"].(map[string]interface{})["ip"], ShouldEqual, nil)
			So(metric["rate"], ShouldEqual, true)
			So(len(metric["rateOptions"].(map[string]interface{})), ShouldEqual, 3)
			So(metric["rateOptions"].(map[string]interface{})["counter"], ShouldEqual, true)
			So(metric["rateOptions"].(map[string]interface{})["counterMax"], ShouldEqual, 45)
			So(metric["rateOptions"].(map[string]interface{})["resetValue"], ShouldEqual, 60)
		})
	})
}
<commit_msg>Chore: Rewrite opentsdb test to standard library (#29792)<commit_after>package opentsdb

import (
	"testing"

	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/tsdb"
	"github.com/stretchr/testify/require"
)

func TestOpenTsdbExecutor(t *testing.T) {
	exec := &OpenTsdbExecutor{}

	t.Run("Build metric with downsampling enabled", func(t *testing.T) {
		query := &tsdb.Query{
			Model: simplejson.New(),
		}

		query.Model.Set("metric", "cpu.average.percent")
		query.Model.Set("aggregator", "avg")
		query.Model.Set("disableDownsampling", false)
		query.Model.Set("downsampleInterval", "")
		query.Model.Set("downsampleAggregator", "avg")
		query.Model.Set("downsampleFillPolicy", "none")

		metric := exec.buildMetric(query)

		require.Len(t, metric, 3)
		require.Equal(t, "cpu.average.percent", metric["metric"])
		require.Equal(t, "avg", metric["aggregator"])
		require.Equal(t, "1m-avg", metric["downsample"])
	})

	t.Run("Build metric with downsampling disabled", func(t *testing.T) {
		query := &tsdb.Query{
			Model: simplejson.New(),
		}

		query.Model.Set("metric", "cpu.average.percent")
		query.Model.Set("aggregator", "avg")
		query.Model.Set("disableDownsampling", true)
		query.Model.Set("downsampleInterval", "")
		query.Model.Set("downsampleAggregator", "avg")
		query.Model.Set("downsampleFillPolicy", "none")

		metric := exec.buildMetric(query)

		require.Len(t, metric, 2)
		require.Equal(t, "cpu.average.percent", metric["metric"])
		require.Equal(t, "avg", metric["aggregator"])
	})

	t.Run("Build metric with downsampling enabled with params", func(t *testing.T) {
		query := &tsdb.Query{
			Model: simplejson.New(),
		}

		query.Model.Set("metric", "cpu.average.percent")
		query.Model.Set("aggregator", "avg")
		query.Model.Set("disableDownsampling", false)
		query.Model.Set("downsampleInterval", "5m")
		query.Model.Set("downsampleAggregator", "sum")
		query.Model.Set("downsampleFillPolicy", "null")

		metric := exec.buildMetric(query)

		require.Len(t, metric, 3)
		require.Equal(t, "cpu.average.percent", metric["metric"])
		require.Equal(t, "avg", metric["aggregator"])
		require.Equal(t, "5m-sum-null", metric["downsample"])
	})

	t.Run("Build metric with tags with downsampling disabled", func(t *testing.T) {
		query := &tsdb.Query{
			Model: simplejson.New(),
		}

		query.Model.Set("metric", "cpu.average.percent")
		query.Model.Set("aggregator", "avg")
		query.Model.Set("disableDownsampling", true)
		query.Model.Set("downsampleInterval", "5m")
		query.Model.Set("downsampleAggregator", "sum")
		query.Model.Set("downsampleFillPolicy", "null")

		tags := simplejson.New()
		tags.Set("env", "prod")
		tags.Set("app", "grafana")
		query.Model.Set("tags", tags.MustMap())

		metric := exec.buildMetric(query)

		require.Len(t, metric, 3)
		require.Equal(t, "cpu.average.percent", metric["metric"])
		require.Equal(t, "avg", metric["aggregator"])
		require.Nil(t, metric["downsample"])

		metricTags := metric["tags"].(map[string]interface{})
		require.Len(t, metricTags, 2)
		require.Equal(t, "prod", metricTags["env"])
		require.Equal(t, "grafana", metricTags["app"])
		require.Nil(t, metricTags["ip"])
	})

	t.Run("Build metric with rate enabled but counter disabled", func(t *testing.T) {
		query := &tsdb.Query{
			Model: simplejson.New(),
		}

		query.Model.Set("metric", "cpu.average.percent")
		query.Model.Set("aggregator", "avg")
		query.Model.Set("disableDownsampling", true)
		query.Model.Set("shouldComputeRate", true)
		query.Model.Set("isCounter", false)

		tags := simplejson.New()
		tags.Set("env", "prod")
		tags.Set("app", "grafana")
		query.Model.Set("tags", tags.MustMap())

		metric := exec.buildMetric(query)

		require.Len(t, metric, 5)
		require.Equal(t, "cpu.average.percent", metric["metric"])
		require.Equal(t, "avg", metric["aggregator"])

		metricTags := metric["tags"].(map[string]interface{})
		require.Len(t, metricTags, 2)
		require.Equal(t, "prod", metricTags["env"])
		require.Equal(t, "grafana", metricTags["app"])
		require.Nil(t, metricTags["ip"])

		require.True(t, metric["rate"].(bool))
		require.False(t, metric["rateOptions"].(map[string]interface{})["counter"].(bool))
	})

	t.Run("Build metric with rate and counter enabled", func(t *testing.T) {
		query := &tsdb.Query{
			Model: simplejson.New(),
		}

		query.Model.Set("metric", "cpu.average.percent")
		query.Model.Set("aggregator", "avg")
		query.Model.Set("disableDownsampling", true)
		query.Model.Set("shouldComputeRate", true)
		query.Model.Set("isCounter", true)
		query.Model.Set("counterMax", 45)
		query.Model.Set("counterResetValue", 60)

		tags := simplejson.New()
		tags.Set("env", "prod")
		tags.Set("app", "grafana")
		query.Model.Set("tags", tags.MustMap())

		metric := exec.buildMetric(query)

		require.Len(t, metric, 5)
		require.Equal(t, "cpu.average.percent", metric["metric"])
		require.Equal(t, "avg", metric["aggregator"])

		metricTags := metric["tags"].(map[string]interface{})
		require.Len(t, metricTags, 2)
		require.Equal(t, "prod", metricTags["env"])
		require.Equal(t, "grafana", metricTags["app"])
		require.Nil(t, metricTags["ip"])

		require.True(t, metric["rate"].(bool))
		metricRateOptions := metric["rateOptions"].(map[string]interface{})
		require.Len(t, metricRateOptions, 3)
		require.True(t, metricRateOptions["counter"].(bool))
		require.Equal(t, float64(45), metricRateOptions["counterMax"])
		require.Equal(t, float64(60), metricRateOptions["resetValue"])
	})
}
<|endoftext|>


<commit_before>package webircgateway

import (
	"errors"
	"math/rand"
	"net"
	"net/http"
	"strings"
)

func (s *Gateway) NewClient() *Client {
	return NewClient(s)
}

func (s *Gateway) IsClientOriginAllowed(originHeader string) bool {
	// Empty list of origins = all origins allowed
	if len(s.Config.RemoteOrigins) == 0 {
		return true
	}

	// No origin header = running on the same page
	if originHeader == "" {
		return true
	}

	foundMatch := false

	for _, originMatch := range s.Config.RemoteOrigins {
		if originMatch.Match(originHeader) {
			foundMatch = true
			break
		}
	}

	return foundMatch
}

func (s *Gateway) isIrcAddressAllowed(addr string) bool {
	// Empty whitelist = all destinations allowed
	if len(s.Config.GatewayWhitelist) == 0 {
		return true
	}

	foundMatch := false

	for _, addrMatch := range s.Config.GatewayWhitelist {
		if addrMatch.Match(addr) {
			foundMatch = true
			break
		}
	}

	return foundMatch
}

func (s *Gateway) findUpstream() (ConfigUpstream, error) {
	var ret ConfigUpstream

	if len(s.Config.Upstreams) == 0 {
		return ret, errors.New("No upstreams available")
	}

	randIdx := rand.Intn(len(s.Config.Upstreams))
	ret = s.Config.Upstreams[randIdx]

	return ret, nil
}

func (s *Gateway) findWebircPassword(ircHost string) string {
	pass, exists := s.Config.GatewayWebircPassword[strings.ToLower(ircHost)]
	if !exists {
		pass = ""
	}

	return pass
}

func (s *Gateway) GetRemoteAddressFromRequest(req *http.Request) net.IP {
	remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)

	// Some web listeners such as unix sockets don't get a RemoteAddr, so default to localhost
	if remoteAddr == "" {
		remoteAddr = "127.0.0.1"
	}

	remoteIP := net.ParseIP(remoteAddr)

	isInRange := false
	for _, cidrRange := range s.Config.ReverseProxies {
		if cidrRange.Contains(remoteIP) {
			isInRange = true
			break
		}
	}

	// If the remoteIP is not in a whitelisted reverse proxy range, don't trust
	// the headers and use the remoteIP as the users IP
	if !isInRange {
		return remoteIP
	}

	headerVal := req.Header.Get("x-forwarded-for")
	ips := strings.Split(headerVal, ",")
	ipStr := strings.Trim(ips[0], " ")
	if ipStr != "" {
		ip := net.ParseIP(ipStr)
		if ip != nil {
			remoteIP = ip
		}
	}

	return remoteIP

}

func (s *Gateway) isRequestSecure(req *http.Request) bool {
	remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
	remoteIP := net.ParseIP(remoteAddr)

	isInRange := false
	for _, cidrRange := range s.Config.ReverseProxies {
		if cidrRange.Contains(remoteIP) {
			isInRange = true
			break
		}
	}

	// If the remoteIP is not in a whitelisted reverse proxy range, don't trust
	// the headers and check the request directly
	if !isInRange && req.TLS == nil {
		return false
	} else if !isInRange {
		return true
	}

	headerVal := strings.ToLower(req.Header.Get("x-forwarded-proto"))
	if headerVal == "https" {
		return true
	}

	return false
}
<commit_msg>fix secure flag for unix domain sockets (#65)<commit_after>package webircgateway

import (
	"errors"
	"math/rand"
	"net"
	"net/http"
	"strings"
)

var v4LoopbackAddr = net.ParseIP("127.0.0.1")

func (s *Gateway) NewClient() *Client {
	return NewClient(s)
}

func (s *Gateway) IsClientOriginAllowed(originHeader string) bool {
	// Empty list of origins = all origins allowed
	if len(s.Config.RemoteOrigins) == 0 {
		return true
	}

	// No origin header = running on the same page
	if originHeader == "" {
		return true
	}

	foundMatch := false

	for _, originMatch := range s.Config.RemoteOrigins {
		if originMatch.Match(originHeader) {
			foundMatch = true
			break
		}
	}

	return foundMatch
}

func (s *Gateway) isIrcAddressAllowed(addr string) bool {
	// Empty whitelist = all destinations allowed
	if len(s.Config.GatewayWhitelist) == 0 {
		return true
	}

	foundMatch := false

	for _, addrMatch := range s.Config.GatewayWhitelist {
		if addrMatch.Match(addr) {
			foundMatch = true
			break
		}
	}

	return foundMatch
}

func (s *Gateway) findUpstream() (ConfigUpstream, error) {
	var ret ConfigUpstream

	if len(s.Config.Upstreams) == 0 {
		return ret, errors.New("No upstreams available")
	}

	randIdx := rand.Intn(len(s.Config.Upstreams))
	ret = s.Config.Upstreams[randIdx]

	return ret, nil
}

func (s *Gateway) findWebircPassword(ircHost string) string {
	pass, exists := s.Config.GatewayWebircPassword[strings.ToLower(ircHost)]
	if !exists {
		pass = ""
	}

	return pass
}

func (s *Gateway) GetRemoteAddressFromRequest(req *http.Request) net.IP {
	remoteIP := remoteIPFromRequest(req)

	// If the remoteIP is not in a whitelisted reverse proxy range, don't trust
	// the headers and use the remoteIP as the users IP
	if !s.isTrustedProxy(remoteIP) {
		return remoteIP
	}

	headerVal := req.Header.Get("x-forwarded-for")
	ips := strings.Split(headerVal, ",")
	ipStr := strings.Trim(ips[0], " ")
	if ipStr != "" {
		ip := net.ParseIP(ipStr)
		if ip != nil {
			remoteIP = ip
		}
	}

	return remoteIP

}

func (s *Gateway) isRequestSecure(req *http.Request) bool {
	remoteIP := remoteIPFromRequest(req)

	// If the remoteIP is not in a whitelisted reverse proxy range, don't trust
	// the headers and check the request directly
	if !s.isTrustedProxy(remoteIP) {
		return req.TLS != nil
	}

	fwdProto := req.Header.Get("x-forwarded-proto")
	return strings.EqualFold(fwdProto, "https")
}

func (s *Gateway) isTrustedProxy(remoteIP net.IP) bool {
	for _, cidrRange := range s.Config.ReverseProxies {
		if cidrRange.Contains(remoteIP) {
			return true
		}
	}
	return false
}

func remoteIPFromRequest(req *http.Request) net.IP {
	if req.RemoteAddr == "@" {
		// remote address is unix socket, treat it as loopback interface
		return v4LoopbackAddr
	}

	remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
	return net.ParseIP(remoteAddr)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
)

func hwaf_make_cmd_self_init() *commander.Command {
	cmd := &commander.Command{
		Run:       hwaf_run_cmd_self_init,
		UsageLine: "init [options] <workarea>",
		Short:     "initialize hwaf itself",
		Long: `
init initializes hwaf internal files.

ex:
 $ hwaf self init
`,
		Flag: *flag.NewFlagSet("hwaf-self-init", flag.ExitOnError),
	}
	cmd.Flag.Bool("q", true, "only print error and warning messages, all other output will be suppressed")

	return cmd
}

func hwaf_run_cmd_self_init(cmd *commander.Command, args []string) {
	var err error
	n := "hwaf-self-" + cmd.Name()

	switch len(args) {
	case 0:
		// ok
	default:
		err = fmt.Errorf("%s: does NOT take any argument", n)
		handle_err(err)
	}

	quiet := cmd.Flag.Lookup("q").Value.Get().(bool)

	if !quiet {
		fmt.Printf("%s...\n", n)
	}

	home := os.Getenv("HOME")
	if home == "" {
		err = fmt.Errorf("%s: no ${HOME} environment variable", n)
		handle_err(err)
	}

	top := filepath.Join(home, ".config", "hwaf")
	if !path_exists(top) {
		err = os.MkdirAll(top, 0700)
		handle_err(err)
	}

	// add hep-waftools cache
	hwaf_tools := filepath.Join(top, "tools")
	if path_exists(hwaf_tools) {
		err = os.RemoveAll(hwaf_tools)
		handle_err(err)
	}
	// first try the r/w url...
	git := exec.Command(
		"git", "clone", "git@github.com:hwaf/hep-waftools",
		hwaf_tools,
	)
	// if !quiet {
	// 	git.Stdout = os.Stdout
	// 	git.Stderr = os.Stderr
	// }

	if git.Run() != nil {
		git := exec.Command(
			"git", "clone", "git://github.com/hwaf/hep-waftools",
			hwaf_tools,
		)
		if !quiet {
			git.Stdout = os.Stdout
			git.Stderr = os.Stderr
		}
		err = git.Run()
		handle_err(err)
	}

	// add bin dir
	bin := filepath.Join(top, "bin")
	if !path_exists(bin) {
		err = os.MkdirAll(bin, 0700)
		handle_err(err)
	}

	// add waf-bin
	waf_fname := filepath.Join(bin, "waf")
	if path_exists(waf_fname) {
		err = os.Remove(waf_fname)
		handle_err(err)
	}
	waf, err := os.OpenFile(waf_fname, os.O_WRONLY|os.O_CREATE, 0777)
	handle_err(err)
	defer func() {
		err = waf.Sync()
		handle_err(err)
		err = waf.Close()
		handle_err(err)
	}()

	resp, err := http.Get("https://github.com/hwaf/hwaf/raw/master/waf")
	handle_err(err)
	defer resp.Body.Close()
	_, err = io.Copy(waf, resp.Body)
	handle_err(err)

	if !quiet {
		fmt.Printf("%s... [ok]\n", n)
	}
}

// EOF
<commit_msg>self-init: protection against HWAF_ROOT. dummied out self-init<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
)

func hwaf_make_cmd_self_init() *commander.Command {
	cmd := &commander.Command{
		Run:       hwaf_run_cmd_self_init,
		UsageLine: "init [options] <workarea>",
		Short:     "initialize hwaf itself",
		Long: `
init initializes hwaf internal files.

ex:
 $ hwaf self init
`,
		Flag: *flag.NewFlagSet("hwaf-self-init", flag.ExitOnError),
	}
	cmd.Flag.Bool("q", true, "only print error and warning messages, all other output will be suppressed")

	return cmd
}

func hwaf_run_cmd_self_init(cmd *commander.Command, args []string) {
	var err error
	n := "hwaf-self-" + cmd.Name()

	switch len(args) {
	case 0:
		// ok
	default:
		err = fmt.Errorf("%s: does NOT take any argument", n)
		handle_err(err)
	}

	quiet := cmd.Flag.Lookup("q").Value.Get().(bool)

	if !quiet {
		fmt.Printf("%s...\n", n)
	}

	hwaf_root := os.Getenv("HWAF_ROOT")
	for _, dir := range []string{g_ctx.Root, hwaf_root} {
		if dir != "" {
			g_ctx.Warn("you are trying to 'hwaf self init' while running a HWAF_ROOT-based installation\n")
			g_ctx.Warn("this is like crossing the streams in Ghostbusters (ie: it's bad.)\n")
			g_ctx.Warn("if you think you know what you are doing, unset HWAF_ROOT and re-run 'hwaf self init'\n")
			err = fmt.Errorf("${HWAF_ROOT} was set (%s)", dir)
			handle_err(err)
		}
	}

	// 'hwaf self init' is now dummied out...

	if !quiet {
		fmt.Printf("%s... [ok]\n", n)
	}
}

// EOF
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"strings"
)

func init() {
	addCommand(Command{
		name: "emote",
		help: "allows you to send an action to the room.",
		exec: emoteCmd,
	})
}

func emoteCmd(user Player, args []string) (ok bool) {
	if len(args) == 0 {
		return false
	}

	str := strings.Join(args, " ")

	inv := user.Room().Inventory()
	var tmp []Player
	for _, i := range inv {
		p, ok := i.(Player)
		if ok {
			tmp = append(tmp, p)
		}
	}

	for _, p := range tmp {
		if p == user {
			user.Write(fmt.Sprintf("you %s", str))
		} else {
			p.Write(fmt.Sprintf("%s %s", user.Name(), str))
		}
	}

	return true
}
<commit_msg>Fix minor typo in emote<commit_after>package command

import (
	"fmt"
	"strings"
)

func init() {
	addCommand(Command{
		name: "emote",
		help: "allows you to send an action to the room.",
		exec: emoteCmd,
	})
}

func emoteCmd(user Player, args []string) (ok bool) {
	if len(args) == 0 {
		return false
	}

	str := strings.Join(args, " ")

	inv := user.Room().Inventory()
	var tmp []Player
	for _, i := range inv {
		p, ok := i.(Player)
		if ok {
			tmp = append(tmp, p)
		}
	}

	for _, p := range tmp {
		if p == user {
			user.Write(fmt.Sprintf("you emote: %s %s", user.Name(), str))
		} else {
			p.Write(fmt.Sprintf("%s %s", user.Name(), str))
		}
	}

	return true
}
<|endoftext|>


<commit_before>package commands

import (
  "fmt"
  "net/http"
  "encoding/base64"
  "encoding/json"
  "io/ioutil"
  "strings"
  "github.com/mmcquillan/jane/models"
)

type Ticket struct {
  Key string `json:"key"`
  Fields Fields `json:"fields"`
}

type Fields struct {
  Summary string `json:"summary"`
  Assignee Assignee `json:"assignee"`
  Status Status `json:"status"`
  Description string `json:"description"`
  Creator Creator `json:"creator"`
  Priority Priority `json:"priority"`
}

type Assignee struct {
  Name string `json:"name"`
}

type Creator struct {
  Name string `json:"name"`
}

type IssueType struct {
  Name string `json:"name"`
}

type Status struct {
  Description string `json:"description"`
  Name string `json:"name"`
}

type Priority struct {
  Name string `json:"name"`
}

func Jira(msg string, command models.Command) string {
  msg = strings.TrimSpace(strings.Replace(msg, command.Match, "", 1))
  client := &http.Client {

  }

  baseUrl := "https://prysminc.atlassian.net/"
  issueNumber := msg

  auth := command.ApiKey
  encodedAuth := EncodeB64(auth)

  req, err := http.NewRequest("GET", baseUrl + "rest/api/2/issue/" + issueNumber, nil)
  if err != nil {
    fmt.Println(err)
  } else {
    req.Header.Add("Content-Type", "application/json")
    req.Header.Add("Authorization", "Basic " + encodedAuth)
  }

  response, err := client.Do(req)
  if err != nil {
    fmt.Println(err)
  }

  defer response.Body.Close()

  body, err := ioutil.ReadAll(response.Body)
  if err != nil {
    fmt.Println(err)
  }

  var ticket Ticket
  json.Unmarshal(body, &ticket)

  link := baseUrl + "browse/" + issueNumber

  return fmt.Sprintf("Status: %s\nAssignee: %s\nPriority: %s\nSummary: %s\nDescription: %s\n\n%s\n",
    ticket.Fields.Status.Name, ticket.Fields.Assignee.Name, ticket.Fields.Priority.Name,
    ticket.Fields.Summary, ticket.Fields.Description, link)
}

func EncodeB64(message string) string {
    base64Text := make([]byte, base64.StdEncoding.EncodedLen(len(message)))
    base64.StdEncoding.Encode(base64Text, []byte(message))
    return string(base64Text)
}
<commit_msg>Using the args for now to set the base url<commit_after>package commands

import (
  "fmt"
  "net/http"
  "encoding/base64"
  "encoding/json"
  "io/ioutil"
  "strings"
  "github.com/mmcquillan/jane/models"
)

type Ticket struct {
  Key string `json:"key"`
  Fields Fields `json:"fields"`
}

type Fields struct {
  Summary string `json:"summary"`
  Assignee Assignee `json:"assignee"`
  Status Status `json:"status"`
  Description string `json:"description"`
  Creator Creator `json:"creator"`
  Priority Priority `json:"priority"`
}

type Assignee struct {
  Name string `json:"name"`
}

type Creator struct {
  Name string `json:"name"`
}

type IssueType struct {
  Name string `json:"name"`
}

type Status struct {
  Description string `json:"description"`
  Name string `json:"name"`
}

type Priority struct {
  Name string `json:"name"`
}

func Jira(msg string, command models.Command) string {
  msg = strings.TrimSpace(strings.Replace(msg, command.Match, "", 1))
  client := &http.Client {

  }

  baseUrl := command.Args
  issueNumber := msg

  auth := command.ApiKey
  encodedAuth := EncodeB64(auth)

  req, err := http.NewRequest("GET", baseUrl + "rest/api/2/issue/" + issueNumber, nil)
  if err != nil {
    fmt.Println(err)
  } else {
    req.Header.Add("Content-Type", "application/json")
    req.Header.Add("Authorization", "Basic " + encodedAuth)
  }

  response, err := client.Do(req)
  if err != nil {
    fmt.Println(err)
  }

  defer response.Body.Close()

  body, err := ioutil.ReadAll(response.Body)
  if err != nil {
    fmt.Println(err)
  }

  var ticket Ticket
  json.Unmarshal(body, &ticket)

  link := baseUrl + "browse/" + issueNumber

  return fmt.Sprintf("Status: %s\nAssignee: %s\nPriority: %s\nSummary: %s\nDescription: %s\n\n%s\n",
    ticket.Fields.Status.Name, ticket.Fields.Assignee.Name, ticket.Fields.Priority.Name,
    ticket.Fields.Summary, ticket.Fields.Description, link)
}

func EncodeB64(message string) string {
    base64Text := make([]byte, base64.StdEncoding.EncodedLen(len(message)))
    base64.StdEncoding.Encode(base64Text, []byte(message))
    return string(base64Text)
}
<|endoftext|>


<commit_before>package vshard

import (
	"testing"

	"github.com/stretchr/testify/suite"
)

type VShardCommandsTestSuite struct {
	suite.Suite
	Pool *Pool
}

func (suite *VShardCommandsTestSuite) SetupSuite() {
	suite.Pool = setupPool(suite.T())
}

func (suite *VShardCommandsTestSuite) TearDownTest() {
	tearDownPool(suite.T(), suite.Pool)
}

func (suite *VShardCommandsTestSuite) TestSetGet() {
	key := "x"
	expected := "hello-test"
	ok, err := suite.Pool.Set(key, 0, 0, []byte(expected))
	suite.True(ok)
	suite.NoError(err)

	value, err := suite.Pool.Get(key)
	suite.NoError(err)
	suite.Equal(expected, string(value))
}

func (suite *VShardCommandsTestSuite) TestGetInexistentKey() {
	key := "key-do-not-exist"
	value, err := suite.Pool.Get(key)
	suite.Equal(ErrKeyNotFound, err)
	suite.Empty(value)
}

func TestVShardCommandsTestSuite(t *testing.T) {
	suite.Run(t, new(VShardCommandsTestSuite))
}
<commit_msg>tests Add()<commit_after>package vshard

import (
	"testing"

	"github.com/stretchr/testify/suite"
)

type VShardCommandsTestSuite struct {
	suite.Suite
	Pool *Pool
}

func (suite *VShardCommandsTestSuite) SetupSuite() {
	suite.Pool = setupPool(suite.T())
}

func (suite *VShardCommandsTestSuite) TearDownTest() {
	tearDownPool(suite.T(), suite.Pool)
}

func (suite *VShardCommandsTestSuite) TestSetGet() {
	key := "x"
	expected := "hello-test"
	ok, err := suite.Pool.Set(key, 0, 0, []byte(expected))
	suite.True(ok)
	suite.NoError(err)

	value, err := suite.Pool.Get(key)
	suite.NoError(err)
	suite.Equal(expected, string(value))
}

func (suite *VShardCommandsTestSuite) TestGetInexistentKey() {
	key := "key-do-not-exist"
	value, err := suite.Pool.Get(key)
	suite.Equal(ErrKeyNotFound, err)
	suite.Empty(value)
}

func (suite *VShardCommandsTestSuite) TestAdd() {
	key := "add-key"
	expected := "vshard-test-add"
	ok, err := suite.Pool.Add(key, 0, 0, []byte(expected))
	suite.True(ok)
	suite.NoError(err)

	newValue := "this-should-not-work"
	ok, err = suite.Pool.Add(key, 0, 0, []byte(newValue))
	suite.False(ok)
	suite.NoError(err)

	value, err := suite.Pool.Get(key)
	suite.NoError(err)
	suite.Equal(expected, string(value))
}

func TestVShardCommandsTestSuite(t *testing.T) {
	suite.Run(t, new(VShardCommandsTestSuite))
}
<|endoftext|>


<commit_before>package dbus

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"
	"strconv"
)

// Variant represents the D-Bus variant type.
type Variant struct {
	sig   Signature
	value interface{}
}

// MakeVariant converts the given value to a Variant. It panics if v cannot be
// represented as a D-Bus type.
func MakeVariant(v interface{}) Variant {
	return MakeVariantWithSignature(v, SignatureOf(v))
}

// MakeVariantWithSignature converts the given value to a Variant.
func MakeVariantWithSignature(v interface{}, s Signature) Variant {
	return Variant{s, v}
}

// ParseVariant parses the given string as a variant as described at
// https://developer.gnome.org/glib/unstable/gvariant-text.html. If sig is not
// empty, it is taken to be the expected signature for the variant.
func ParseVariant(s string, sig Signature) (Variant, error) {
	tokens := varLex(s)
	p := &varParser{tokens: tokens}
	n, err := varMakeNode(p)
	if err != nil {
		return Variant{}, err
	}
	if sig.str == "" {
		sig, err = varInfer(n)
		if err != nil {
			return Variant{}, err
		}
	}
	v, err := n.Value(sig)
	if err != nil {
		return Variant{}, err
	}
	return MakeVariant(v), nil
}

// format returns a formatted version of v and whether this string can be parsed
// unambigously.
func (v Variant) format() (string, bool) {
	switch v.sig.str[0] {
	case 'b', 'i':
		return fmt.Sprint(v.value), true
	case 'n', 'q', 'u', 'x', 't', 'd', 'h':
		return fmt.Sprint(v.value), false
	case 's':
		return strconv.Quote(v.value.(string)), true
	case 'o':
		return strconv.Quote(string(v.value.(ObjectPath))), false
	case 'g':
		return strconv.Quote(v.value.(Signature).str), false
	case 'v':
		s, unamb := v.value.(Variant).format()
		if !unamb {
			return "<@" + v.value.(Variant).sig.str + " " + s + ">", true
		}
		return "<" + s + ">", true
	case 'y':
		return fmt.Sprintf("%#x", v.value.(byte)), false
	}
	rv := reflect.ValueOf(v.value)
	switch rv.Kind() {
	case reflect.Slice:
		if rv.Len() == 0 {
			return "[]", false
		}
		unamb := true
		buf := bytes.NewBuffer([]byte("["))
		for i := 0; i < rv.Len(); i++ {
			// TODO: slooow
			s, b := MakeVariant(rv.Index(i).Interface()).format()
			unamb = unamb && b
			buf.WriteString(s)
			if i != rv.Len()-1 {
				buf.WriteString(", ")
			}
		}
		buf.WriteByte(']')
		return buf.String(), unamb
	case reflect.Map:
		if rv.Len() == 0 {
			return "{}", false
		}
		unamb := true
		var buf bytes.Buffer
		kvs := make([]string, rv.Len())
		for i, k := range rv.MapKeys() {
			s, b := MakeVariant(k.Interface()).format()
			unamb = unamb && b
			buf.Reset()
			buf.WriteString(s)
			buf.WriteString(": ")
			s, b = MakeVariant(rv.MapIndex(k).Interface()).format()
			unamb = unamb && b
			buf.WriteString(s)
			kvs[i] = buf.String()
		}
		buf.Reset()
		buf.WriteByte('{')
		sort.Strings(kvs)
		for i, kv := range kvs {
			if i > 0 {
				buf.WriteString(", ")
			}
			buf.WriteString(kv)
		}
		buf.WriteByte('}')
		return buf.String(), unamb
	}
	return `"INVALID"`, true
}

// Signature returns the D-Bus signature of the underlying value of v.
func (v Variant) Signature() Signature {
	return v.sig
}

// String returns the string representation of the underlying value of v as
// described at https://developer.gnome.org/glib/unstable/gvariant-text.html.
func (v Variant) String() string {
	s, unamb := v.format()
	if !unamb {
		return "@" + v.sig.str + " " + s
	}
	return s
}

// Value returns the underlying value of v.
func (v Variant) Value() interface{} {
	return v.value
}
<commit_msg>Fix link to gnome developer documentation.<commit_after>package dbus

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"
	"strconv"
)

// Variant represents the D-Bus variant type.
type Variant struct {
	sig   Signature
	value interface{}
}

// MakeVariant converts the given value to a Variant. It panics if v cannot be
// represented as a D-Bus type.
func MakeVariant(v interface{}) Variant {
	return MakeVariantWithSignature(v, SignatureOf(v))
}

// MakeVariantWithSignature converts the given value to a Variant.
func MakeVariantWithSignature(v interface{}, s Signature) Variant {
	return Variant{s, v}
}

// ParseVariant parses the given string as a variant as described at
// https://developer.gnome.org/glib/stable/gvariant-text.html. If sig is not
// empty, it is taken to be the expected signature for the variant.
func ParseVariant(s string, sig Signature) (Variant, error) {
	tokens := varLex(s)
	p := &varParser{tokens: tokens}
	n, err := varMakeNode(p)
	if err != nil {
		return Variant{}, err
	}
	if sig.str == "" {
		sig, err = varInfer(n)
		if err != nil {
			return Variant{}, err
		}
	}
	v, err := n.Value(sig)
	if err != nil {
		return Variant{}, err
	}
	return MakeVariant(v), nil
}

// format returns a formatted version of v and whether this string can be parsed
// unambigously.
func (v Variant) format() (string, bool) {
	switch v.sig.str[0] {
	case 'b', 'i':
		return fmt.Sprint(v.value), true
	case 'n', 'q', 'u', 'x', 't', 'd', 'h':
		return fmt.Sprint(v.value), false
	case 's':
		return strconv.Quote(v.value.(string)), true
	case 'o':
		return strconv.Quote(string(v.value.(ObjectPath))), false
	case 'g':
		return strconv.Quote(v.value.(Signature).str), false
	case 'v':
		s, unamb := v.value.(Variant).format()
		if !unamb {
			return "<@" + v.value.(Variant).sig.str + " " + s + ">", true
		}
		return "<" + s + ">", true
	case 'y':
		return fmt.Sprintf("%#x", v.value.(byte)), false
	}
	rv := reflect.ValueOf(v.value)
	switch rv.Kind() {
	case reflect.Slice:
		if rv.Len() == 0 {
			return "[]", false
		}
		unamb := true
		buf := bytes.NewBuffer([]byte("["))
		for i := 0; i < rv.Len(); i++ {
			// TODO: slooow
			s, b := MakeVariant(rv.Index(i).Interface()).format()
			unamb = unamb && b
			buf.WriteString(s)
			if i != rv.Len()-1 {
				buf.WriteString(", ")
			}
		}
		buf.WriteByte(']')
		return buf.String(), unamb
	case reflect.Map:
		if rv.Len() == 0 {
			return "{}", false
		}
		unamb := true
		var buf bytes.Buffer
		kvs := make([]string, rv.Len())
		for i, k := range rv.MapKeys() {
			s, b := MakeVariant(k.Interface()).format()
			unamb = unamb && b
			buf.Reset()
			buf.WriteString(s)
			buf.WriteString(": ")
			s, b = MakeVariant(rv.MapIndex(k).Interface()).format()
			unamb = unamb && b
			buf.WriteString(s)
			kvs[i] = buf.String()
		}
		buf.Reset()
		buf.WriteByte('{')
		sort.Strings(kvs)
		for i, kv := range kvs {
			if i > 0 {
				buf.WriteString(", ")
			}
			buf.WriteString(kv)
		}
		buf.WriteByte('}')
		return buf.String(), unamb
	}
	return `"INVALID"`, true
}

// Signature returns the D-Bus signature of the underlying value of v.
func (v Variant) Signature() Signature {
	return v.sig
}

// String returns the string representation of the underlying value of v as
// described at https://developer.gnome.org/glib/stable/gvariant-text.html.
func (v Variant) String() string {
	s, unamb := v.format()
	if !unamb {
		return "@" + v.sig.str + " " + s
	}
	return s
}

// Value returns the underlying value of v.
func (v Variant) Value() interface{} {
	return v.value
}
<|endoftext|>


<commit_before>package heat

import (
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/256dpi/jsonapi/v2"

	"github.com/256dpi/fire/coal"
)

// Key is a structure used to encode a key.
type Key interface {
	// Validate should validate the token.
	Validate() error

	base() *Base
}

// Base can be embedded in a struct to turn it into a key.
type Base struct {
	ID     coal.ID
	Expiry time.Time
}

func (b *Base) base() *Base {
	return b
}

var baseType = reflect.TypeOf(Base{})

// Meta will parse the keys "heat" tag on the embedded heat.Base struct and
// return the encoded name and default expiry.
func Meta(key Key) (string, time.Duration) {
	// get first field
	field := reflect.TypeOf(key).Elem().Field(0)

	// check field type
	if field.Type != baseType {
		panic(`heat: expected first struct field to be of type "heat.Base"`)
	}

	// check field name
	if field.Name != "Base" {
		panic(`heat: expected an embedded "heat.Base" as the first struct field`)
	}

	// split tag
	tag := strings.Split(field.Tag.Get("heat"), ",")

	// check tag
	if len(tag) != 2 || tag[0] == "" || tag[1] == "" {
		panic(`heat: expected to find a tag of the form 'heat:"name,expiry"' on "heat.Base"`)
	}

	// get expiry
	expiry, err := time.ParseDuration(tag[1])
	if err != nil {
		panic(err)
	}

	return tag[0], expiry
}

// Notary is used to issue and verify tokens from keys.
type Notary struct {
	issuer string
	secret []byte
}

// NewNotary creates a new notary with the specified name and secret. It will
// panic if the name is missing or the specified secret is less that 16 bytes.
func NewNotary(name string, secret []byte) *Notary {
	// check name
	if name == "" {
		panic("heat: missing name")
	}

	// set random secret if missing
	if len(secret) == 0 {
		panic("heat: missing secret")
	}

	// check secret
	if len(secret) < 16 {
		panic("heat: secret too short")
	}

	return &Notary{
		secret: secret,
		issuer: name,
	}
}

// Issue will generate a token from the specified key.
func (n *Notary) Issue(key Key) (string, error) {
	// get key name and default expiry
	name, expiry := Meta(key)

	// get base
	base := key.base()

	// ensure id
	if base.ID.IsZero() {
		base.ID = coal.New()
	}

	// ensure expiry
	if base.Expiry.IsZero() {
		base.Expiry = time.Now().Add(expiry)
	}

	// validate key
	err := key.Validate()
	if err != nil {
		return "", err
	}

	// get data
	data, err := jsonapi.StructToMap(key, nil)
	if err != nil {
		return "", err
	}

	// issue token
	token, err := Issue(n.secret, n.issuer, name, RawKey{
		ID:     base.ID.Hex(),
		Expiry: time.Now().Add(expiry),
		Data:   Data(data),
	})
	if err != nil {
		return "", err
	}

	return token, nil
}

// Verify will verify the specified token and fill the specified key.
func (n *Notary) Verify(key Key, token string) error {
	// get key name
	name, _ := Meta(key)

	// verify token
	rawKey, err := Verify(n.secret, n.issuer, name, token)
	if err != nil {
		return err
	}

	// check id
	kid, err := coal.FromHex(rawKey.ID)
	if err != nil {
		return err
	} else if kid.IsZero() {
		return fmt.Errorf("zero key id")
	}

	// set id and expiry
	key.base().ID = kid
	key.base().Expiry = rawKey.Expiry

	// assign data
	err = jsonapi.Map(rawKey.Data).Assign(key)
	if err != nil {
		return err
	}

	// validate key
	err = key.Validate()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>cache key meta<commit_after>package heat

import (
	"fmt"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/256dpi/jsonapi/v2"

	"github.com/256dpi/fire/coal"
)

// Key is a structure used to encode a key.
type Key interface {
	// Validate should validate the token.
	Validate() error

	base() *Base
}

// Base can be embedded in a struct to turn it into a key.
type Base struct {
	ID     coal.ID
	Expiry time.Time
}

func (b *Base) base() *Base {
	return b
}

var baseType = reflect.TypeOf(Base{})

type meta struct {
	name   string
	expiry time.Duration
}

var metaCache = map[reflect.Type]meta{}
var metaMutex sync.Mutex

// Meta will parse the keys "heat" tag on the embedded heat.Base struct and
// return the encoded name and default expiry.
func Meta(key Key) (string, time.Duration) {
	// acquire mutex
	metaMutex.Lock()
	defer metaMutex.Unlock()

	// get typ
	typ := reflect.TypeOf(key)

	// check cache
	if meta, ok := metaCache[typ]; ok {
		return meta.name, meta.expiry
	}

	// get first field
	field := typ.Elem().Field(0)

	// check field type
	if field.Type != baseType {
		panic(`heat: expected first struct field to be of type "heat.Base"`)
	}

	// check field name
	if field.Name != "Base" {
		panic(`heat: expected an embedded "heat.Base" as the first struct field`)
	}

	// split tag
	tag := strings.Split(field.Tag.Get("heat"), ",")

	// check tag
	if len(tag) != 2 || tag[0] == "" || tag[1] == "" {
		panic(`heat: expected to find a tag of the form 'heat:"name,expiry"' on "heat.Base"`)
	}

	// get expiry
	expiry, err := time.ParseDuration(tag[1])
	if err != nil {
		panic(err)
	}

	// cache meta
	metaCache[typ] = meta{
		name:   tag[0],
		expiry: expiry,
	}

	return tag[0], expiry
}

// Notary is used to issue and verify tokens from keys.
type Notary struct {
	issuer string
	secret []byte
}

// NewNotary creates a new notary with the specified name and secret. It will
// panic if the name is missing or the specified secret is less that 16 bytes.
func NewNotary(name string, secret []byte) *Notary {
	// check name
	if name == "" {
		panic("heat: missing name")
	}

	// set random secret if missing
	if len(secret) == 0 {
		panic("heat: missing secret")
	}

	// check secret
	if len(secret) < 16 {
		panic("heat: secret too short")
	}

	return &Notary{
		secret: secret,
		issuer: name,
	}
}

// Issue will generate a token from the specified key.
func (n *Notary) Issue(key Key) (string, error) {
	// get key name and default expiry
	name, expiry := Meta(key)

	// get base
	base := key.base()

	// ensure id
	if base.ID.IsZero() {
		base.ID = coal.New()
	}

	// ensure expiry
	if base.Expiry.IsZero() {
		base.Expiry = time.Now().Add(expiry)
	}

	// validate key
	err := key.Validate()
	if err != nil {
		return "", err
	}

	// get data
	data, err := jsonapi.StructToMap(key, nil)
	if err != nil {
		return "", err
	}

	// issue token
	token, err := Issue(n.secret, n.issuer, name, RawKey{
		ID:     base.ID.Hex(),
		Expiry: time.Now().Add(expiry),
		Data:   Data(data),
	})
	if err != nil {
		return "", err
	}

	return token, nil
}

// Verify will verify the specified token and fill the specified key.
func (n *Notary) Verify(key Key, token string) error {
	// get key name
	name, _ := Meta(key)

	// verify token
	rawKey, err := Verify(n.secret, n.issuer, name, token)
	if err != nil {
		return err
	}

	// check id
	kid, err := coal.FromHex(rawKey.ID)
	if err != nil {
		return err
	} else if kid.IsZero() {
		return fmt.Errorf("zero key id")
	}

	// set id and expiry
	key.base().ID = kid
	key.base().Expiry = rawKey.Expiry

	// assign data
	err = jsonapi.Map(rawKey.Data).Assign(key)
	if err != nil {
		return err
	}

	// validate key
	err = key.Validate()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package helper

import (
	"log"

	"github.com/SYNQfm/SYNQ-Golang/synq"
	"github.com/SYNQfm/SYNQ-Golang/upload"
	"github.com/SYNQfm/helpers/common"
)

func LoadVideosByQuery(query, name string, c common.Cacheable, api synq.Api) (videos []synq.Video, err error) {
	ok := common.LoadFromCache(name, c, &videos)
	if ok {
		return videos, nil
	}
	log.Printf("querying '%s'\n", query)
	videos, err = api.Query(query)
	if err != nil {
		return videos, err
	}
	common.SaveToCache(name, c, videos)
	return videos, err
}

// fow now, the query will be the account id
func LoadVideosByQueryV2(query, name string, c common.Cacheable, api synq.ApiV2) (videos []synq.VideoV2, err error) {
	ok := common.LoadFromCache(name, c, &videos)
	if ok {
		return videos, nil
	}
	log.Printf("get all videos (filter by account '%s')\n", query)
	videos, err = api.GetVideos(query)
	if err != nil {
		return videos, err
	}
	common.SaveToCache(name, c, videos)
	return videos, err
}

func LoadVideo(id string, c common.Cacheable, api synq.Api) (video synq.Video, err error) {
	ok := common.LoadFromCache(id, c, &video)
	if ok {
		video.Api = &api
		return video, nil
	}
	// need to use the v1 api to get the raw video data
	log.Printf("Getting video %s", id)
	video, e := api.GetVideo(id)
	if e != nil {
		return video, e
	}
	common.SaveToCache(id, c, &video)
	return video, nil
}

func LoadVideoV2(id string, c common.Cacheable, api synq.ApiV2) (video synq.VideoV2, err error) {
	ok := common.LoadFromCache(id, c, &video)
	if ok {
		video.Api = &api
		for _, a := range video.Assets {
			a.Api = api
		}
		return video, nil
	}
	log.Printf("Getting video %s\n", id)
	video, err = api.GetVideo(id)
	if err != nil {
		return video, err
	}
	common.SaveToCache(id, c, &video)
	video.Api = &api
	return video, nil
}

func LoadUploadParameters(id string, req synq.UnicornParam, c common.Cacheable, api synq.ApiV2) (up upload.UploadParameters, err error) {
	lookId := id
	if req.AssetId != "" {
		lookId = req.AssetId
	}
	ok := common.LoadFromCache(lookId+"_up", c, &up)
	if ok {
		return up, nil
	}
	log.Printf("Getting upload parameters for %s\n", id)
	up, err = api.GetUploadParams(id, req)
	if err != nil {
		return up, err
	}
	common.SaveToCache(lookId+"_up", c, &up)
	return up, nil
}

func LoadAsset(id string, c common.Cacheable, api synq.ApiV2) (asset synq.Asset, err error) {
	ok := common.LoadFromCache(id, c, &asset)
	if !ok {
		log.Printf("Getting asset %s\n", id)
		asset, err = api.GetAsset(id)
		if err != nil {
			return asset, err
		}
	} else {
		asset.Api = api
	}

	if asset.Video.Id == "" {
		video, err := LoadVideoV2(asset.VideoId, c, api)
		if err != nil {
			return asset, err
		}
		asset.Video = video
	} else {
		// cache the video for re-use later
		common.SaveToCache(asset.Video.Id, c, &asset.Video)
	}
	common.SaveToCache(id, c, &asset)
	return asset, nil
}
<commit_msg>on load, set the video and api object<commit_after>package helper

import (
	"log"

	"github.com/SYNQfm/SYNQ-Golang/synq"
	"github.com/SYNQfm/SYNQ-Golang/upload"
	"github.com/SYNQfm/helpers/common"
)

func LoadVideosByQuery(query, name string, c common.Cacheable, api synq.Api) (videos []synq.Video, err error) {
	ok := common.LoadFromCache(name, c, &videos)
	if ok {
		return videos, nil
	}
	log.Printf("querying '%s'\n", query)
	videos, err = api.Query(query)
	if err != nil {
		return videos, err
	}
	common.SaveToCache(name, c, videos)
	return videos, err
}

// fow now, the query will be the account id
func LoadVideosByQueryV2(query, name string, c common.Cacheable, api synq.ApiV2) (videos []synq.VideoV2, err error) {
	ok := common.LoadFromCache(name, c, &videos)
	if ok {
		return videos, nil
	}
	log.Printf("get all videos (filter by account '%s')\n", query)
	videos, err = api.GetVideos(query)
	if err != nil {
		return videos, err
	}
	common.SaveToCache(name, c, videos)
	return videos, err
}

func LoadVideo(id string, c common.Cacheable, api synq.Api) (video synq.Video, err error) {
	ok := common.LoadFromCache(id, c, &video)
	if ok {
		video.Api = &api
		return video, nil
	}
	// need to use the v1 api to get the raw video data
	log.Printf("Getting video %s", id)
	video, e := api.GetVideo(id)
	if e != nil {
		return video, e
	}
	common.SaveToCache(id, c, &video)
	return video, nil
}

func LoadVideoV2(id string, c common.Cacheable, api synq.ApiV2) (video synq.VideoV2, err error) {
	ok := common.LoadFromCache(id, c, &video)
	if ok {
		video.Api = &api
		for _, a := range video.Assets {
			a.Video = video
			a.Api = api
		}
		return video, nil
	}
	log.Printf("Getting video %s\n", id)
	video, err = api.GetVideo(id)
	if err != nil {
		return video, err
	}
	common.SaveToCache(id, c, &video)
	video.Api = &api
	return video, nil
}

func LoadUploadParameters(id string, req synq.UnicornParam, c common.Cacheable, api synq.ApiV2) (up upload.UploadParameters, err error) {
	lookId := id
	if req.AssetId != "" {
		lookId = req.AssetId
	}
	ok := common.LoadFromCache(lookId+"_up", c, &up)
	if ok {
		return up, nil
	}
	log.Printf("Getting upload parameters for %s\n", id)
	up, err = api.GetUploadParams(id, req)
	if err != nil {
		return up, err
	}
	common.SaveToCache(lookId+"_up", c, &up)
	return up, nil
}

func LoadAsset(id string, c common.Cacheable, api synq.ApiV2) (asset synq.Asset, err error) {
	ok := common.LoadFromCache(id, c, &asset)
	if !ok {
		log.Printf("Getting asset %s\n", id)
		asset, err = api.GetAsset(id)
		if err != nil {
			return asset, err
		}
	} else {
		asset.Api = api
	}

	if asset.Video.Id == "" {
		video, err := LoadVideoV2(asset.VideoId, c, api)
		if err != nil {
			return asset, err
		}
		asset.Video = video
	} else {
		// cache the video for re-use later
		common.SaveToCache(asset.Video.Id, c, &asset.Video)
	}
	common.SaveToCache(id, c, &asset)
	return asset, nil
}
<|endoftext|>