<commit_before>/*
Copyright 2017 WALLIX

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package commands

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"syscall"
	"text/template"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/spf13/cobra"
	"github.com/wallix/awless/aws"
	awsconfig "github.com/wallix/awless/aws/config"
	"github.com/wallix/awless/cloud"
	"github.com/wallix/awless/cloud/properties"
	"github.com/wallix/awless/config"
	"github.com/wallix/awless/console"
	"github.com/wallix/awless/graph"
	"github.com/wallix/awless/logger"
)

var keyPathFlag string
var printSSHConfigFlag bool
var printSSHCLIFlag bool

func init() {
	RootCmd.AddCommand(sshCmd)
	sshCmd.Flags().StringVarP(&keyPathFlag, "identity", "i", "", "Set path toward the identity (key file) to use to connect through SSH")
	sshCmd.Flags().BoolVar(&printSSHConfigFlag, "print-config", false, "Print SSH configuration for ~/.ssh/config file.")
	sshCmd.Flags().BoolVar(&printSSHCLIFlag, "print-cli", false, "Print the CLI one-liner to connect with SSH. (/usr/bin/ssh user@ip -i ...)")
}

var sshCmd = &cobra.Command{
	Use:   "ssh [USER@]INSTANCE",
	Short: "Launch a SSH (Secure Shell) session to an instance given an id or alias",
	Example: `  awless ssh i-8d43b21b                       # using the instance id
  awless ssh ec2-user@redis-prod              # using the instance name and specify a user
  awless ssh @redis-prod -i ./path/toward/key # with a keyfile`,
	PersistentPreRun:  applyHooks(initLoggerHook, initAwlessEnvHook, initCloudServicesHook),
	PersistentPostRun: applyHooks(saveHistoryHook, verifyNewVersionHook),

	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) != 1 {
			return fmt.Errorf("instance required")
		}

		var user, instanceID string
		if strings.Contains(args[0], "@") {
			user = strings.Split(args[0], "@")[0]
			instanceID = strings.Split(args[0], "@")[1]
		} else {
			instanceID = args[0]
		}

		resourcesGraph, ip := fetchConnectionInfo()

		var inst *graph.Resource

		instanceResolvers := []graph.Resolver{&graph.ByProperty{Key: "Name", Value: instanceID}, &graph.ByType{Typ: cloud.Instance}}
		resources, err := resourcesGraph.ResolveResources(&graph.And{Resolvers: instanceResolvers})
		exitOn(err)
		switch len(resources) {
		case 0:
			// No instance with that name, use the id
			inst, err = findResource(resourcesGraph, instanceID, cloud.Instance)
			exitOn(err)
		case 1:
			inst = resources[0]
		default:
			idStatus := graph.Resources(resources).Map(func(r *graph.Resource) string {
				return fmt.Sprintf("%s (%s)", r.Id(), r.Properties[properties.State])
			})
			logger.Infof("Found %d resources with name '%s': %s", len(resources), instanceID, strings.Join(idStatus, ", "))

			var running []*graph.Resource
			running, err = resourcesGraph.ResolveResources(&graph.And{Resolvers: append(instanceResolvers, &graph.ByProperty{Key: properties.State, Value: "running"})})
			exitOn(err)

			switch len(running) {
			case 0:
				logger.Warning("None of them is running, cannot connect through SSH")
				return nil
			case 1:
				logger.Infof("Found only one instance running: %s. Will connect to this instance.", running[0].Id())
				inst = running[0]
			default:
				logger.Warning("Connect through the running ones using their id:")
				for _, res := range running {
					var up string
					if uptime, ok := res.Properties[properties.Launched].(time.Time); ok {
						up = fmt.Sprintf("\t\t(uptime: %s)", console.HumanizeTime(uptime))
					}
					logger.Warningf("\t`awless ssh %s`%s", res.Id(), up)
				}
				return nil
			}
		}

		cred, err := instanceCredentialsFromGraph(resourcesGraph, inst, keyPathFlag)
		exitOn(err)

		var client *ssh.Client
		if user != "" {
			cred.User = user
			client, err = console.NewSSHClient(cred)
			if err != nil {
				checkInstanceAccessible(resourcesGraph, inst, ip)
				exitOn(err)
			}
			exitOn(sshConnect(instanceID, client, cred))
		} else {
			for _, user := range awsconfig.DefaultAMIUsers {
				logger.Verbosef("trying user '%s'", user)
				cred.User = user
				client, err = console.NewSSHClient(cred)
				if err != nil && strings.Contains(err.Error(), "unable to authenticate") {
					continue
				}
				if err != nil {
					checkInstanceAccessible(resourcesGraph, inst, ip)
					exitOn(err)
				}
				exitOn(sshConnect(instanceID, client, cred))
				return nil
			}
		}
		exitOn(err)
		return nil
	},
}

func instanceCredentialsFromGraph(g *graph.Graph, inst *graph.Resource, keyPathFlag string) (*console.Credentials, error) {
	ip, ok := inst.Properties[properties.PublicIP]
	if !ok {
		return nil, fmt.Errorf("no public IP address for instance %s", inst.Id())
	}

	var keyPath string
	if keyPathFlag != "" {
		keyPath = keyPathFlag
	} else {
		keypair, ok := inst.Properties[properties.KeyPair]
		if !ok {
			return nil, fmt.Errorf("no access key set for instance %s", inst.Id())
		}
		keyPath = path.Join(config.KeysDir, fmt.Sprint(keypair))
	}
	return &console.Credentials{IP: fmt.Sprint(ip), User: "", KeyPath: keyPath}, nil
}

func sshConnect(name string, sshClient *ssh.Client, cred *console.Credentials) error {
	defer sshClient.Close()
	if printSSHConfigFlag {
		params := struct {
			*console.Credentials
			Name string
		}{cred, name}
		return template.Must(template.New("ssh_config").Parse(`
Host {{ .Name }}
	Hostname {{ .IP }}
	User {{ .User }}
	IdentityFile {{ .KeyPath }}
`)).Execute(os.Stdout, params)
	}

	sshPath, sshErr := exec.LookPath("ssh")
	args := []string{"ssh", "-i", cred.KeyPath, fmt.Sprintf("%s@%s", cred.User, cred.IP)}
	if sshErr == nil {
		if printSSHCLIFlag {
			fmt.Println(sshPath + " " + strings.Join(args[1:], " "))
			return nil
		}
		logger.Infof("Login as '%s' on '%s', using keypair '%s' with ssh client at '%s'", cred.User, cred.IP, cred.KeyPath, sshPath)
		return syscall.Exec(sshPath, args, os.Environ())
	} else { // Fallback SSH
		if printSSHCLIFlag {
			fmt.Println(strings.Join(args, " "))
			return nil
		}
		logger.Infof("No SSH. Fallback on builtin client. Login as '%s' on '%s', using keypair '%s'", cred.User, cred.IP, cred.KeyPath)
		return console.InteractiveTerminal(sshClient)
	}
}

func fetchConnectionInfo() (*graph.Graph, net.IP) {
	var resourcesGraph, sgroupsGraph *graph.Graph
	var myip net.IP
	var wg sync.WaitGroup
	var errc = make(chan error)

	wg.Add(1)
	go func() {
		var err error
		defer wg.Done()
		resourcesGraph, err = aws.InfraService.FetchByType(cloud.Instance)
		if err != nil {
			errc <- err
		}
	}()

	wg.Add(1)
	go func() {
		var err error
		defer wg.Done()
		sgroupsGraph, err = aws.InfraService.FetchByType(cloud.SecurityGroup)
		if err != nil {
			errc <- err
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		client := &http.Client{
			Timeout: 2 * time.Second,
		}
		resp, err := client.Get("http://checkip.amazonaws.com/")
		if err == nil {
			b, _ := ioutil.ReadAll(resp.Body)
			resp.Body.Close()
			myip = net.ParseIP(strings.TrimSpace(string(b)))
		}
	}()
	go func() {
		wg.Wait()
		close(errc)
	}()
	for err := range errc {
		if err != nil {
			exitOn(err)
		}
	}
	resourcesGraph.AddGraph(sgroupsGraph)

	return resourcesGraph, myip

}

func checkInstanceAccessible(g *graph.Graph, inst *graph.Resource, myip net.IP) {
	state, ok := inst.Properties[properties.State]
	if st := fmt.Sprint(state); ok && st != "running" {
		logger.Warningf("This instance is '%s' (cannot ssh to a non running state)", st)
		if st == "stopped" {
			logger.Warningf("You can start it with `awless -f start instance id=%s`", inst.Id())
		}
		return
	}

	sgroups, ok := inst.Properties[properties.SecurityGroups].([]string)
	if ok {
		var sshPortOpen, myIPAllowed bool
		for _, id := range sgroups {
			sgroup, err := findResource(g, id, cloud.SecurityGroup)
			if err != nil {
				break
			}

			rules, ok := sgroup.Properties[properties.InboundRules].([]*graph.FirewallRule)
			if ok {
				for _, r := range rules {
					if r.PortRange.Contains(22) {
						sshPortOpen = true
					}
					if myip != nil && r.Contains(myip.String()) {
						myIPAllowed = true
					}
				}
			}
		}

		if !sshPortOpen {
			logger.Warning("Port 22 is not open on this instance")
		}
		if !myIPAllowed && myip != nil {
			logger.Warningf("Your ip %s is not authorized for this instance. You might want to update the securitygroup with:", myip)
			var group = "mygroup"
			if len(sgroups) == 1 {
				group = sgroups[0]
			}
			logger.Warningf("`awless update securitygroup id=%s inbound=authorize protocol=tcp cidr=%s/32 portrange=22`", group, myip)
		}
	}
}

func findResource(g *graph.Graph, id, typ string) (*graph.Resource, error) {
	if found, err := g.FindResource(id); found == nil || err != nil {
		return nil, fmt.Errorf("instance '%s' not found", id)
	}

	return g.GetResource(typ, id)
}
<commit_msg>add ssh private ip<commit_after>/*
Copyright 2017 WALLIX

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package commands

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"syscall"
	"text/template"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/spf13/cobra"
	"github.com/wallix/awless/aws"
	awsconfig "github.com/wallix/awless/aws/config"
	"github.com/wallix/awless/cloud"
	"github.com/wallix/awless/cloud/properties"
	"github.com/wallix/awless/config"
	"github.com/wallix/awless/console"
	"github.com/wallix/awless/graph"
	"github.com/wallix/awless/logger"
)

var keyPathFlag string
var printSSHConfigFlag bool
var printSSHCLIFlag bool
var privateIpFlag bool

func init() {
	RootCmd.AddCommand(sshCmd)
	sshCmd.Flags().StringVarP(&keyPathFlag, "identity", "i", "", "Set path toward the identity (key file) to use to connect through SSH")
	sshCmd.Flags().BoolVar(&printSSHConfigFlag, "print-config", false, "Print SSH configuration for ~/.ssh/config file.")
	sshCmd.Flags().BoolVar(&printSSHCLIFlag, "print-cli", false, "Print the CLI one-liner to connect with SSH. (/usr/bin/ssh user@ip -i ...)")
	sshCmd.Flags().BoolVarP(&privateIpFlag, "private", "p", false, "Use private ip to connect to host")
}

var sshCmd = &cobra.Command{
	Use:   "ssh [USER@]INSTANCE",
	Short: "Launch a SSH (Secure Shell) session to an instance given an id or alias",
	Example: `  awless ssh i-8d43b21b                       # using the instance id
  awless ssh ec2-user@redis-prod              # using the instance name and specify a user
  awless ssh @redis-prod -i ./path/toward/key # with a keyfile`,
	PersistentPreRun:  applyHooks(initLoggerHook, initAwlessEnvHook, initCloudServicesHook),
	PersistentPostRun: applyHooks(saveHistoryHook, verifyNewVersionHook),

	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) != 1 {
			return fmt.Errorf("instance required")
		}

		var user, instanceID string
		if strings.Contains(args[0], "@") {
			user = strings.Split(args[0], "@")[0]
			instanceID = strings.Split(args[0], "@")[1]
		} else {
			instanceID = args[0]
		}

		resourcesGraph, ip := fetchConnectionInfo()

		var inst *graph.Resource

		instanceResolvers := []graph.Resolver{&graph.ByProperty{Key: "Name", Value: instanceID}, &graph.ByType{Typ: cloud.Instance}}
		resources, err := resourcesGraph.ResolveResources(&graph.And{Resolvers: instanceResolvers})
		exitOn(err)
		switch len(resources) {
		case 0:
			// No instance with that name, use the id
			inst, err = findResource(resourcesGraph, instanceID, cloud.Instance)
			exitOn(err)
		case 1:
			inst = resources[0]
		default:
			idStatus := graph.Resources(resources).Map(func(r *graph.Resource) string {
				return fmt.Sprintf("%s (%s)", r.Id(), r.Properties[properties.State])
			})
			logger.Infof("Found %d resources with name '%s': %s", len(resources), instanceID, strings.Join(idStatus, ", "))

			var running []*graph.Resource
			running, err = resourcesGraph.ResolveResources(&graph.And{Resolvers: append(instanceResolvers, &graph.ByProperty{Key: properties.State, Value: "running"})})
			exitOn(err)

			switch len(running) {
			case 0:
				logger.Warning("None of them is running, cannot connect through SSH")
				return nil
			case 1:
				logger.Infof("Found only one instance running: %s. Will connect to this instance.", running[0].Id())
				inst = running[0]
			default:
				logger.Warning("Connect through the running ones using their id:")
				for _, res := range running {
					var up string
					if uptime, ok := res.Properties[properties.Launched].(time.Time); ok {
						up = fmt.Sprintf("\t\t(uptime: %s)", console.HumanizeTime(uptime))
					}
					logger.Warningf("\t`awless ssh %s`%s", res.Id(), up)
				}
				return nil
			}
		}

		cred, err := instanceCredentialsFromGraph(resourcesGraph, inst, keyPathFlag)
		exitOn(err)

		var client *ssh.Client
		if user != "" {
			cred.User = user
			client, err = console.NewSSHClient(cred)
			if err != nil {
				checkInstanceAccessible(resourcesGraph, inst, ip)
				exitOn(err)
			}
			exitOn(sshConnect(instanceID, client, cred))
		} else {
			for _, user := range awsconfig.DefaultAMIUsers {
				logger.Verbosef("trying user '%s'", user)
				cred.User = user
				client, err = console.NewSSHClient(cred)
				if err != nil && strings.Contains(err.Error(), "unable to authenticate") {
					continue
				}
				if err != nil {
					checkInstanceAccessible(resourcesGraph, inst, ip)
					exitOn(err)
				}
				exitOn(sshConnect(instanceID, client, cred))
				return nil
			}
		}
		exitOn(err)
		return nil
	},
}

func getIp(inst *graph.Resource) (string, error){
	var ipKeyType string

	if privateIpFlag {
		ipKeyType = properties.PrivateIP
	} else {
		ipKeyType = properties.PublicIP
	}

	ip, ok := inst.Properties[ipKeyType]

	if !ok {
		return "", fmt.Errorf("no IP address for instance %s", inst.Id())
	}

	return fmt.Sprint(ip), nil
}

func instanceCredentialsFromGraph(g *graph.Graph, inst *graph.Resource, keyPathFlag string) (*console.Credentials, error) {
	ip, err := getIp(inst)
	if err != nil {
		return nil, err
	}

	var keyPath string
	if keyPathFlag != "" {
		keyPath = keyPathFlag
	} else {
		keypair, ok := inst.Properties[properties.KeyPair]
		if !ok {
			return nil, fmt.Errorf("no access key set for instance %s", inst.Id())
		}
		keyPath = path.Join(config.KeysDir, fmt.Sprint(keypair))
	}
	return &console.Credentials{IP: fmt.Sprint(ip), User: "", KeyPath: keyPath}, nil
}

func sshConnect(name string, sshClient *ssh.Client, cred *console.Credentials) error {
	defer sshClient.Close()
	if printSSHConfigFlag {
		params := struct {
			*console.Credentials
			Name string
		}{cred, name}
		return template.Must(template.New("ssh_config").Parse(`
Host {{ .Name }}
	Hostname {{ .IP }}
	User {{ .User }}
	IdentityFile {{ .KeyPath }}
`)).Execute(os.Stdout, params)
	}

	sshPath, sshErr := exec.LookPath("ssh")
	args := []string{"ssh", "-i", cred.KeyPath, fmt.Sprintf("%s@%s", cred.User, cred.IP)}
	if sshErr == nil {
		if printSSHCLIFlag {
			fmt.Println(sshPath + " " + strings.Join(args[1:], " "))
			return nil
		}
		logger.Infof("Login as '%s' on '%s', using keypair '%s' with ssh client at '%s'", cred.User, cred.IP, cred.KeyPath, sshPath)
		return syscall.Exec(sshPath, args, os.Environ())
	} else { // Fallback SSH
		if printSSHCLIFlag {
			fmt.Println(strings.Join(args, " "))
			return nil
		}
		logger.Infof("No SSH. Fallback on builtin client. Login as '%s' on '%s', using keypair '%s'", cred.User, cred.IP, cred.KeyPath)
		return console.InteractiveTerminal(sshClient)
	}
}

func fetchConnectionInfo() (*graph.Graph, net.IP) {
	var resourcesGraph, sgroupsGraph *graph.Graph
	var myip net.IP
	var wg sync.WaitGroup
	var errc = make(chan error)

	wg.Add(1)
	go func() {
		var err error
		defer wg.Done()
		resourcesGraph, err = aws.InfraService.FetchByType(cloud.Instance)
		if err != nil {
			errc <- err
		}
	}()

	wg.Add(1)
	go func() {
		var err error
		defer wg.Done()
		sgroupsGraph, err = aws.InfraService.FetchByType(cloud.SecurityGroup)
		if err != nil {
			errc <- err
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		client := &http.Client{
			Timeout: 2 * time.Second,
		}
		resp, err := client.Get("http://checkip.amazonaws.com/")
		if err == nil {
			b, _ := ioutil.ReadAll(resp.Body)
			resp.Body.Close()
			myip = net.ParseIP(strings.TrimSpace(string(b)))
		}
	}()
	go func() {
		wg.Wait()
		close(errc)
	}()
	for err := range errc {
		if err != nil {
			exitOn(err)
		}
	}
	resourcesGraph.AddGraph(sgroupsGraph)

	return resourcesGraph, myip

}

func checkInstanceAccessible(g *graph.Graph, inst *graph.Resource, myip net.IP) {
	state, ok := inst.Properties[properties.State]
	if st := fmt.Sprint(state); ok && st != "running" {
		logger.Warningf("This instance is '%s' (cannot ssh to a non running state)", st)
		if st == "stopped" {
			logger.Warningf("You can start it with `awless -f start instance id=%s`", inst.Id())
		}
		return
	}

	sgroups, ok := inst.Properties[properties.SecurityGroups].([]string)
	if ok {
		var sshPortOpen, myIPAllowed bool
		for _, id := range sgroups {
			sgroup, err := findResource(g, id, cloud.SecurityGroup)
			if err != nil {
				break
			}

			rules, ok := sgroup.Properties[properties.InboundRules].([]*graph.FirewallRule)
			if ok {
				for _, r := range rules {
					if r.PortRange.Contains(22) {
						sshPortOpen = true
					}
					if myip != nil && r.Contains(myip.String()) {
						myIPAllowed = true
					}
				}
			}
		}

		if !sshPortOpen {
			logger.Warning("Port 22 is not open on this instance")
		}
		if !myIPAllowed && myip != nil {
			logger.Warningf("Your ip %s is not authorized for this instance. You might want to update the securitygroup with:", myip)
			var group = "mygroup"
			if len(sgroups) == 1 {
				group = sgroups[0]
			}
			logger.Warningf("`awless update securitygroup id=%s inbound=authorize protocol=tcp cidr=%s/32 portrange=22`", group, myip)
		}
	}
}

func findResource(g *graph.Graph, id, typ string) (*graph.Resource, error) {
	if found, err := g.FindResource(id); found == nil || err != nil {
		return nil, fmt.Errorf("instance '%s' not found", id)
	}

	return g.GetResource(typ, id)
}
<|endoftext|>


<commit_before>package utils

import (
	"strconv"
	"sync/atomic"
	"time"
)

// v2 types

type NonceGenerator interface {
	GetNonce() string
}

type EpochNonceGenerator struct {
	nonce uint64
}

// GetNonce is a naive nonce producer that takes the current Unix nano epoch
// and counts upwards.
// This is a naive approach because the nonce bound to the currently used API
// key and as such needs to be synchronised with other instances using the same
// key in order to avoid race conditions.
func (u *EpochNonceGenerator) GetNonce() string {
	return strconv.FormatUint(atomic.AddUint64(&u.nonce, 1), 10)
}

func NewEpochNonceGenerator() *EpochNonceGenerator {
	return &EpochNonceGenerator{
		nonce: uint64(time.Now().Unix()) * 1000,
	}
}

// v1 support

var nonce uint64

func init() {
	nonce = uint64(time.Now().UnixNano()) * 1000
}

// GetNonce is a naive nonce producer that takes the current Unix nano epoch
// and counts upwards.
// This is a naive approach because the nonce bound to the currently used API
// key and as such needs to be synchronised with other instances using the same
// key in order to avoid race conditions.
func GetNonce() string {
	return strconv.FormatUint(atomic.AddUint64(&nonce, 1), 10)
}
<commit_msg>removed utils/none.go<commit_after><|endoftext|>


<commit_before>package utils

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/yaml.v2"
)

//NewDefaultManager -
func NewDefaultManager() (mgr Manager) {
	return &DefaultManager{}
}

//FindFiles -
func (m *DefaultManager) FindFiles(configDir, pattern string) (files []string, err error) {
	m.filePattern = pattern
	err = filepath.Walk(configDir, m.walkDirectories)
	files = m.filePaths
	return
}

func (m *DefaultManager) walkDirectories(path string, info os.FileInfo, e error) (err error) {
	if strings.Contains(path, m.filePattern) {
		m.filePaths = append(m.filePaths, path)
	}
	return e
}

//FileOrDirectoryExists - checks if file exists
func (m *DefaultManager) FileOrDirectoryExists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if os.IsNotExist(err) {
		return false
	}
	return true
}

//LoadFile -
func (m *DefaultManager) LoadFile(configFile string, dataType interface{}) (err error) {
	var data []byte
	if data, err = ioutil.ReadFile(configFile); err == nil {
		err = yaml.Unmarshal(data, dataType)
	}
	return
}

//WriteFileBytes -
func (m *DefaultManager) WriteFileBytes(configFile string, data []byte) (err error) {
	err = ioutil.WriteFile(configFile, data, 0755)
	return
}

//WriteFile -
func (m *DefaultManager) WriteFile(configFile string, dataType interface{}) (err error) {
	var data []byte
	if data, err = yaml.Marshal(dataType); err == nil {
		err = m.WriteFileBytes(configFile, data)
	}
	return
}

//Manager -
type Manager interface {
	FindFiles(directoryName, pattern string) (files []string, err error)
	LoadFile(configFile string, dataType interface{}) (err error)
	WriteFile(configFile string, dataType interface{}) (err error)
	WriteFileBytes(configFile string, data []byte) (err error)
	FileOrDirectoryExists(path string) bool
}

//DefaultManager -
type DefaultManager struct {
	filePattern string
	filePaths   []string
}
<commit_msg>utils: simplify<commit_after>package utils

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/yaml.v2"
)

//NewDefaultManager -
func NewDefaultManager() (mgr Manager) {
	return &DefaultManager{}
}

//FindFiles -
func (m *DefaultManager) FindFiles(configDir, pattern string) ([]string, error) {
	m.filePattern = pattern
	err := filepath.Walk(configDir, m.walkDirectories)
	return m.filePaths, err
}

func (m *DefaultManager) walkDirectories(path string, info os.FileInfo, e error) error {
	if strings.Contains(path, m.filePattern) {
		m.filePaths = append(m.filePaths, path)
	}
	return e
}

//FileOrDirectoryExists - checks if file exists
func (m *DefaultManager) FileOrDirectoryExists(path string) bool {
	_, err := os.Stat(path)
	return !os.IsNotExist(err)
}

//LoadFile -
func (m *DefaultManager) LoadFile(configFile string, dataType interface{}) error {
	var data []byte
	data, err := ioutil.ReadFile(configFile)
	if err != nil {
		return err
	}
	return yaml.Unmarshal(data, dataType)
}

//WriteFileBytes -
func (m *DefaultManager) WriteFileBytes(configFile string, data []byte) error {
	return ioutil.WriteFile(configFile, data, 0755)
}

//WriteFile -
func (m *DefaultManager) WriteFile(configFile string, dataType interface{}) error {
	data, err := yaml.Marshal(dataType)
	if err != nil {
		return err
	}
	return m.WriteFileBytes(configFile, data)
}

//Manager -
type Manager interface {
	FindFiles(directoryName, pattern string) ([]string, error)
	LoadFile(configFile string, dataType interface{}) error
	WriteFile(configFile string, dataType interface{}) error
	WriteFileBytes(configFile string, data []byte) error
	FileOrDirectoryExists(path string) bool
}

//DefaultManager -
type DefaultManager struct {
	filePattern string
	filePaths   []string
}
<|endoftext|>


<commit_before>package utils

import (
	"encoding/base64"
	"fmt"
	"io"
	"mime"
	"net/http"
	"os"
	"strconv"
)

var (
	GITCOMMIT string = "0"
	VERSION   string = "0.1"

	IAMSTATIC string = "true"
	INITSHA1  string = ""
	INITPATH  string = ""
)

const (
	APIVERSION = "1.17"
)

func MatchesContentType(contentType, expectedType string) bool {
	mimetype, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		fmt.Printf("Error parsing media type: %s error: %v", contentType, err)
	}
	return err == nil && mimetype == expectedType
}

func DownloadFile(uri, target string) error {
	f, err := os.OpenFile(target, os.O_RDWR|os.O_CREATE, 0666)
	stat, err := f.Stat()
	if err != nil {
		return err
	}

	req, _ := http.NewRequest("GET", uri, nil)
	req.Header.Set("Range", "bytes="+strconv.FormatInt(stat.Size(), 10)+"-")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	_, err = io.Copy(f, resp.Body)
	if err != nil {
		return err
	}
	return nil
}

func Base64Decode(fileContent string) (string, error) {
	b64 := base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")
	decodeBytes, err := b64.DecodeString(fileContent)
	if err != nil {
		return "", err
	}
	return string(decodeBytes), nil
}

// FormatMountLabel returns a string to be used by the mount command.
// The format of this string will be used to alter the labeling of the mountpoint.
// The string returned is suitable to be used as the options field of the mount command.
// If you need to have additional mount point options, you can pass them in as
// the first parameter.  Second parameter is the label that you wish to apply
// to all content in the mount point.
func FormatMountLabel(src, mountLabel string) string {
	if mountLabel != "" {
		switch src {
		case "":
			src = fmt.Sprintf("context=%q", mountLabel)
		default:
			src = fmt.Sprintf("%s,context=%q", src, mountLabel)
		}
	}
	return src
}
<commit_msg>bump version to 0.2<commit_after>package utils

import (
	"encoding/base64"
	"fmt"
	"io"
	"mime"
	"net/http"
	"os"
	"strconv"
)

var (
	GITCOMMIT string = "0"
	VERSION   string = "0.2"

	IAMSTATIC string = "true"
	INITSHA1  string = ""
	INITPATH  string = ""
)

const (
	APIVERSION = "1.17"
)

func MatchesContentType(contentType, expectedType string) bool {
	mimetype, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		fmt.Printf("Error parsing media type: %s error: %v", contentType, err)
	}
	return err == nil && mimetype == expectedType
}

func DownloadFile(uri, target string) error {
	f, err := os.OpenFile(target, os.O_RDWR|os.O_CREATE, 0666)
	stat, err := f.Stat()
	if err != nil {
		return err
	}

	req, _ := http.NewRequest("GET", uri, nil)
	req.Header.Set("Range", "bytes="+strconv.FormatInt(stat.Size(), 10)+"-")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	_, err = io.Copy(f, resp.Body)
	if err != nil {
		return err
	}
	return nil
}

func Base64Decode(fileContent string) (string, error) {
	b64 := base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")
	decodeBytes, err := b64.DecodeString(fileContent)
	if err != nil {
		return "", err
	}
	return string(decodeBytes), nil
}

// FormatMountLabel returns a string to be used by the mount command.
// The format of this string will be used to alter the labeling of the mountpoint.
// The string returned is suitable to be used as the options field of the mount command.
// If you need to have additional mount point options, you can pass them in as
// the first parameter.  Second parameter is the label that you wish to apply
// to all content in the mount point.
func FormatMountLabel(src, mountLabel string) string {
	if mountLabel != "" {
		switch src {
		case "":
			src = fmt.Sprintf("context=%q", mountLabel)
		default:
			src = fmt.Sprintf("%s,context=%q", src, mountLabel)
		}
	}
	return src
}
<|endoftext|>


<commit_before>package common

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"time"
)

type BuildState string

type GitStrategy int

const (
	GitClone GitStrategy = iota
	GitFetch
)

const (
	Pending BuildState = "pending"
	Running            = "running"
	Failed             = "failed"
	Success            = "success"
)

type Build struct {
	GetBuildResponse `yaml:",inline"`

	Trace        BuildTrace
	BuildAbort   chan os.Signal `json:"-" yaml:"-"`
	RootDir      string         `json:"-" yaml:"-"`
	BuildDir     string         `json:"-" yaml:"-"`
	CacheDir     string         `json:"-" yaml:"-"`
	Hostname     string         `json:"-" yaml:"-"`
	Runner       *RunnerConfig  `json:"runner"`
	ExecutorData ExecutorData

	// Unique ID for all running builds on this runner
	RunnerID int `json:"runner_id"`

	// Unique ID for all running builds on this runner and this project
	ProjectRunnerID int `json:"project_runner_id"`
}

func (b *Build) Log() *logrus.Entry {
	return b.Runner.Log().WithField("build", b.ID).WithField("project", b.ProjectID)
}

func (b *Build) ProjectUniqueName() string {
	return fmt.Sprintf("runner-%s-project-%d-concurrent-%d",
		b.Runner.ShortDescription(), b.ProjectID, b.ProjectRunnerID)
}

func (b *Build) ProjectSlug() (string, error) {
	url, err := url.Parse(b.RepoURL)
	if err != nil {
		return "", err
	}
	if url.Host == "" {
		return "", errors.New("only URI reference supported")
	}

	slug := url.Path
	slug = strings.TrimSuffix(slug, ".git")
	slug = path.Clean(slug)
	if slug == "." {
		return "", errors.New("invalid path")
	}
	if strings.Contains(slug, "..") {
		return "", errors.New("it doesn't look like a valid path")
	}
	return slug, nil
}

func (b *Build) ProjectUniqueDir(sharedDir bool) string {
	dir, err := b.ProjectSlug()
	if err != nil {
		dir = fmt.Sprintf("project-%d", b.ProjectID)
	}

	// for shared dirs path is constructed like this:
	// <some-path>/runner-short-id/concurrent-id/group-name/project-name/
	// ex.<some-path>/01234567/0/group/repo/
	if sharedDir {
		dir = path.Join(
			fmt.Sprintf("%s", b.Runner.ShortDescription()),
			fmt.Sprintf("%d", b.ProjectRunnerID),
			dir,
		)
	}
	return dir
}

func (b *Build) FullProjectDir() string {
	return helpers.ToSlash(b.BuildDir)
}

func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {
	b.RootDir = rootDir
	b.BuildDir = path.Join(rootDir, b.ProjectUniqueDir(sharedDir))
	b.CacheDir = path.Join(cacheDir, b.ProjectUniqueDir(false))
}

func (b *Build) executeShellScript(scriptType ShellScriptType, executor Executor, abort chan interface{}) error {
	shell := executor.Shell()
	if shell == nil {
		return errors.New("No shell defined")
	}

	script, err := GenerateShellScript(scriptType, *shell)
	if err != nil {
		return err
	}

	// Nothing to execute
	if script == "" {
		return nil
	}

	cmd := ExecutorCommand{
		Script: script,
		Abort:  abort,
	}

	switch scriptType {
	case ShellBuildScript, ShellAfterScript: // use custom build environment
		cmd.Predefined = false
	default: // all other stages use a predefined build environment
		cmd.Predefined = true
	}

	return executor.Run(cmd)
}

func (b *Build) executeUploadArtifacts(state error, executor Executor, abort chan interface{}) (err error) {
	when, _ := b.Options.GetString("artifacts", "when")

	if state == nil {
		// Previous stages were successful
		if when == "" || when == "on_success" || when == "always" {
			err = b.executeShellScript(ShellUploadArtifacts, executor, abort)
		}
	} else {
		// Previous stage did fail
		if when == "on_failure" || when == "always" {
			err = b.executeShellScript(ShellUploadArtifacts, executor, abort)
		}
	}

	// Use previous error if set
	if state != nil {
		err = state
	}
	return
}

func (b *Build) executeScript(executor Executor, abort chan interface{}) error {
	// Execute pre script (git clone, cache restore, artifacts download)
	err := b.executeShellScript(ShellPrepareScript, executor, abort)

	if err == nil {
		// Execute user build script (before_script + script)
		err = b.executeShellScript(ShellBuildScript, executor, abort)

		// Execute after script (after_script)
		timeoutCh := make(chan interface{}, 1)
		timeout := time.AfterFunc(time.Minute*5, func() {
			close(timeoutCh)
		})
		b.executeShellScript(ShellAfterScript, executor, timeoutCh)
		timeout.Stop()
	}

	// Execute post script (cache store, artifacts upload)
	if err == nil {
		err = b.executeShellScript(ShellArchiveCache, executor, abort)
	}
	err = b.executeUploadArtifacts(err, executor, abort)
	return err
}

func (b *Build) run(executor Executor) (err error) {
	buildTimeout := b.Timeout
	if buildTimeout <= 0 {
		buildTimeout = DefaultTimeout
	}

	buildCanceled := make(chan bool)
	buildFinish := make(chan error)
	buildAbort := make(chan interface{})

	// Wait for cancel notification
	b.Trace.Notify(func() {
		buildCanceled <- true
	})

	// Run build script
	go func() {
		buildFinish <- b.executeScript(executor, buildAbort)
	}()

	// Wait for signals: cancel, timeout, abort or finish
	b.Log().Debugln("Waiting for signals...")
	select {
	case <-buildCanceled:
		err = errors.New("canceled")

	case <-time.After(time.Duration(buildTimeout) * time.Second):
		err = fmt.Errorf("execution took longer than %v seconds", buildTimeout)

	case signal := <-b.BuildAbort:
		err = fmt.Errorf("aborted: %v", signal)

	case err = <-buildFinish:
		return err
	}

	b.Log().Debugln("Waiting for build to finish...", err)

	// Wait till we receive that build did finish
	for {
		select {
		case buildAbort <- true:
		case <-buildFinish:
			return err
		}
	}
}

func (b *Build) Run(globalConfig *Config, trace BuildTrace) (err error) {
	defer func() {
		if err != nil {
			trace.Fail(err)
		} else {
			trace.Success()
		}
	}()
	b.Trace = trace

	executor := NewExecutor(b.Runner.Executor)
	if executor == nil {
		fmt.Fprint(trace, "Executor not found:", b.Runner.Executor)
		return errors.New("executor not found")
	}
	defer executor.Cleanup()

	err = executor.Prepare(globalConfig, b.Runner, b)
	if err == nil {
		err = b.run(executor)
	}
	executor.Finish(err)
	return err
}

func (b *Build) String() string {
	return helpers.ToYAML(b)
}

func (b *Build) GetDefaultVariables() BuildVariables {
	return BuildVariables{
		{"CI", "true", true, true, false},
		{"CI_BUILD_REF", b.Sha, true, true, false},
		{"CI_BUILD_BEFORE_SHA", b.BeforeSha, true, true, false},
		{"CI_BUILD_REF_NAME", b.RefName, true, true, false},
		{"CI_BUILD_ID", strconv.Itoa(b.ID), true, true, false},
		{"CI_BUILD_REPO", b.RepoURL, true, true, false},
		{"CI_BUILD_TOKEN", b.Token, true, true, false},
		{"CI_PROJECT_ID", strconv.Itoa(b.ProjectID), true, true, false},
		{"CI_PROJECT_DIR", b.FullProjectDir(), true, true, false},
		{"CI_SERVER", "yes", true, true, false},
		{"CI_SERVER_NAME", "GitLab CI", true, true, false},
		{"CI_SERVER_VERSION", "", true, true, false},
		{"CI_SERVER_REVISION", "", true, true, false},
		{"GITLAB_CI", "true", true, true, false},
	}
}

func (b *Build) GetAllVariables() BuildVariables {
	variables := b.Runner.GetVariables()
	variables = append(variables, b.GetDefaultVariables()...)
	variables = append(variables, b.Variables...)
	return variables.Expand()
}

func (b *Build) GetGitDepth() string {
	return b.GetAllVariables().Get("GIT_DEPTH")
}

func (b *Build) GetGitStrategy() GitStrategy {
	switch b.GetAllVariables().Get("GIT_STRATEGY") {
	case "clone":
		return GitClone

	case "fetch":
		return GitFetch

	default:
		if b.AllowGitFetch {
			return GitFetch
		}

		return GitClone
	}
}
<commit_msg>upload the final build trace before cleanup<commit_after>package common

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"time"
)

type BuildState string

type GitStrategy int

const (
	GitClone GitStrategy = iota
	GitFetch
)

const (
	Pending BuildState = "pending"
	Running            = "running"
	Failed             = "failed"
	Success            = "success"
)

type Build struct {
	GetBuildResponse `yaml:",inline"`

	Trace        BuildTrace
	BuildAbort   chan os.Signal `json:"-" yaml:"-"`
	RootDir      string         `json:"-" yaml:"-"`
	BuildDir     string         `json:"-" yaml:"-"`
	CacheDir     string         `json:"-" yaml:"-"`
	Hostname     string         `json:"-" yaml:"-"`
	Runner       *RunnerConfig  `json:"runner"`
	ExecutorData ExecutorData

	// Unique ID for all running builds on this runner
	RunnerID int `json:"runner_id"`

	// Unique ID for all running builds on this runner and this project
	ProjectRunnerID int `json:"project_runner_id"`
}

func (b *Build) Log() *logrus.Entry {
	return b.Runner.Log().WithField("build", b.ID).WithField("project", b.ProjectID)
}

func (b *Build) ProjectUniqueName() string {
	return fmt.Sprintf("runner-%s-project-%d-concurrent-%d",
		b.Runner.ShortDescription(), b.ProjectID, b.ProjectRunnerID)
}

func (b *Build) ProjectSlug() (string, error) {
	url, err := url.Parse(b.RepoURL)
	if err != nil {
		return "", err
	}
	if url.Host == "" {
		return "", errors.New("only URI reference supported")
	}

	slug := url.Path
	slug = strings.TrimSuffix(slug, ".git")
	slug = path.Clean(slug)
	if slug == "." {
		return "", errors.New("invalid path")
	}
	if strings.Contains(slug, "..") {
		return "", errors.New("it doesn't look like a valid path")
	}
	return slug, nil
}

func (b *Build) ProjectUniqueDir(sharedDir bool) string {
	dir, err := b.ProjectSlug()
	if err != nil {
		dir = fmt.Sprintf("project-%d", b.ProjectID)
	}

	// for shared dirs path is constructed like this:
	// <some-path>/runner-short-id/concurrent-id/group-name/project-name/
	// ex.<some-path>/01234567/0/group/repo/
	if sharedDir {
		dir = path.Join(
			fmt.Sprintf("%s", b.Runner.ShortDescription()),
			fmt.Sprintf("%d", b.ProjectRunnerID),
			dir,
		)
	}
	return dir
}

func (b *Build) FullProjectDir() string {
	return helpers.ToSlash(b.BuildDir)
}

func (b *Build) StartBuild(rootDir, cacheDir string, sharedDir bool) {
	b.RootDir = rootDir
	b.BuildDir = path.Join(rootDir, b.ProjectUniqueDir(sharedDir))
	b.CacheDir = path.Join(cacheDir, b.ProjectUniqueDir(false))
}

func (b *Build) executeShellScript(scriptType ShellScriptType, executor Executor, abort chan interface{}) error {
	shell := executor.Shell()
	if shell == nil {
		return errors.New("No shell defined")
	}

	script, err := GenerateShellScript(scriptType, *shell)
	if err != nil {
		return err
	}

	// Nothing to execute
	if script == "" {
		return nil
	}

	cmd := ExecutorCommand{
		Script: script,
		Abort:  abort,
	}

	switch scriptType {
	case ShellBuildScript, ShellAfterScript: // use custom build environment
		cmd.Predefined = false
	default: // all other stages use a predefined build environment
		cmd.Predefined = true
	}

	return executor.Run(cmd)
}

func (b *Build) executeUploadArtifacts(state error, executor Executor, abort chan interface{}) (err error) {
	when, _ := b.Options.GetString("artifacts", "when")

	if state == nil {
		// Previous stages were successful
		if when == "" || when == "on_success" || when == "always" {
			err = b.executeShellScript(ShellUploadArtifacts, executor, abort)
		}
	} else {
		// Previous stage did fail
		if when == "on_failure" || when == "always" {
			err = b.executeShellScript(ShellUploadArtifacts, executor, abort)
		}
	}

	// Use previous error if set
	if state != nil {
		err = state
	}
	return
}

func (b *Build) executeScript(executor Executor, abort chan interface{}) error {
	// Execute pre script (git clone, cache restore, artifacts download)
	err := b.executeShellScript(ShellPrepareScript, executor, abort)

	if err == nil {
		// Execute user build script (before_script + script)
		err = b.executeShellScript(ShellBuildScript, executor, abort)

		// Execute after script (after_script)
		timeoutCh := make(chan interface{}, 1)
		timeout := time.AfterFunc(time.Minute*5, func() {
			close(timeoutCh)
		})
		b.executeShellScript(ShellAfterScript, executor, timeoutCh)
		timeout.Stop()
	}

	// Execute post script (cache store, artifacts upload)
	if err == nil {
		err = b.executeShellScript(ShellArchiveCache, executor, abort)
	}
	err = b.executeUploadArtifacts(err, executor, abort)
	return err
}

func (b *Build) run(executor Executor) (err error) {
	buildTimeout := b.Timeout
	if buildTimeout <= 0 {
		buildTimeout = DefaultTimeout
	}

	buildCanceled := make(chan bool)
	buildFinish := make(chan error)
	buildAbort := make(chan interface{})

	// Wait for cancel notification
	b.Trace.Notify(func() {
		buildCanceled <- true
	})

	// Run build script
	go func() {
		buildFinish <- b.executeScript(executor, buildAbort)
	}()

	// Wait for signals: cancel, timeout, abort or finish
	b.Log().Debugln("Waiting for signals...")
	select {
	case <-buildCanceled:
		err = errors.New("canceled")

	case <-time.After(time.Duration(buildTimeout) * time.Second):
		err = fmt.Errorf("execution took longer than %v seconds", buildTimeout)

	case signal := <-b.BuildAbort:
		err = fmt.Errorf("aborted: %v", signal)

	case err = <-buildFinish:
		return err
	}

	b.Log().Debugln("Waiting for build to finish...", err)

	// Wait till we receive that build did finish
	for {
		select {
		case buildAbort <- true:
		case <-buildFinish:
			return err
		}
	}
}

func (b *Build) Run(globalConfig *Config, trace BuildTrace) (err error) {
	var executor Executor

	defer func() {
		if err != nil {
			trace.Fail(err)
		} else {
			trace.Success()
		}
		if executor != nil {
			executor.Cleanup()
		}
	}()

	b.Trace = trace
	executor = NewExecutor(b.Runner.Executor)
	if executor == nil {
		fmt.Fprint(trace, "Executor not found:", b.Runner.Executor)
		return errors.New("executor not found")
	}

	err = executor.Prepare(globalConfig, b.Runner, b)
	if err == nil {
		err = b.run(executor)
	}
	executor.Finish(err)
	return err
}

func (b *Build) String() string {
	return helpers.ToYAML(b)
}

func (b *Build) GetDefaultVariables() BuildVariables {
	return BuildVariables{
		{"CI", "true", true, true, false},
		{"CI_BUILD_REF", b.Sha, true, true, false},
		{"CI_BUILD_BEFORE_SHA", b.BeforeSha, true, true, false},
		{"CI_BUILD_REF_NAME", b.RefName, true, true, false},
		{"CI_BUILD_ID", strconv.Itoa(b.ID), true, true, false},
		{"CI_BUILD_REPO", b.RepoURL, true, true, false},
		{"CI_BUILD_TOKEN", b.Token, true, true, false},
		{"CI_PROJECT_ID", strconv.Itoa(b.ProjectID), true, true, false},
		{"CI_PROJECT_DIR", b.FullProjectDir(), true, true, false},
		{"CI_SERVER", "yes", true, true, false},
		{"CI_SERVER_NAME", "GitLab CI", true, true, false},
		{"CI_SERVER_VERSION", "", true, true, false},
		{"CI_SERVER_REVISION", "", true, true, false},
		{"GITLAB_CI", "true", true, true, false},
	}
}

func (b *Build) GetAllVariables() BuildVariables {
	variables := b.Runner.GetVariables()
	variables = append(variables, b.GetDefaultVariables()...)
	variables = append(variables, b.Variables...)
	return variables.Expand()
}

func (b *Build) GetGitDepth() string {
	return b.GetAllVariables().Get("GIT_DEPTH")
}

func (b *Build) GetGitStrategy() GitStrategy {
	switch b.GetAllVariables().Get("GIT_STRATEGY") {
	case "clone":
		return GitClone

	case "fetch":
		return GitFetch

	default:
		if b.AllowGitFetch {
			return GitFetch
		}

		return GitClone
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"fmt"
	"strings"
	"unicode/utf8"

	"golang.org/x/text/width"
)

// Error type which references a location within source and a message.
type Error struct {
	Location Location
	Message  string
}

const (
	dot = "."
	ind = "^"
)

var (
	wideDot = width.Widen.String(dot)
	wideInd = width.Widen.String(ind)
)

// ToDisplayString decorates the error message with the source location.
func (e *Error) ToDisplayString(source Source) string {
	var result = fmt.Sprintf("ERROR: %s:%d:%d: %s",
		source.Description(),
		e.Location.Line(),
		e.Location.Column()+1, // add one to the 0-based column for display
		e.Message)
	if snippet, found := source.Snippet(e.Location.Line()); found {
		snippet := strings.Replace(snippet, "\t", " ", -1)
		srcLine := "\n | " + snippet
		var bytes = []byte(snippet)
		var indLine = "\n | "
		for i := 0; i < e.Location.Column() && len(bytes) > 0; i++ {
			_, sz := utf8.DecodeRune(bytes)
			bytes = bytes[sz:]
			if sz > 1 {
				indLine += wideDot
			} else {
				indLine += dot
			}
		}
		if _, sz := utf8.DecodeRune(bytes); sz > 1 {
			indLine += wideInd
		} else {
			indLine += ind
		}
		result += srcLine + indLine
	}
	return result
}
<commit_msg>Omit large error strings (#537)<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"fmt"
	"strings"
	"unicode/utf8"

	"golang.org/x/text/width"
)

// Error type which references a location within source and a message.
type Error struct {
	Location Location
	Message  string
}

const (
	dot = "."
	ind = "^"

	// maxSnippetLength is the largest number of characters which can be rendered in an error message snippet.
	maxSnippetLength = 16384
)

var (
	wideDot = width.Widen.String(dot)
	wideInd = width.Widen.String(ind)
)

// ToDisplayString decorates the error message with the source location.
func (e *Error) ToDisplayString(source Source) string {
	var result = fmt.Sprintf("ERROR: %s:%d:%d: %s",
		source.Description(),
		e.Location.Line(),
		e.Location.Column()+1, // add one to the 0-based column for display
		e.Message)
	if snippet, found := source.Snippet(e.Location.Line()); found && len(snippet) <= maxSnippetLength {
		snippet := strings.Replace(snippet, "\t", " ", -1)
		srcLine := "\n | " + snippet
		var bytes = []byte(snippet)
		var indLine = "\n | "
		for i := 0; i < e.Location.Column() && len(bytes) > 0; i++ {
			_, sz := utf8.DecodeRune(bytes)
			bytes = bytes[sz:]
			if sz > 1 {
				indLine += wideDot
			} else {
				indLine += dot
			}
		}
		if _, sz := utf8.DecodeRune(bytes); sz > 1 {
			indLine += wideInd
		} else {
			indLine += ind
		}
		result += srcLine + indLine
	}
	return result
}
<|endoftext|>


<commit_before>package mdata

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/dgryski/go-tsz"
	"github.com/gocql/gocql"
	"github.com/raintank/met"
	"github.com/raintank/metrictank/cass"
	"github.com/raintank/metrictank/iter"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/worldping-api/pkg/log"
)

// write aggregated data to cassandra.

const Month_sec = 60 * 60 * 24 * 28

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS %s.metric (
    key ascii,
    ts int,
    data blob,
    PRIMARY KEY (key, ts)
) WITH CLUSTERING ORDER BY (ts DESC)
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy', 'compaction_window_unit': 'DAYS', 'compaction_window_size': '1' }
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}`

var (
	errChunkTooSmall      = errors.New("unpossibly small chunk in cassandra")
	errUnknownChunkFormat = errors.New("unrecognized chunk format in cassandra")
	errStartBeforeEnd     = errors.New("start must be before end.")

	cassBlockDuration met.Timer
	cassGetDuration   met.Timer
	cassPutDuration   met.Timer

	cassChunksPerRow    met.Meter
	cassRowsPerResponse met.Meter
	cassToIterDuration  met.Timer

	chunkSaveOk   met.Count
	chunkSaveFail met.Count
	// it's pretty expensive/impossible to do chunk size in mem vs in cassandra etc, but we can more easily measure chunk sizes when we operate on them
	chunkSizeAtSave met.Meter
	chunkSizeAtLoad met.Meter
)

/*
https://godoc.org/github.com/gocql/gocql#Session
Session is the interface used by users to interact with the database.
It's safe for concurrent use by multiple goroutines and a typical usage scenario is to have one global session
object to interact with the whole Cassandra cluster.
*/

type cassandraStore struct {
	session          *gocql.Session
	writeQueues      []chan *ChunkWriteRequest
	readQueue        chan *ChunkReadRequest
	writeQueueMeters []met.Meter
	metrics          cass.Metrics
}

func NewCassandraStore(stats met.Backend, addrs, keyspace, consistency string, timeout, readers, writers, readqsize, writeqsize, protoVer int) (*cassandraStore, error) {
	cluster := gocql.NewCluster(strings.Split(addrs, ",")...)
	cluster.Consistency = gocql.ParseConsistency(consistency)
	cluster.Timeout = time.Duration(timeout) * time.Millisecond
	cluster.NumConns = writers
	cluster.ProtoVersion = protoVer
	var err error
	tmpSession, err := cluster.CreateSession()
	if err != nil {
		return nil, err
	}
	// ensure the keyspace and table exist.
	err = tmpSession.Query(fmt.Sprintf(keyspace_schema, keyspace)).Exec()
	if err != nil {
		return nil, err
	}
	err = tmpSession.Query(fmt.Sprintf(table_schema, keyspace)).Exec()
	if err != nil {
		return nil, err
	}
	tmpSession.Close()
	cluster.Keyspace = keyspace
	session, err := cluster.CreateSession()
	if err != nil {
		return nil, err
	}
	c := &cassandraStore{
		session:          session,
		writeQueues:      make([]chan *ChunkWriteRequest, writers),
		readQueue:        make(chan *ChunkReadRequest, readqsize),
		writeQueueMeters: make([]met.Meter, writers),
		metrics:          cass.Metrics{},
	}

	for i := 0; i < writers; i++ {
		c.writeQueues[i] = make(chan *ChunkWriteRequest, writeqsize)
		c.writeQueueMeters[i] = stats.NewMeter(fmt.Sprintf("cassandra.write_queue.%d.items", i+1), 0)
		go c.processWriteQueue(c.writeQueues[i], c.writeQueueMeters[i])
	}

	for i := 0; i < readers; i++ {
		go c.processReadQueue()
	}

	return c, err
}

func (c *cassandraStore) InitMetrics(stats met.Backend) {
	cassBlockDuration = stats.NewTimer("cassandra.block_duration", 0)
	cassGetDuration = stats.NewTimer("cassandra.get_duration", 0)
	cassPutDuration = stats.NewTimer("cassandra.put_duration", 0)

	cassChunksPerRow = stats.NewMeter("cassandra.chunks_per_row", 0)
	cassRowsPerResponse = stats.NewMeter("cassandra.rows_per_response", 0)
	cassToIterDuration = stats.NewTimer("cassandra.to_iter_duration", 0)

	chunkSaveOk = stats.NewCount("chunks.save_ok")
	chunkSaveFail = stats.NewCount("chunks.save_fail")
	chunkSizeAtSave = stats.NewMeter("chunk_size.at_save", 0)
	chunkSizeAtLoad = stats.NewMeter("chunk_size.at_load", 0)

	c.metrics.Init("cassandra", stats)
}

func (c *cassandraStore) Add(cwr *ChunkWriteRequest) {
	sum := 0
	for _, char := range cwr.key {
		sum += int(char)
	}
	which := sum % len(c.writeQueues)
	c.writeQueueMeters[which].Value(int64(len(c.writeQueues[which])))
	c.writeQueues[which] <- cwr
	c.writeQueueMeters[which].Value(int64(len(c.writeQueues[which])))
}

/* process writeQueue.
 */
func (c *cassandraStore) processWriteQueue(queue chan *ChunkWriteRequest, meter met.Meter) {
	tick := time.Tick(time.Duration(1) * time.Second)
	for {
		select {
		case <-tick:
			meter.Value(int64(len(queue)))
		case cwr := <-queue:
			meter.Value(int64(len(queue)))
			log.Debug("CS: starting to save %s:%d %v", cwr.key, cwr.chunk.T0, cwr.chunk)
			//log how long the chunk waited in the queue before we attempted to save to cassandra
			cassBlockDuration.Value(time.Now().Sub(cwr.timestamp))

			data := cwr.chunk.Series.Bytes()
			chunkSizeAtSave.Value(int64(len(data)))
			version := chunk.FormatStandardGoTsz
			buf := new(bytes.Buffer)
			binary.Write(buf, binary.LittleEndian, uint8(version))
			buf.Write(data)
			success := false
			attempts := 0
			for !success {
				err := c.insertChunk(cwr.key, cwr.chunk.T0, buf.Bytes(), int(cwr.ttl))
				if err == nil {
					success = true
					cwr.chunk.Saved = true
					SendPersistMessage(cwr.key, cwr.chunk.T0)
					log.Debug("CS: save complete. %s:%d %v", cwr.key, cwr.chunk.T0, cwr.chunk)
					chunkSaveOk.Inc(1)
				} else {
					if (attempts % 20) == 0 {
						log.Warn("CS: failed to save chunk to cassandra after %d attempts. %v, %s", attempts+1, cwr.chunk, err)
					}
					chunkSaveFail.Inc(1)
					sleepTime := 100 * attempts
					if sleepTime > 2000 {
						sleepTime = 2000
					}
					time.Sleep(time.Duration(sleepTime) * time.Millisecond)
					attempts++
				}
			}
		}
	}
}

// Insert Chunks into Cassandra.
//
// key: is the metric_id
// ts: is the start of the aggregated time range.
// data: is the payload as bytes.
func (c *cassandraStore) insertChunk(key string, t0 uint32, data []byte, ttl int) error {
	// for unit tests
	if c.session == nil {
		return nil
	}
	query := fmt.Sprintf("INSERT INTO metric (key, ts, data) values(?,?,?) USING TTL %d", ttl)
	row_key := fmt.Sprintf("%s_%d", key, t0/Month_sec) // "month number" based on unix timestamp (rounded down)
	pre := time.Now()
	ret := c.session.Query(query, row_key, t0, data).Exec()
	cassPutDuration.Value(time.Now().Sub(pre))
	return ret
}

type outcome struct {
	month   uint32
	sortKey uint32
	i       *gocql.Iter
}
type asc []outcome

func (o asc) Len() int           { return len(o) }
func (o asc) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }
func (o asc) Less(i, j int) bool { return o[i].sortKey < o[j].sortKey }

func (c *cassandraStore) processReadQueue() {
	for crr := range c.readQueue {
		crr.out <- outcome{crr.month, crr.sortKey, c.session.Query(crr.q, crr.p...).Iter()}
	}
}

// Basic search of cassandra.
// start inclusive, end exclusive
func (c *cassandraStore) Search(key string, start, end uint32) ([]iter.Iter, error) {
	iters := make([]iter.Iter, 0)
	if start > end {
		return iters, errStartBeforeEnd
	}

	crrs := make([]*ChunkReadRequest, 0)

	query := func(month, sortKey uint32, q string, p ...interface{}) {
		crrs = append(crrs, &ChunkReadRequest{month, sortKey, q, p, nil})
	}

	start_month := start - (start % Month_sec)       // starting row has to be at, or before, requested start
	end_month := (end - 1) - ((end - 1) % Month_sec) // ending row has to be include the last point we might need

	pre := time.Now()

	// unfortunately in the database we only have the t0's of all chunks.
	// this means we can easily make sure to include the correct last chunk (just query for a t0 < end, the last chunk will contain the last needed data)
	// but it becomes hard to find which should be the first chunk to include. we can't just query for start <= t0 because than will miss some data at the beginning
	// we can't assume we know the chunkSpan so we can't just calculate the t0 >= start - <some-predefined-number> because chunkSpans may change over time.
	// we effectively need all chunks with a t0 > start, as well as the last chunk with a t0 <= start.
	// since we make sure that you can only use chunkSpans so that Month_sec % chunkSpan == 0, we know that this previous chunk will always be in the same row
	// as the one that has start_month.

	row_key := fmt.Sprintf("%s_%d", key, start_month/Month_sec)

	query(start_month, start_month, "SELECT ts, data FROM metric WHERE key=? AND ts <= ? Limit 1", row_key, start)

	if start_month == end_month {
		// we need a selection of the row between startTs and endTs
		row_key = fmt.Sprintf("%s_%d", key, start_month/Month_sec)
		query(start_month, start_month+1, "SELECT ts, data FROM metric WHERE key = ? AND ts > ? AND ts < ? ORDER BY ts ASC", row_key, start, end)
	} else {
		// get row_keys for each row we need to query.
		for month := start_month; month <= end_month; month += Month_sec {
			row_key = fmt.Sprintf("%s_%d", key, month/Month_sec)
			if month == start_month {
				// we want from startTs to the end of the row.
				query(month, month+1, "SELECT ts, data FROM metric WHERE key = ? AND ts >= ? ORDER BY ts ASC", row_key, start+1)
			} else if month == end_month {
				// we want from start of the row till the endTs.
				query(month, month, "SELECT ts, data FROM metric WHERE key = ? AND ts <= ? ORDER BY ts ASC", row_key, end-1)
			} else {
				// we want all columns
				query(month, month, "SELECT ts, data FROM metric WHERE key = ? ORDER BY ts ASC", row_key)
			}
		}
	}
	numQueries := len(crrs)
	results := make(chan outcome, numQueries)
	for i := range crrs {
		crrs[i].out = results
		c.readQueue <- crrs[i]
	}
	outcomes := make([]outcome, 0, numQueries)

	seen := 0
	for o := range results {
		seen += 1
		outcomes = append(outcomes, o)
		if seen == numQueries {
			close(results)
			cassGetDuration.Value(time.Now().Sub(pre))
			break
		}
	}
	pre = time.Now()
	// we have all of the results, but they could have arrived in any order.
	sort.Sort(asc(outcomes))

	var b []byte
	var ts int
	for _, outcome := range outcomes {
		chunks := int64(0)
		for outcome.i.Scan(&ts, &b) {
			chunks += 1
			chunkSizeAtLoad.Value(int64(len(b)))
			if len(b) < 2 {
				log.Error(3, errChunkTooSmall.Error())
				return iters, errChunkTooSmall
			}
			if chunk.Format(b[0]) != chunk.FormatStandardGoTsz {
				log.Error(3, errUnknownChunkFormat.Error())
				return iters, errUnknownChunkFormat
			}
			it, err := tsz.NewIterator(b[1:])
			if err != nil {
				log.Error(3, "failed to unpack cassandra payload. %s", err)
				return iters, err
			}
			iters = append(iters, iter.New(it, true))
		}
		err := outcome.i.Close()
		if err != nil {
			log.Error(3, "cassandra query error. %s", err)
			c.metrics.Inc(err)
		} else {
			cassChunksPerRow.Value(chunks)
		}
	}
	cassToIterDuration.Value(time.Now().Sub(pre))
	cassRowsPerResponse.Value(int64(len(outcomes)))
	log.Debug("CS: searchCassandra(): %d outcomes (queries), %d total iters", len(outcomes), len(iters))
	return iters, nil
}

func (c *cassandraStore) Stop() {
	c.session.Close()
}
<commit_msg>also instrument errors when trying to insert chunks<commit_after>package mdata

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/dgryski/go-tsz"
	"github.com/gocql/gocql"
	"github.com/raintank/met"
	"github.com/raintank/metrictank/cass"
	"github.com/raintank/metrictank/iter"
	"github.com/raintank/metrictank/mdata/chunk"
	"github.com/raintank/worldping-api/pkg/log"
)

// write aggregated data to cassandra.

const Month_sec = 60 * 60 * 24 * 28

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS %s.metric (
    key ascii,
    ts int,
    data blob,
    PRIMARY KEY (key, ts)
) WITH CLUSTERING ORDER BY (ts DESC)
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy', 'compaction_window_unit': 'DAYS', 'compaction_window_size': '1' }
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}`

var (
	errChunkTooSmall      = errors.New("unpossibly small chunk in cassandra")
	errUnknownChunkFormat = errors.New("unrecognized chunk format in cassandra")
	errStartBeforeEnd     = errors.New("start must be before end.")

	cassBlockDuration met.Timer
	cassGetDuration   met.Timer
	cassPutDuration   met.Timer

	cassChunksPerRow    met.Meter
	cassRowsPerResponse met.Meter
	cassToIterDuration  met.Timer

	chunkSaveOk   met.Count
	chunkSaveFail met.Count
	// it's pretty expensive/impossible to do chunk size in mem vs in cassandra etc, but we can more easily measure chunk sizes when we operate on them
	chunkSizeAtSave met.Meter
	chunkSizeAtLoad met.Meter
)

/*
https://godoc.org/github.com/gocql/gocql#Session
Session is the interface used by users to interact with the database.
It's safe for concurrent use by multiple goroutines and a typical usage scenario is to have one global session
object to interact with the whole Cassandra cluster.
*/

type cassandraStore struct {
	session          *gocql.Session
	writeQueues      []chan *ChunkWriteRequest
	readQueue        chan *ChunkReadRequest
	writeQueueMeters []met.Meter
	metrics          cass.Metrics
}

func NewCassandraStore(stats met.Backend, addrs, keyspace, consistency string, timeout, readers, writers, readqsize, writeqsize, protoVer int) (*cassandraStore, error) {
	cluster := gocql.NewCluster(strings.Split(addrs, ",")...)
	cluster.Consistency = gocql.ParseConsistency(consistency)
	cluster.Timeout = time.Duration(timeout) * time.Millisecond
	cluster.NumConns = writers
	cluster.ProtoVersion = protoVer
	var err error
	tmpSession, err := cluster.CreateSession()
	if err != nil {
		return nil, err
	}
	// ensure the keyspace and table exist.
	err = tmpSession.Query(fmt.Sprintf(keyspace_schema, keyspace)).Exec()
	if err != nil {
		return nil, err
	}
	err = tmpSession.Query(fmt.Sprintf(table_schema, keyspace)).Exec()
	if err != nil {
		return nil, err
	}
	tmpSession.Close()
	cluster.Keyspace = keyspace
	session, err := cluster.CreateSession()
	if err != nil {
		return nil, err
	}
	c := &cassandraStore{
		session:          session,
		writeQueues:      make([]chan *ChunkWriteRequest, writers),
		readQueue:        make(chan *ChunkReadRequest, readqsize),
		writeQueueMeters: make([]met.Meter, writers),
		metrics:          cass.Metrics{},
	}

	for i := 0; i < writers; i++ {
		c.writeQueues[i] = make(chan *ChunkWriteRequest, writeqsize)
		c.writeQueueMeters[i] = stats.NewMeter(fmt.Sprintf("cassandra.write_queue.%d.items", i+1), 0)
		go c.processWriteQueue(c.writeQueues[i], c.writeQueueMeters[i])
	}

	for i := 0; i < readers; i++ {
		go c.processReadQueue()
	}

	return c, err
}

func (c *cassandraStore) InitMetrics(stats met.Backend) {
	cassBlockDuration = stats.NewTimer("cassandra.block_duration", 0)
	cassGetDuration = stats.NewTimer("cassandra.get_duration", 0)
	cassPutDuration = stats.NewTimer("cassandra.put_duration", 0)

	cassChunksPerRow = stats.NewMeter("cassandra.chunks_per_row", 0)
	cassRowsPerResponse = stats.NewMeter("cassandra.rows_per_response", 0)
	cassToIterDuration = stats.NewTimer("cassandra.to_iter_duration", 0)

	chunkSaveOk = stats.NewCount("chunks.save_ok")
	chunkSaveFail = stats.NewCount("chunks.save_fail")
	chunkSizeAtSave = stats.NewMeter("chunk_size.at_save", 0)
	chunkSizeAtLoad = stats.NewMeter("chunk_size.at_load", 0)

	c.metrics.Init("cassandra", stats)
}

func (c *cassandraStore) Add(cwr *ChunkWriteRequest) {
	sum := 0
	for _, char := range cwr.key {
		sum += int(char)
	}
	which := sum % len(c.writeQueues)
	c.writeQueueMeters[which].Value(int64(len(c.writeQueues[which])))
	c.writeQueues[which] <- cwr
	c.writeQueueMeters[which].Value(int64(len(c.writeQueues[which])))
}

/* process writeQueue.
 */
func (c *cassandraStore) processWriteQueue(queue chan *ChunkWriteRequest, meter met.Meter) {
	tick := time.Tick(time.Duration(1) * time.Second)
	for {
		select {
		case <-tick:
			meter.Value(int64(len(queue)))
		case cwr := <-queue:
			meter.Value(int64(len(queue)))
			log.Debug("CS: starting to save %s:%d %v", cwr.key, cwr.chunk.T0, cwr.chunk)
			//log how long the chunk waited in the queue before we attempted to save to cassandra
			cassBlockDuration.Value(time.Now().Sub(cwr.timestamp))

			data := cwr.chunk.Series.Bytes()
			chunkSizeAtSave.Value(int64(len(data)))
			version := chunk.FormatStandardGoTsz
			buf := new(bytes.Buffer)
			binary.Write(buf, binary.LittleEndian, uint8(version))
			buf.Write(data)
			success := false
			attempts := 0
			for !success {
				err := c.insertChunk(cwr.key, cwr.chunk.T0, buf.Bytes(), int(cwr.ttl))
				if err == nil {
					success = true
					cwr.chunk.Saved = true
					SendPersistMessage(cwr.key, cwr.chunk.T0)
					log.Debug("CS: save complete. %s:%d %v", cwr.key, cwr.chunk.T0, cwr.chunk)
					chunkSaveOk.Inc(1)
				} else {
					c.metrics.Inc(err)
					if (attempts % 20) == 0 {
						log.Warn("CS: failed to save chunk to cassandra after %d attempts. %v, %s", attempts+1, cwr.chunk, err)
					}
					chunkSaveFail.Inc(1)
					sleepTime := 100 * attempts
					if sleepTime > 2000 {
						sleepTime = 2000
					}
					time.Sleep(time.Duration(sleepTime) * time.Millisecond)
					attempts++
				}
			}
		}
	}
}

// Insert Chunks into Cassandra.
//
// key: is the metric_id
// ts: is the start of the aggregated time range.
// data: is the payload as bytes.
func (c *cassandraStore) insertChunk(key string, t0 uint32, data []byte, ttl int) error {
	// for unit tests
	if c.session == nil {
		return nil
	}
	query := fmt.Sprintf("INSERT INTO metric (key, ts, data) values(?,?,?) USING TTL %d", ttl)
	row_key := fmt.Sprintf("%s_%d", key, t0/Month_sec) // "month number" based on unix timestamp (rounded down)
	pre := time.Now()
	ret := c.session.Query(query, row_key, t0, data).Exec()
	cassPutDuration.Value(time.Now().Sub(pre))
	return ret
}

type outcome struct {
	month   uint32
	sortKey uint32
	i       *gocql.Iter
}
type asc []outcome

func (o asc) Len() int           { return len(o) }
func (o asc) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }
func (o asc) Less(i, j int) bool { return o[i].sortKey < o[j].sortKey }

func (c *cassandraStore) processReadQueue() {
	for crr := range c.readQueue {
		crr.out <- outcome{crr.month, crr.sortKey, c.session.Query(crr.q, crr.p...).Iter()}
	}
}

// Basic search of cassandra.
// start inclusive, end exclusive
func (c *cassandraStore) Search(key string, start, end uint32) ([]iter.Iter, error) {
	iters := make([]iter.Iter, 0)
	if start > end {
		return iters, errStartBeforeEnd
	}

	crrs := make([]*ChunkReadRequest, 0)

	query := func(month, sortKey uint32, q string, p ...interface{}) {
		crrs = append(crrs, &ChunkReadRequest{month, sortKey, q, p, nil})
	}

	start_month := start - (start % Month_sec)       // starting row has to be at, or before, requested start
	end_month := (end - 1) - ((end - 1) % Month_sec) // ending row has to be include the last point we might need

	pre := time.Now()

	// unfortunately in the database we only have the t0's of all chunks.
	// this means we can easily make sure to include the correct last chunk (just query for a t0 < end, the last chunk will contain the last needed data)
	// but it becomes hard to find which should be the first chunk to include. we can't just query for start <= t0 because than will miss some data at the beginning
	// we can't assume we know the chunkSpan so we can't just calculate the t0 >= start - <some-predefined-number> because chunkSpans may change over time.
	// we effectively need all chunks with a t0 > start, as well as the last chunk with a t0 <= start.
	// since we make sure that you can only use chunkSpans so that Month_sec % chunkSpan == 0, we know that this previous chunk will always be in the same row
	// as the one that has start_month.

	row_key := fmt.Sprintf("%s_%d", key, start_month/Month_sec)

	query(start_month, start_month, "SELECT ts, data FROM metric WHERE key=? AND ts <= ? Limit 1", row_key, start)

	if start_month == end_month {
		// we need a selection of the row between startTs and endTs
		row_key = fmt.Sprintf("%s_%d", key, start_month/Month_sec)
		query(start_month, start_month+1, "SELECT ts, data FROM metric WHERE key = ? AND ts > ? AND ts < ? ORDER BY ts ASC", row_key, start, end)
	} else {
		// get row_keys for each row we need to query.
		for month := start_month; month <= end_month; month += Month_sec {
			row_key = fmt.Sprintf("%s_%d", key, month/Month_sec)
			if month == start_month {
				// we want from startTs to the end of the row.
				query(month, month+1, "SELECT ts, data FROM metric WHERE key = ? AND ts >= ? ORDER BY ts ASC", row_key, start+1)
			} else if month == end_month {
				// we want from start of the row till the endTs.
				query(month, month, "SELECT ts, data FROM metric WHERE key = ? AND ts <= ? ORDER BY ts ASC", row_key, end-1)
			} else {
				// we want all columns
				query(month, month, "SELECT ts, data FROM metric WHERE key = ? ORDER BY ts ASC", row_key)
			}
		}
	}
	numQueries := len(crrs)
	results := make(chan outcome, numQueries)
	for i := range crrs {
		crrs[i].out = results
		c.readQueue <- crrs[i]
	}
	outcomes := make([]outcome, 0, numQueries)

	seen := 0
	for o := range results {
		seen += 1
		outcomes = append(outcomes, o)
		if seen == numQueries {
			close(results)
			cassGetDuration.Value(time.Now().Sub(pre))
			break
		}
	}
	pre = time.Now()
	// we have all of the results, but they could have arrived in any order.
	sort.Sort(asc(outcomes))

	var b []byte
	var ts int
	for _, outcome := range outcomes {
		chunks := int64(0)
		for outcome.i.Scan(&ts, &b) {
			chunks += 1
			chunkSizeAtLoad.Value(int64(len(b)))
			if len(b) < 2 {
				log.Error(3, errChunkTooSmall.Error())
				return iters, errChunkTooSmall
			}
			if chunk.Format(b[0]) != chunk.FormatStandardGoTsz {
				log.Error(3, errUnknownChunkFormat.Error())
				return iters, errUnknownChunkFormat
			}
			it, err := tsz.NewIterator(b[1:])
			if err != nil {
				log.Error(3, "failed to unpack cassandra payload. %s", err)
				return iters, err
			}
			iters = append(iters, iter.New(it, true))
		}
		err := outcome.i.Close()
		if err != nil {
			log.Error(3, "cassandra query error. %s", err)
			c.metrics.Inc(err)
		} else {
			cassChunksPerRow.Value(chunks)
		}
	}
	cassToIterDuration.Value(time.Now().Sub(pre))
	cassRowsPerResponse.Value(int64(len(outcomes)))
	log.Debug("CS: searchCassandra(): %d outcomes (queries), %d total iters", len(outcomes), len(iters))
	return iters, nil
}

func (c *cassandraStore) Stop() {
	c.session.Close()
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Paul Kramme
package main

import (
  "fmt";
  "os";
  "path/filepath";
  "io";
  "crypto/md5";
  "encoding/hex"
)

//type WalkCallb func(path string, info os.FileInfo, err error) error

func md5sum(filePath string) (result string, err error) {
  file, err := os.Open(filePath)
  if err != nil {
    return
  }
  defer file.Close()

  hash := md5.New()
  _, err = io.Copy(hash, file)
  if err != nil {
    return
  }

  result = hex.EncodeToString(hash.Sum(nil))
  return
}

func scanfiles(location string) (m map[int]string, err error) {
  var walkcallback = func(path string, fileinfo os.FileInfo, inputerror error) (err error) {
    checksum,_ := md5sum(path)
    fmt.Println(path, checksum)
    return
  }
  var something map[int]string
  err = filepath.Walk(location, walkcallback)
  return something, err
}

func main() {
  fmt.Println("MEDIA VALIDATION TOOL")
  fmt.Println("Copyright (C) 2017 Paul Kramme")

  if len(os.Args) > 1 {
    if os.Args[1] == "create" {
      fmt.Println("\n:: Creating media record for current directory\n")
      scanfiles(".")
    } else {
      fmt.Println("Invalid argument:", os.Args[1])
    }
  }
}
<commit_msg>Add check<commit_after>// Copyright (C) 2017 Paul Kramme
package main

import (
  "fmt";
  "os";
  "path/filepath";
  "io";
  "crypto/md5";
  "bufio";
  "strings";
  "encoding/hex";

)

func md5sum(filePath string) (result string, err error) {
  file, err := os.Open(filePath)
  if err != nil {
    return
  }
  defer file.Close()

  hash := md5.New()
  _, err = io.Copy(hash, file)
  if err != nil {
    return
  }

  result = hex.EncodeToString(hash.Sum(nil))
  return
}

func scanfiles(location string) (m map[string]string, err error) {
  m = make(map[string]string)
  var walkcallback = func(path string, fileinfo os.FileInfo, inputerror error) (err error) {
    checksum,_ := md5sum(path)
    m[path] = checksum
    return
  }
  err = filepath.Walk(location, walkcallback)
  return
}

func main() {
  fmt.Println("MEDIA VALIDATION TOOL")
  fmt.Println("Copyright (C) 2017 Paul Kramme")

  if len(os.Args) > 1 {
    if os.Args[1] == "create" {
      fmt.Println("\n:: Creating media record for current directory\n")
      table, err := scanfiles(".")
      if err != nil {
        fmt.Println("Error during scan...")
      }
      //open file
      f, fileopenerror := os.Create("./media_record.csv")
      if fileopenerror != nil {
        panic(fileopenerror)
      }
      for path, hash := range table {
        if table[path] != "" {
          fmt.Fprintf(f, "%s,%s\n", path, hash)
        }
      }
    } else {
      fmt.Println("Invalid argument:", os.Args[1])
    }
  } else {
    fmt.Println("Checking file integrity")
    table, err := scanfiles(".")
    if err != nil {
      fmt.Println("Error during scan...")
    }
    fmt.Println("Checking against media_record.csv")
    file, _ := os.Open("./media_record.csv")
    scanner := bufio.NewScanner(file) //read lines
    old_data := make(map[string]string)
    for scanner.Scan() {
         splitted_string := strings.Split(scanner.Text(), ",")
         if splitted_string[1] != "" {
              old_data[splitted_string[0]] = splitted_string[1]
         }
    }
    if err != nil {
         panic(err)
    }
    for path, _ := range old_data {
         if val, ok := table[path]; ok {
              if val == table[path] {
                   fmt.Printf("SUCCESS %s\n", path)
              } else {
                   fmt.Println("Fail.")
              }
         }
    }
  }
}
<|endoftext|>


<commit_before>// The timeout package provides an http.Client that closes a connection if it takes
// too long to establish, or stays idle for too long.
package timeout

import (
	"net"
	"net/http"
	"time"

	"github.com/efarrer/iothrottler"
	"github.com/getlantern/idletiming"
	"github.com/pkg/errors"
)

const (
	DefaultConnectTimeout time.Duration = 30 * time.Second
	DefaultIdleTimeout                  = 60 * time.Second
)

var ThrottlerPool *iothrottler.IOThrottlerPool

func init() {
	ThrottlerPool = iothrottler.NewIOThrottlerPool(iothrottler.Unlimited)
}

func timeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (net.Conn, error) {
	return func(netw, addr string) (net.Conn, error) {
		// if it takes too long to establish a connection, give up
		timeoutConn, err := net.DialTimeout(netw, addr, cTimeout)
		if err != nil {
			return nil, errors.WithStack(err)
		}
		// respect global throttle settings
		throttledConn, err := ThrottlerPool.AddConn(timeoutConn)
		if err != nil {
			return nil, errors.WithStack(err)
		}
		// measure bps
		monitorConn := &monitoringConn{
			Conn: throttledConn,
		}
		// if we stay idle too long, close
		idleConn := idletiming.Conn(monitorConn, rwTimeout, func() {
			monitorConn.Close()
		})
		return idleConn, nil
	}
}

func NewClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial:  timeoutDialer(connectTimeout, readWriteTimeout),
		},
	}
}

func NewDefaultClient() *http.Client {
	return NewClient(DefaultConnectTimeout, DefaultIdleTimeout)
}
<commit_msg>Add simulated offline mode<commit_after>// The timeout package provides an http.Client that closes a connection if it takes
// too long to establish, or stays idle for too long.
package timeout

import (
	"net"
	"net/http"
	"time"

	"github.com/efarrer/iothrottler"
	"github.com/getlantern/idletiming"
	"github.com/pkg/errors"
)

const (
	DefaultConnectTimeout time.Duration = 30 * time.Second
	DefaultIdleTimeout                  = 60 * time.Second
)

var ThrottlerPool *iothrottler.IOThrottlerPool

func init() {
	ThrottlerPool = iothrottler.NewIOThrottlerPool(iothrottler.Unlimited)
}

var simulateOffline bool = false
var conns = make(map[net.Conn]bool)

func SetSimulateOffline(enabled bool) {
	simulateOffline = enabled
}

func timeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (net.Conn, error) {
	return func(netw, addr string) (net.Conn, error) {
		if simulateOffline {
			return nil, &net.OpError{
				Op:  "dial",
				Err: errors.New("simulated offline"),
			}
		}

		// if it takes too long to establish a connection, give up
		timeoutConn, err := net.DialTimeout(netw, addr, cTimeout)
		if err != nil {
			return nil, errors.WithStack(err)
		}
		// respect global throttle settings
		throttledConn, err := ThrottlerPool.AddConn(timeoutConn)
		if err != nil {
			return nil, errors.WithStack(err)
		}
		// measure bps
		monitorConn := &monitoringConn{
			Conn: throttledConn,
		}
		// if we stay idle too long, close
		idleConn := idletiming.Conn(monitorConn, rwTimeout, func() {
			monitorConn.Close()
		})
		return idleConn, nil
	}
}

func NewClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial:  timeoutDialer(connectTimeout, readWriteTimeout),
		},
	}
}

func NewDefaultClient() *http.Client {
	return NewClient(DefaultConnectTimeout, DefaultIdleTimeout)
}
<|endoftext|>


<commit_before>package command

import (
	"strings"
)

type NewCommand struct {
	Meta
}

func (c *NewCommand) Run(args []string) int {
	// Write your code here

	return 0
}

func (c *NewCommand) Synopsis() string {
	return ""
}

func (c *NewCommand) Help() string {
	helpText := `

`
	return strings.TrimSpace(helpText)
}
<commit_msg>Add support for new subcommand<commit_after>package command

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/tcnksm/go-gitconfig"
	"github.com/wantedly/apig/apig"
)

const defaultVCS = "github.com"

type NewCommand struct {
	Meta

	project  string
	vcs      string
	username string
}

func (c *NewCommand) Run(args []string) int {
	if err := c.parseArgs(args); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}
	if c.vcs == "" {
		c.vcs = defaultVCS
	}
	if c.username == "" {
		var err error
		c.username, err = gitconfig.GithubUser()
		if err != nil {
			c.username, err = gitconfig.Username()
			if err != nil || strings.Contains(c.username, " ") {
				msg := "Cannot find github username in `~/.gitconfig` file.\n" +
					"Please use -u option"
				fmt.Fprintln(os.Stderr, msg)
				return 1
			}
		}
	}

	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		fmt.Fprintln(os.Stderr, "Error: $GOPATH is not found")
		return 1
	}

	return apig.Skeleton(gopath, c.vcs, c.username, c.project)
}

func (c *NewCommand) parseArgs(args []string) error {
	flag := flag.NewFlagSet("apig", flag.ContinueOnError)

	flag.StringVar(&c.vcs, "v", "", "VCS")
	flag.StringVar(&c.vcs, "vcs", "", "VCS")
	flag.StringVar(&c.username, "u", "", "Username")
	flag.StringVar(&c.username, "user", "", "Username")

	if err := flag.Parse(args); err != nil {
		return err
	}
	for 0 < flag.NArg() {
		c.project = flag.Arg(0)
		flag.Parse(flag.Args()[1:])
	}
	if c.project == "" {
		err := errors.New("Please specify project name.")
		return err
	}
	return nil
}

func (c *NewCommand) Synopsis() string {
	return ""
}

func (c *NewCommand) Help() string {
	helpText := `

`
	return strings.TrimSpace(helpText)
}
<|endoftext|>


<commit_before>package todolist

import (
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type Parser struct {
	input string
}

func (p *Parser) ParseNewTodo(input string) *Todo {
	r, _ := regexp.Compile(`^(add|a)(\\ |) `)
	input = r.ReplaceAllString(input, "")
	if input == "" {
		return nil
	}

	todo := NewTodo()
	todo.Subject = p.Subject(input)
	todo.Projects = p.Projects(input)
	todo.Contexts = p.Contexts(input)
	if p.hasDue(input) {
		todo.Due = p.Due(input, time.Now())
	}
	return todo
}

func (p Parser) Parse() (string, int, string) {
	input := p.input
	r := regexp.MustCompile(`(\w+) (\d+) (.*)`)
	matches := r.FindStringSubmatch(input)
	if len(matches) < 4 {
		fmt.Println("Could match command, id or subject")
		return "", -1, input
	}
	id, err := strconv.Atoi(matches[2])
	if err != nil {
		fmt.Println("Invalid id.")
		return "", -1, input
	}

	return matches[1], id, matches[3]
}

func (p *Parser) Subject(input string) string {
	if strings.Contains(input, " due") {
		index := strings.LastIndex(input, " due")
		return input[0:index]
	} else {
		return input
	}
}

func (p *Parser) ExpandProject(input string) string {
	r, _ := regexp.Compile(`(ex|expand) +\d+ +\+[\p{L}\d_-]+:`)
	pattern := r.FindString(input)
	if len(pattern) == 0 {
		return ""
	}

	newProject := pattern[0 : len(pattern)-1]
	project := strings.Split(newProject, " ")
	return project[len(project)-1]
}

func (p *Parser) Projects(input string) []string {
	r, _ := regexp.Compile(`\+[\p{L}\d_-]+`)
	return p.matchWords(input, r)
}

func (p *Parser) Contexts(input string) []string {
	r, err := regexp.Compile(`\@[\p{L}\d_]+`)
	if err != nil {
		fmt.Println("regex error", err)
	}
	return p.matchWords(input, r)
}

func (p *Parser) hasDue(input string) bool {
	r1, _ := regexp.Compile(`due \w+$`)
	r2, _ := regexp.Compile(`due \w+ \d+$`)
	return (r1.MatchString(input) || r2.MatchString(input))
}

func (p *Parser) Due(input string, day time.Time) string {
	r, _ := regexp.Compile(`due .*$`)

	res := r.FindString(input)
	res = res[4:]
	switch res {
	case "none":
		return ""
	case "today", "tod":
		return bod(time.Now()).Format("2006-01-02")
	case "tomorrow", "tom":
		return bod(time.Now()).AddDate(0, 0, 1).Format("2006-01-02")
	case "monday", "mon":
		return p.monday(day)
	case "tuesday", "tue":
		return p.tuesday(day)
	case "wednesday", "wed":
		return p.wednesday(day)
	case "thursday", "thu":
		return p.thursday(day)
	case "friday", "fri":
		return p.friday(day)
	case "saturday", "sat":
		return p.saturday(day)
	case "sunday", "sun":
		return p.sunday(day)
	case "last week":
		n := bod(time.Now())
		return getNearestMonday(n).AddDate(0, 0, -7).Format("2006-01-02")
	case "next week":
		n := bod(time.Now())
		return getNearestMonday(n).AddDate(0, 0, 7).Format("2006-01-02")
	}
	return p.parseArbitraryDate(res, time.Now())
}

func (p *Parser) parseArbitraryDate(_date string, pivot time.Time) string {
	d1 := p.parseArbitraryDateWithYear(_date, pivot.Year())

	var diff1 time.Duration
	if d1.After(time.Now()) {
		diff1 = d1.Sub(pivot)
	} else {
		diff1 = pivot.Sub(d1)
	}
	d2 := p.parseArbitraryDateWithYear(_date, pivot.Year()+1)
	if d2.Sub(pivot) > diff1 {
		return d1.Format("2006-01-02")
	} else {
		return d2.Format("2006-01-02")
	}
}

func (p *Parser) parseArbitraryDateWithYear(_date string, year int) time.Time {
	res := strings.Join([]string{_date, strconv.Itoa(year)}, " ")
	if date, err := time.Parse("Jan 2 2006", res); err == nil {
		return date
	}

	if date, err := time.Parse("2 Jan 2006", res); err == nil {
		return date
	}
	fmt.Printf("Could not parse the date you gave me: %s\n", _date)
	fmt.Println("I'm expecting a date like \"Dec 22\" or \"22 Dec\".")
	fmt.Println("See http://todolist.site/#adding for more info.")
	os.Exit(-1)
	return time.Now()
}

func (p *Parser) monday(day time.Time) string {
	mon := getNearestMonday(day)
	return p.thisOrNextWeek(mon, day)
}

func (p *Parser) tuesday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 1)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) wednesday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 2)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) thursday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 3)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) friday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 4)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) saturday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 5)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) sunday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 6)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) thisOrNextWeek(day time.Time, pivotDay time.Time) string {
	if day.Before(pivotDay) {
		return day.AddDate(0, 0, 7).Format("2006-01-02")
	} else {
		return day.Format("2006-01-02")
	}
}

func (p *Parser) matchWords(input string, r *regexp.Regexp) []string {
	results := r.FindAllString(input, -1)
	ret := []string{}

	for _, val := range results {
		ret = append(ret, val[1:])
	}
	return ret
}
<commit_msg>allow any whitespace between subcommand, id and input<commit_after>package todolist

import (
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type Parser struct {
	input string
}

func (p *Parser) ParseNewTodo(input string) *Todo {
	r, _ := regexp.Compile(`^(add|a)(\\ |) `)
	input = r.ReplaceAllString(input, "")
	if input == "" {
		return nil
	}

	todo := NewTodo()
	todo.Subject = p.Subject(input)
	todo.Projects = p.Projects(input)
	todo.Contexts = p.Contexts(input)
	if p.hasDue(input) {
		todo.Due = p.Due(input, time.Now())
	}
	return todo
}

// Parse accepts user input and splits it into subcommand, the todo id to
// work on and the input to the subcommand function.
func (p Parser) Parse() (subcommand string, id int, input string) {
	r := regexp.MustCompile(`(\w+)\s+(\d+)\s+(.*)`)
	matches := r.FindStringSubmatch(p.input)
	if len(matches) < 4 {
		fmt.Println("Could match command, id or subject")
		return "", -1, ""
	}

	// because of the regexp match, this can never fail
	id, err := strconv.Atoi(matches[2])
	if err != nil {
		panic(err)
	}

	return matches[1], id, matches[3]
}

func (p *Parser) Subject(input string) string {
	if strings.Contains(input, " due") {
		index := strings.LastIndex(input, " due")
		return input[0:index]
	} else {
		return input
	}
}

func (p *Parser) ExpandProject(input string) string {
	r, _ := regexp.Compile(`(ex|expand) +\d+ +\+[\p{L}\d_-]+:`)
	pattern := r.FindString(input)
	if len(pattern) == 0 {
		return ""
	}

	newProject := pattern[0 : len(pattern)-1]
	project := strings.Split(newProject, " ")
	return project[len(project)-1]
}

func (p *Parser) Projects(input string) []string {
	r, _ := regexp.Compile(`\+[\p{L}\d_-]+`)
	return p.matchWords(input, r)
}

func (p *Parser) Contexts(input string) []string {
	r, err := regexp.Compile(`\@[\p{L}\d_]+`)
	if err != nil {
		fmt.Println("regex error", err)
	}
	return p.matchWords(input, r)
}

func (p *Parser) hasDue(input string) bool {
	r1, _ := regexp.Compile(`due \w+$`)
	r2, _ := regexp.Compile(`due \w+ \d+$`)
	return (r1.MatchString(input) || r2.MatchString(input))
}

func (p *Parser) Due(input string, day time.Time) string {
	r, _ := regexp.Compile(`due .*$`)

	res := r.FindString(input)
	res = res[4:]
	switch res {
	case "none":
		return ""
	case "today", "tod":
		return bod(time.Now()).Format("2006-01-02")
	case "tomorrow", "tom":
		return bod(time.Now()).AddDate(0, 0, 1).Format("2006-01-02")
	case "monday", "mon":
		return p.monday(day)
	case "tuesday", "tue":
		return p.tuesday(day)
	case "wednesday", "wed":
		return p.wednesday(day)
	case "thursday", "thu":
		return p.thursday(day)
	case "friday", "fri":
		return p.friday(day)
	case "saturday", "sat":
		return p.saturday(day)
	case "sunday", "sun":
		return p.sunday(day)
	case "last week":
		n := bod(time.Now())
		return getNearestMonday(n).AddDate(0, 0, -7).Format("2006-01-02")
	case "next week":
		n := bod(time.Now())
		return getNearestMonday(n).AddDate(0, 0, 7).Format("2006-01-02")
	}
	return p.parseArbitraryDate(res, time.Now())
}

func (p *Parser) parseArbitraryDate(_date string, pivot time.Time) string {
	d1 := p.parseArbitraryDateWithYear(_date, pivot.Year())

	var diff1 time.Duration
	if d1.After(time.Now()) {
		diff1 = d1.Sub(pivot)
	} else {
		diff1 = pivot.Sub(d1)
	}
	d2 := p.parseArbitraryDateWithYear(_date, pivot.Year()+1)
	if d2.Sub(pivot) > diff1 {
		return d1.Format("2006-01-02")
	} else {
		return d2.Format("2006-01-02")
	}
}

func (p *Parser) parseArbitraryDateWithYear(_date string, year int) time.Time {
	res := strings.Join([]string{_date, strconv.Itoa(year)}, " ")
	if date, err := time.Parse("Jan 2 2006", res); err == nil {
		return date
	}

	if date, err := time.Parse("2 Jan 2006", res); err == nil {
		return date
	}
	fmt.Printf("Could not parse the date you gave me: %s\n", _date)
	fmt.Println("I'm expecting a date like \"Dec 22\" or \"22 Dec\".")
	fmt.Println("See http://todolist.site/#adding for more info.")
	os.Exit(-1)
	return time.Now()
}

func (p *Parser) monday(day time.Time) string {
	mon := getNearestMonday(day)
	return p.thisOrNextWeek(mon, day)
}

func (p *Parser) tuesday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 1)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) wednesday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 2)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) thursday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 3)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) friday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 4)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) saturday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 5)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) sunday(day time.Time) string {
	tue := getNearestMonday(day).AddDate(0, 0, 6)
	return p.thisOrNextWeek(tue, day)
}

func (p *Parser) thisOrNextWeek(day time.Time, pivotDay time.Time) string {
	if day.Before(pivotDay) {
		return day.AddDate(0, 0, 7).Format("2006-01-02")
	} else {
		return day.Format("2006-01-02")
	}
}

func (p *Parser) matchWords(input string, r *regexp.Regexp) []string {
	results := r.FindAllString(input, -1)
	ret := []string{}

	for _, val := range results {
		ret = append(ret, val[1:])
	}
	return ret
}
<|endoftext|>


<commit_before>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package stb

import (
	"fmt"

	"github.com/gopherjs/gopherwasm/js"
)

func init() {
	// Eval wasm.js first to set the Wasm binary to Module.
	js.Global().Get("window").Call("eval", string(wasm_js))

	js.Global().Get("window").Call("eval", string(stbvorbis_js))
	js.Global().Get("window").Call("eval", string(decode_js))

	ch := make(chan struct{})
	js.Global().Get("_ebiten").Call("initializeVorbisDecoder", js.NewCallback(func([]js.Value) {
		close(ch)
	}))
	<-ch
}

func DecodeVorbis(buf []byte) ([]float32, int, int, error) {
	r := js.Global().Get("_ebiten").Call("decodeVorbis", buf)
	if r == js.Null() {
		return nil, 0, 0, fmt.Errorf("audio/vorbis/internal/stb: decode failed")
	}
	data := make([]float32, r.Get("data").Get("length").Int())
	// TODO: Use js.TypeArrayOf
	arr := js.ValueOf(data)
	arr.Call("set", r.Get("data"))
	return data, r.Get("channels").Int(), r.Get("sampleRate").Int(), nil
}
<commit_msg>audio/vorbis/internal/stb: Use TypedArrayOf (#642)<commit_after>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package stb

import (
	"fmt"

	"github.com/gopherjs/gopherwasm/js"
)

func init() {
	// Eval wasm.js first to set the Wasm binary to Module.
	js.Global().Get("window").Call("eval", string(wasm_js))

	js.Global().Get("window").Call("eval", string(stbvorbis_js))
	js.Global().Get("window").Call("eval", string(decode_js))

	ch := make(chan struct{})
	js.Global().Get("_ebiten").Call("initializeVorbisDecoder", js.NewCallback(func([]js.Value) {
		close(ch)
	}))
	<-ch
}

func DecodeVorbis(buf []byte) ([]float32, int, int, error) {
	arr := js.TypedArrayOf(buf)
	r := js.Global().Get("_ebiten").Call("decodeVorbis", arr)
	arr.Release()
	if r == js.Null() {
		return nil, 0, 0, fmt.Errorf("audio/vorbis/internal/stb: decode failed")
	}

	data := make([]float32, r.Get("data").Get("length").Int())
	arr = js.TypedArrayOf(data)
	arr.Call("set", r.Get("data"))
	arr.Release()
	return data, r.Get("channels").Int(), r.Get("sampleRate").Int(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/amlun/linda/linda"
	"github.com/amlun/linda/modules/monitor"
)

func main() {
	var config = linda.Config{
		BrokerURL: "redis://10.60.81.83:6379",
		SaverURL:  "cassandra://cassandra:cassandra@10.60.81.83:9042/linda",
	}
	l := linda.NewLinda(&config)
	m := monitor.New(l)
	m.Start()
}
<commit_msg>delete monitor<commit_after><|endoftext|>


<commit_before>package internal

import (
	"bufio"
	"bytes"
	"errors"
	"runtime/pprof"
	"time"

	"github.com/instana/go-sensor/autoprofile/internal/pprof/profile"
)

type CPUSampler struct {
	top        *CallSite
	profBuffer *bytes.Buffer
	startNano  int64
}

func NewCPUSampler() *CPUSampler {
	return &CPUSampler{}
}

func (cs *CPUSampler) Reset() {
	cs.top = NewCallSite("", "", 0)
}

func (cs *CPUSampler) Start() error {
	err := cs.startCPUSampler()
	if err != nil {
		return err
	}

	return nil
}

func (cs *CPUSampler) Stop() error {
	p, err := cs.stopCPUSampler()
	if err != nil {
		return err
	}
	if p == nil {
		return errors.New("no profile returned")
	}

	if uerr := cs.updateCPUProfile(p); uerr != nil {
		return uerr
	}

	return nil
}

func (cs *CPUSampler) Profile(duration int64, timespan int64) (*Profile, error) {
	roots := make([]*CallSite, 0)
	for _, child := range cs.top.children {
		roots = append(roots, child)
	}
	p := NewProfile(CategoryCPU, TypeCPUUsage, UnitMillisecond, roots, duration, timespan)
	return p, nil
}

func (cs *CPUSampler) updateCPUProfile(p *profile.Profile) error {
	samplesIndex := -1
	cpuIndex := -1
	for i, s := range p.SampleType {
		if s.Type == "samples" {
			samplesIndex = i
		} else if s.Type == "cpu" {
			cpuIndex = i
		}
	}

	if samplesIndex == -1 || cpuIndex == -1 {
		return errors.New("Unrecognized profile data")
	}

	// build call graph
	for _, s := range p.Sample {
		if shouldSkipStack(s) {
			continue
		}

		stackSamples := s.Value[samplesIndex]
		stackDuration := float64(s.Value[cpuIndex])

		current := cs.top
		for i := len(s.Location) - 1; i >= 0; i-- {
			l := s.Location[i]
			funcName, fileName, fileLine := readFuncInfo(l)

			current = current.FindOrAddChild(funcName, fileName, fileLine)
		}

		current.Increment(stackDuration, stackSamples)
	}

	return nil
}

func (cs *CPUSampler) startCPUSampler() error {
	cs.profBuffer = bytes.NewBuffer(nil)
	cs.startNano = time.Now().UnixNano()

	err := pprof.StartCPUProfile(cs.profBuffer)
	if err != nil {
		return err
	}

	return nil
}

func (cs *CPUSampler) stopCPUSampler() (*profile.Profile, error) {
	pprof.StopCPUProfile()

	r := bufio.NewReader(cs.profBuffer)

	if p, perr := profile.Parse(r); perr == nil {
		cs.profBuffer = nil

		if p.TimeNanos == 0 {
			p.TimeNanos = cs.startNano
		}
		if p.DurationNanos == 0 {
			p.DurationNanos = time.Now().UnixNano() - cs.startNano
		}

		if serr := symbolizeProfile(p); serr != nil {
			return nil, serr
		}

		if verr := p.CheckValid(); verr != nil {
			return nil, verr
		}

		return p, nil
	} else {
		cs.profBuffer = nil

		return nil, perr
	}
}

func readFuncInfo(l *profile.Location) (funcName string, fileName string, fileLine int64) {
	for li := range l.Line {
		if fn := l.Line[li].Function; fn != nil {
			return fn.Name, fn.Filename, l.Line[li].Line
		}
	}

	return "", "", 0
}
<commit_msg>Make reads from the CPU sampler buffer directly<commit_after>package internal

import (
	"bytes"
	"errors"
	"runtime/pprof"
	"time"

	"github.com/instana/go-sensor/autoprofile/internal/pprof/profile"
)

type CPUSampler struct {
	top       *CallSite
	buf       *bytes.Buffer
	startNano int64
}

func NewCPUSampler() *CPUSampler {
	return &CPUSampler{}
}

func (cs *CPUSampler) Reset() {
	cs.top = NewCallSite("", "", 0)
}

func (cs *CPUSampler) Start() error {
	if cs.buf != nil {
		return nil
	}

	cs.buf = bytes.NewBuffer(nil)
	cs.startNano = time.Now().UnixNano()

	if err := pprof.StartCPUProfile(cs.buf); err != nil {
		return err
	}

	return nil
}

func (cs *CPUSampler) Stop() error {
	if cs.buf == nil {
		return nil
	}

	pprof.StopCPUProfile()

	p, err := cs.collectProfile()
	if err != nil {
		return err
	}

	if p == nil {
		return errors.New("no profile returned")
	}

	if uerr := cs.updateCPUProfile(p); uerr != nil {
		return uerr
	}

	return nil
}

func (cs *CPUSampler) Profile(duration int64, timespan int64) (*Profile, error) {
	roots := make([]*CallSite, 0)
	for _, child := range cs.top.children {
		roots = append(roots, child)
	}
	p := NewProfile(CategoryCPU, TypeCPUUsage, UnitMillisecond, roots, duration, timespan)

	return p, nil
}

func (cs *CPUSampler) updateCPUProfile(p *profile.Profile) error {
	samplesIndex := -1
	cpuIndex := -1
	for i, s := range p.SampleType {
		if s.Type == "samples" {
			samplesIndex = i
		} else if s.Type == "cpu" {
			cpuIndex = i
		}
	}

	if samplesIndex == -1 || cpuIndex == -1 {
		return errors.New("Unrecognized profile data")
	}

	// build call graph
	for _, s := range p.Sample {
		if shouldSkipStack(s) {
			continue
		}

		stackSamples := s.Value[samplesIndex]
		stackDuration := float64(s.Value[cpuIndex])

		current := cs.top
		for i := len(s.Location) - 1; i >= 0; i-- {
			l := s.Location[i]
			funcName, fileName, fileLine := readFuncInfo(l)

			current = current.FindOrAddChild(funcName, fileName, fileLine)
		}

		current.Increment(stackDuration, stackSamples)
	}

	return nil
}

func (cs *CPUSampler) collectProfile() (*profile.Profile, error) {
	defer func() {
		cs.buf = nil
	}()

	p, err := profile.Parse(cs.buf)
	if err != nil {
		return nil, err
	}

	if p.TimeNanos == 0 {
		p.TimeNanos = cs.startNano
	}

	if p.DurationNanos == 0 {
		p.DurationNanos = time.Now().UnixNano() - cs.startNano
	}

	if err := symbolizeProfile(p); err != nil {
		return nil, err
	}

	if err := p.CheckValid(); err != nil {
		return nil, err
	}

	return p, nil
}

func readFuncInfo(l *profile.Location) (funcName string, fileName string, fileLine int64) {
	for li := range l.Line {
		if fn := l.Line[li].Function; fn != nil {
			return fn.Name, fn.Filename, l.Line[li].Line
		}
	}

	return "", "", 0
}
<|endoftext|>