<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"
)

const (
	CmdDisplayMax = 32
	// max length of /proc/$PID/comm
	CommMax = 16
	// from ps_mem - average error due to truncation in the kernel
	// pss calculations
	PssAdjust = .5
)

// store info about a command (group of processes), similar to how
// ps_mem works.
type CmdMemInfo struct {
	PIDs    []int
	Name    string
	Pss     float64
	Shared  int64
	Private int64
	Swapped int64
}

type MapInfo struct {
	Inode int64
	Name  string
}

// mapLine is a line from /proc/$PID/maps, or one of the same header
// lines from smaps.
func NewMapInfo(mapLine []byte) MapInfo {
	var mi MapInfo
	var err error
	pieces := splitSpaces(mapLine)
	if len(pieces) == 6 {
		mi.Name = string(pieces[5])
	}
	mi.Inode, err = strconv.ParseInt(string(pieces[4]), 10, 64)
	if err != nil {
		panic(fmt.Sprintf("NewMapInfo: Atoi(%s): %s (%s)",
			string(pieces[4]), err, string(mapLine)))
	}
	return mi
}

func (mi MapInfo) IsAnon() bool {
	return mi.Inode == 0
}

// isDigit returns true if the rune d represents an ascii digit
// between 0 and 9, inclusive.
func isDigit(d uint8) bool {
	return d >= '0' && d <= '9'
}

// pidList returns a list of the process-IDs of every currently
// running process on the local system.
func pidList() ([]int, error) {
	procLs, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, fmt.Errorf("ReadDir(/proc): %s", err)
	}

	pids := make([]int, 0, len(procLs))
	for _, pInfo := range procLs {
		if !isDigit(pInfo.Name()[0]) || !pInfo.IsDir() {
			continue
		}
		pidInt, err := strconv.Atoi(pInfo.Name())
		if err != nil {
			return nil, fmt.Errorf("Atoi(%s): %s", pInfo.Name(), err)
		}
		pids = append(pids, pidInt)
	}
	return pids, nil
}

// procName gets the process name for a worker.  It first checks the
// value of /proc/$PID/cmdline.  If setproctitle(3) has been called,
// it will use this.  Otherwise it uses the value of
// path.Base(/proc/$PID/exe), which has info on whether the executable
// has changed since the process was exec'ed.
func procName(pid int) (string, error) {
	p, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid))
	// this would return an error if the PID doesn't
	// exist, or if the PID refers to a kernel thread.
	if err != nil {
		return "", nil
	}
	// cmdline is the null separated list of command line
	// arguments for the process, unless setproctitle(3)
	// has been called, in which case it is the
	argsB, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid))
	if err != nil {
		return "", fmt.Errorf("ReadFile(%s): %s", fmt.Sprintf("/proc/%d/cmdline", pid), err)
	}
	args := strings.Split(string(argsB), "\000")
	n := args[0]

	nTrunc := n
	if len(n) > CommMax {
		nTrunc = n[:CommMax]
	}
	if strings.HasPrefix(p, nTrunc) {
		n = path.Base(p)
	}
	return n, nil
}

func splitSpaces(b []byte) [][]byte {
	res := make([][]byte, 0, 6)
	s := bytes.SplitN(b, []byte{' '}, 2)
	for len(s) > 1 {
		res = append(res, s[0])
		s = bytes.SplitN(bytes.TrimSpace(s[1]), []byte{' '}, 2)
	}
	res = append(res, s[0])
	return res
}

// procMem returns the amount of Pss, shared, and swapped out memory
// used.  The swapped out amount refers to anonymous pages only.
func procMem(pid int) (pss float64, shared, priv, swap int64, err error) {
	smapB, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/smaps", pid))
	if err != nil {
		err = fmt.Errorf("ReadFile(%s): %s", fmt.Sprintf("/proc/%d/smaps", pid), err)
		return
	}
	smapLines := bytes.Split(smapB, []byte{'\n'})
	var curr MapInfo
	for _, l := range smapLines {
		if bytes.Contains(l, []byte{'-'}) {
			curr = NewMapInfo(l)
			continue
		}
		pieces := splitSpaces(l)
		ty := string(pieces[0])
		var v int64
		switch ty {
		case "Pss:":
			v, err = strconv.ParseInt(string(pieces[1]), 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			pss += float64(v) + PssAdjust
		case "Shared_Clean:", "Shared_Dirty:":
			v, err = strconv.ParseInt(string(pieces[1]), 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			shared += v
		case "Private_Clean:", "Private_Dirty:":
			v, err = strconv.ParseInt(string(pieces[1]), 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			priv += v
		case "Swap:":
			v, err = strconv.ParseInt(string(pieces[1]), 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			swap += v
		}
	}
	_ = curr
	return
}

func worker(pidRequest chan int, wg *sync.WaitGroup, result chan *CmdMemInfo) {
	for pid := range pidRequest {
		var err error
		cmi := new(CmdMemInfo)

		cmi.PIDs = []int{pid}
		cmi.Name, err = procName(pid)
		if err != nil {
			log.Printf("procName(%d): %s", pid, err)
			wg.Done()
			continue
		} else if cmi.Name == "" {
			// XXX: This happens with kernel
			// threads. maybe warn? idk.
			wg.Done()
			continue
		}

		cmi.Pss, cmi.Shared, cmi.Private, cmi.Swapped, err = procMem(pid)
		if err != nil {
			log.Printf("procMem(%d): %s", pid, err)
			wg.Done()
			continue
		}

		result <- cmi
		wg.Done()
	}
}

type byTotal []*CmdMemInfo

func (c byTotal) Len() int           { return len(c) }
func (c byTotal) Less(i, j int) bool { return c[i].Pss < c[j].Pss }
func (c byTotal) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }

func main() {
	nCPU := runtime.NumCPU()
	// give us as much parallelism as possible
	runtime.GOMAXPROCS(nCPU)

	if os.Geteuid() != 0 {
		fmt.Printf("FATAL: root required.\n")
		return
	}

	pids, err := pidList()
	if err != nil {
		log.Printf("pidList: %s", err)
		return
	}

	var wg sync.WaitGroup
	work := make(chan int, len(pids))
	result := make(chan *CmdMemInfo, len(pids))

	for i := 0; i < nCPU; i++ {
		go worker(work, &wg, result)
	}

	wg.Add(len(pids))
	for _, pid := range pids {
		work <- pid
	}
	wg.Wait()

	cmdMap := map[string]*CmdMemInfo{}
loop:
	for {
		select {
		case c := <-result:
			n := c.Name
			if _, ok := cmdMap[n]; !ok {
				cmdMap[n] = c
				continue
			}
			cmdMap[n].PIDs = append(cmdMap[n].PIDs, c.PIDs...)
			cmdMap[n].Pss += c.Pss
			cmdMap[n].Private += c.Private
			cmdMap[n].Shared += c.Shared
			cmdMap[n].Swapped += c.Swapped
		default:
			break loop
		}
	}

	cmds := make([]*CmdMemInfo, 0, len(cmdMap))

	for _, c := range cmdMap {
		cmds = append(cmds, c)
	}

	sort.Sort(byTotal(cmds))

	fmt.Printf("  MB TOTAL      PRIV      SWAP\tPROCESS (COUNT)\n")
	for _, c := range cmds {
		n := c.Name
		if len(n) > CmdDisplayMax {
			n = n[:CmdDisplayMax]
		}
		s := ""
		if c.Swapped > 0 {
			s = fmt.Sprintf("%10.2f", float64(c.Swapped)/1024.)
		}
		fmt.Printf("%10.2f%10.2f%10s\t%s (%d)\n", float64(c.Pss)/1024., float64(c.Private)/1024, s, n, len(c.PIDs))
	}
}
<commit_msg>use sprintf to create header, rather than manual spacing<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"
)

const (
	CmdDisplayMax = 32
	// max length of /proc/$PID/comm
	CommMax = 16
	// from ps_mem - average error due to truncation in the kernel
	// pss calculations
	PssAdjust = .5
)

// store info about a command (group of processes), similar to how
// ps_mem works.
type CmdMemInfo struct {
	PIDs    []int
	Name    string
	Pss     float64
	Shared  int64
	Private int64
	Swapped int64
}

type MapInfo struct {
	Inode int64
	Name  string
}

// mapLine is a line from /proc/$PID/maps, or one of the same header
// lines from smaps.
func NewMapInfo(mapLine []byte) MapInfo {
	var mi MapInfo
	var err error
	pieces := splitSpaces(mapLine)
	if len(pieces) == 6 {
		mi.Name = string(pieces[5])
	}
	mi.Inode, err = strconv.ParseInt(string(pieces[4]), 10, 64)
	if err != nil {
		panic(fmt.Sprintf("NewMapInfo: Atoi(%s): %s (%s)",
			string(pieces[4]), err, string(mapLine)))
	}
	return mi
}

func (mi MapInfo) IsAnon() bool {
	return mi.Inode == 0
}

// isDigit returns true if the rune d represents an ascii digit
// between 0 and 9, inclusive.
func isDigit(d uint8) bool {
	return d >= '0' && d <= '9'
}

// pidList returns a list of the process-IDs of every currently
// running process on the local system.
func pidList() ([]int, error) {
	procLs, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, fmt.Errorf("ReadDir(/proc): %s", err)
	}

	pids := make([]int, 0, len(procLs))
	for _, pInfo := range procLs {
		if !isDigit(pInfo.Name()[0]) || !pInfo.IsDir() {
			continue
		}
		pidInt, err := strconv.Atoi(pInfo.Name())
		if err != nil {
			return nil, fmt.Errorf("Atoi(%s): %s", pInfo.Name(), err)
		}
		pids = append(pids, pidInt)
	}
	return pids, nil
}

// procName gets the process name for a worker.  It first checks the
// value of /proc/$PID/cmdline.  If setproctitle(3) has been called,
// it will use this.  Otherwise it uses the value of
// path.Base(/proc/$PID/exe), which has info on whether the executable
// has changed since the process was exec'ed.
func procName(pid int) (string, error) {
	p, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid))
	// this would return an error if the PID doesn't
	// exist, or if the PID refers to a kernel thread.
	if err != nil {
		return "", nil
	}
	// cmdline is the null separated list of command line
	// arguments for the process, unless setproctitle(3)
	// has been called, in which case it is the
	argsB, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid))
	if err != nil {
		return "", fmt.Errorf("ReadFile(%s): %s", fmt.Sprintf("/proc/%d/cmdline", pid), err)
	}
	args := strings.Split(string(argsB), "\000")
	n := args[0]

	nTrunc := n
	if len(n) > CommMax {
		nTrunc = n[:CommMax]
	}
	if strings.HasPrefix(p, nTrunc) {
		n = path.Base(p)
	}
	return n, nil
}

func splitSpaces(b []byte) [][]byte {
	res := make([][]byte, 0, 6)
	s := bytes.SplitN(b, []byte{' '}, 2)
	for len(s) > 1 {
		res = append(res, s[0])
		s = bytes.SplitN(bytes.TrimSpace(s[1]), []byte{' '}, 2)
	}
	res = append(res, s[0])
	return res
}

// procMem returns the amount of Pss, shared, and swapped out memory
// used.  The swapped out amount refers to anonymous pages only.
func procMem(pid int) (pss float64, shared, priv, swap int64, err error) {
	smapB, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/smaps", pid))
	if err != nil {
		err = fmt.Errorf("ReadFile(%s): %s", fmt.Sprintf("/proc/%d/smaps", pid), err)
		return
	}
	smapLines := bytes.Split(smapB, []byte{'\n'})
	var curr MapInfo
	for _, l := range smapLines {
		if bytes.Contains(l, []byte{'-'}) {
			curr = NewMapInfo(l)
			continue
		}
		pieces := splitSpaces(l)
		ty := string(pieces[0])
		var v int64
		switch ty {
		case "Pss:":
			v, err = strconv.ParseInt(string(pieces[1]), 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			pss += float64(v) + PssAdjust
		case "Shared_Clean:", "Shared_Dirty:":
			v, err = strconv.ParseInt(string(pieces[1]), 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			shared += v
		case "Private_Clean:", "Private_Dirty:":
			v, err = strconv.ParseInt(string(pieces[1]), 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			priv += v
		case "Swap:":
			v, err = strconv.ParseInt(string(pieces[1]), 10, 64)
			if err != nil {
				err = fmt.Errorf("Atoi(%s): %s", string(pieces[1]), err)
				return
			}
			swap += v
		}
	}
	_ = curr
	return
}

func worker(pidRequest chan int, wg *sync.WaitGroup, result chan *CmdMemInfo) {
	for pid := range pidRequest {
		var err error
		cmi := new(CmdMemInfo)

		cmi.PIDs = []int{pid}
		cmi.Name, err = procName(pid)
		if err != nil {
			log.Printf("procName(%d): %s", pid, err)
			wg.Done()
			continue
		} else if cmi.Name == "" {
			// XXX: This happens with kernel
			// threads. maybe warn? idk.
			wg.Done()
			continue
		}

		cmi.Pss, cmi.Shared, cmi.Private, cmi.Swapped, err = procMem(pid)
		if err != nil {
			log.Printf("procMem(%d): %s", pid, err)
			wg.Done()
			continue
		}

		result <- cmi
		wg.Done()
	}
}

type byTotal []*CmdMemInfo

func (c byTotal) Len() int           { return len(c) }
func (c byTotal) Less(i, j int) bool { return c[i].Pss < c[j].Pss }
func (c byTotal) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }

func main() {
	nCPU := runtime.NumCPU()
	// give us as much parallelism as possible
	runtime.GOMAXPROCS(nCPU)

	if os.Geteuid() != 0 {
		fmt.Printf("FATAL: root required.\n")
		return
	}

	pids, err := pidList()
	if err != nil {
		log.Printf("pidList: %s", err)
		return
	}

	var wg sync.WaitGroup
	work := make(chan int, len(pids))
	result := make(chan *CmdMemInfo, len(pids))

	for i := 0; i < nCPU; i++ {
		go worker(work, &wg, result)
	}

	wg.Add(len(pids))
	for _, pid := range pids {
		work <- pid
	}
	wg.Wait()

	cmdMap := map[string]*CmdMemInfo{}
loop:
	for {
		select {
		case c := <-result:
			n := c.Name
			if _, ok := cmdMap[n]; !ok {
				cmdMap[n] = c
				continue
			}
			cmdMap[n].PIDs = append(cmdMap[n].PIDs, c.PIDs...)
			cmdMap[n].Pss += c.Pss
			cmdMap[n].Private += c.Private
			cmdMap[n].Shared += c.Shared
			cmdMap[n].Swapped += c.Swapped
		default:
			break loop
		}
	}

	cmds := make([]*CmdMemInfo, 0, len(cmdMap))

	for _, c := range cmdMap {
		cmds = append(cmds, c)
	}

	sort.Sort(byTotal(cmds))

	fmt.Printf("%10s%10s%10s\t%s\n", "MB RAM", "PRIVATE", "SWAPPED", "PROCESS (COUNT)")
	for _, c := range cmds {
		n := c.Name
		if len(n) > CmdDisplayMax {
			n = n[:CmdDisplayMax]
		}
		s := ""
		if c.Swapped > 0 {
			s = fmt.Sprintf("%10.2f", float64(c.Swapped)/1024.)
		}
		fmt.Printf("%10.2f%10.2f%10s\t%s (%d)\n", float64(c.Pss)/1024., float64(c.Private)/1024, s, n, len(c.PIDs))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/rancher/convoy/api"
	"github.com/rancher/convoy/client"
	"os"
)

const (
	// version of Convoy
	VERSION = "0.5.0-dev"
)

func cleanup() {
	if r := recover(); r != nil {
		api.ResponseLogAndError(r)
		os.Exit(1)
	}
}

func main() {
	defer cleanup()

	cli := client.NewCli(VERSION)
	err := cli.Run(os.Args)
	if err != nil {
		panic(fmt.Errorf("Error when executing command: %v", err))
	}
}
<commit_msg>Convoy v0.5.0.1-rancher release<commit_after>package main

import (
	"fmt"
	"github.com/rancher/convoy/api"
	"github.com/rancher/convoy/client"
	"os"
)

const (
	// version of Convoy
	VERSION = "0.5.0.1-rancher"
)

func cleanup() {
	if r := recover(); r != nil {
		api.ResponseLogAndError(r)
		os.Exit(1)
	}
}

func main() {
	defer cleanup()

	cli := client.NewCli(VERSION)
	err := cli.Run(os.Args)
	if err != nil {
		panic(fmt.Errorf("Error when executing command: %v", err))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
	cobra "github.com/spf13/cobra"
)

type handler func(*cobra.Command, []string) error

func genCommand(use, short string, fn handler) *cobra.Command {
	run := func(cmd *cobra.Command, args []string) {
		if err := fn(cmd, args); err != nil {
			log.Fatal(err)
		} else {
			log.Debug("success")
		}
	}
	return &cobra.Command{
		Use:   use,
		Short: short,
		Run:   run,
	}

}

func main() {
	dummyPersistentPreRun := func(cmd *cobra.Command, args []string) {
		// Use PersistentPreRun here to replace the root one
	}
	root := &cobra.Command{
		Use:  "gozfs",
		Long: "gozfs is the cli for testing the go zfs ioctl api",
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			if cmd.Use == "gozfs" {
				return
			}
			name, err := cmd.Flags().GetString("name")
			if err != nil {
				log.Fatal(err)
			}
			if name == "" {
				log.Fatal("missing name")
			}
		},
		Run: help,
	}
	root.PersistentFlags().StringP("name", "n", "", "dataset name")

	cmdExists := genCommand("exists", "Test for dataset existence.",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			return exists(name)
		})

	cmdDestroy := genCommand("destroy", "Destroys a dataset or volume.",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			recursive, _ := cmd.Flags().GetBool("recursive")
			recursiveClones, _ := cmd.Flags().GetBool("recursiveclones")
			forceUnmount, _ := cmd.Flags().GetBool("forceunmount")
			deferDestroy, _ := cmd.Flags().GetBool("defer")

			d, err := GetDataset(name)
			if err != nil {
				return err
			}

			opts := &DestroyOptions{
				Recursive:       recursive,
				RecursiveClones: recursiveClones,
				ForceUnmount:    forceUnmount,
				Defer:           deferDestroy,
			}
			return d.Destroy(opts)
		})
	cmdDestroy.Flags().BoolP("defer", "d", false, "defer destroy")
	cmdDestroy.Flags().BoolP("recursive", "r", false, "recursively destroy datasets")
	cmdDestroy.Flags().BoolP("recursiveclones", "c", false, "recursively destroy clones")
	cmdDestroy.Flags().BoolP("forceunmount", "f", false, "force unmount")

	cmdHolds := genCommand("holds", "Retrieve list of user holds on the specified snapshot.",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			holds, err := holds(name)
			if err == nil {
				fmt.Println(holds)
			}
			return err
		})

	cmdSnapshot := genCommand("snapshot", "Creates a snapshot of a dataset or volume",
		func(cmd *cobra.Command, snaps []string) error {
			recursive, _ := cmd.Flags().GetBool("recursive")
			name, _ := cmd.Flags().GetString("name")
			nameParts := strings.Split(name, "@")
			if len(nameParts) != 2 {
				log.Fatal("invalid snapshot name")
			}

			var props map[string]string
			propJSON, _ := cmd.Flags().GetString("props")
			if err := json.Unmarshal([]byte(propJSON), &props); err != nil {
				log.Fatal("bad prop json")
			}

			d, err := GetDataset(nameParts[0])
			if err != nil {
				return err
			}

			return d.Snapshot(nameParts[1], recursive)
		},
	)
	cmdSnapshot.Flags().StringP("props", "p", "{}", "snapshot properties")
	cmdSnapshot.Flags().BoolP("recursive", "r", false, "recurisvely create snapshots")

	cmdRollback := genCommand("rollback", "Rollback this filesystem or volume to its most recent snapshot.",
		func(cmd *cobra.Command, args []string) error {
			destroyMoreRecent, _ := cmd.Flags().GetBool("destroyrecent")
			name, _ := cmd.Flags().GetString("name")

			d, err := GetDataset(name)
			if err != nil {
				return err
			}

			return d.Rollback(destroyMoreRecent)
		},
	)
	cmdRollback.Flags().BoolP("destroyrecent", "r", false, "destroy more recent snapshots and their clones")

	cmdCreate := genCommand("create", "Create a ZFS dataset or volume",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			volsize, _ := cmd.Flags().GetUint64("volsize")
			createTypeName, _ := cmd.Flags().GetString("type")

			propJSON, _ := cmd.Flags().GetString("props")
			var props map[string]interface{}
			if err := json.Unmarshal([]byte(propJSON), &props); err != nil {
				log.Fatal("bad prop json")
			}

			if createTypeName == "zvol" {
				if volblocksize, ok := props["volblocksize"]; ok {
					if volblocksizeFloat, ok := volblocksize.(float64); ok {
						props["volblocksize"] = uint64(volblocksizeFloat)
					}
				}

				_, err := CreateVolume(name, volsize, props)
				return err
			} else {
				_, err := CreateFilesystem(name, props)
				return err
			}
		},
	)
	cmdCreate.Flags().StringP("type", "t", "zfs", "zfs or zvol")
	cmdCreate.Flags().Uint64P("volsize", "s", 0, "volume size")
	cmdCreate.Flags().StringP("props", "p", "{}", "create properties")

	cmdSend := genCommand("send", "Generate a send stream from a snapshot",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			output, _ := cmd.Flags().GetString("output")

			outputFD := os.Stdout.Fd()
			if output != "/dev/stdout" {
				outputFile, err := os.Create(output)
				if err != nil {
					return err
				}
				defer outputFile.Close()

				outputFD = outputFile.Fd()
			} else {
				// If sending on stdout, don't log anything else unless there's
				// an error
				log.SetLevel(log.ErrorLevel)
			}

			d, err := GetDataset(name)
			if err != nil {
				return err
			}

			return d.Send(outputFD)
		},
	)
	cmdSend.Flags().StringP("output", "o", "/dev/stdout", "output file")

	cmdClone := genCommand("clone", "Creates a clone from a snapshot",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			origin, _ := cmd.Flags().GetString("origin")
			if origin == "" {
				log.Fatal("missing origin snapshot name")
			}

			var props map[string]interface{}
			propJSON, _ := cmd.Flags().GetString("props")
			if err := json.Unmarshal([]byte(propJSON), &props); err != nil {
				log.Fatal("bad prop json")
			}

			d, err := GetDataset(origin)
			if err != nil {
				return err
			}
			_, err = d.Clone(name, props)
			return err
		},
	)
	cmdClone.Flags().StringP("origin", "o", "", "name of origin snapshot")
	cmdClone.Flags().StringP("props", "p", "{}", "snapshot properties")

	cmdRename := genCommand("rename", "Rename a dataset",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			newName, _ := cmd.Flags().GetString("newname")
			recursive, _ := cmd.Flags().GetBool("recursive")

			failedName, err := rename(name, newName, recursive)
			if failedName != "" {
				log.Error(failedName)
			}
			return err
		},
	)
	cmdRename.Flags().StringP("newname", "o", "", "new name of dataset")
	cmdRename.Flags().BoolP("recursive", "r", false, "recursively rename snapshots")

	cmdList := genCommand("list", "List filesystems, volumes, snapshots and bookmarks.",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			dsType, _ := cmd.Flags().GetString("type")
			if dsType == "" {
				dsType = "all"
			}

			var datasets []*Dataset
			var err error
			switch dsType {
			case "all":
				datasets, err = Datasets(name)
			case DatasetFilesystem:
				datasets, err = Filesystems(name)
			case DatasetSnapshot:
				datasets, err = Snapshots(name)
			case DatasetVolume:
				datasets, err = Volumes(name)
			}
			if err != nil {
				return err
			}
			out, err := json.MarshalIndent(datasets, "", "  ")
			if err != nil {
				return err
			}
			fmt.Printf("%s\n", out)
			return nil
		})
	cmdList.PersistentPreRun = dummyPersistentPreRun
	cmdList.Flags().StringP("type", "t", "all", strings.Join([]string{"all", DatasetFilesystem, DatasetVolume, DatasetSnapshot}, ","))

	cmdGet := genCommand("get", "Get dataset properties",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			recurse, _ := cmd.Flags().GetBool("recurse")
			depth, _ := cmd.Flags().GetUint64("depth")
			typesList, _ := cmd.Flags().GetString("types")

			types := map[string]bool{}
			if typesList != "" {
				for _, t := range strings.Split(typesList, ",") {
					types[t] = true
				}
			}

			m, err := properties(name, types, recurse, depth)
			if err != nil {
				return err
			}
			out, err := json.MarshalIndent(m, "", "  ")
			if err != nil {
				return err
			}
			fmt.Printf("%s\n", out)
			return nil
		})
	cmdGet.Flags().StringP("types", "t", "", "Comma seperated list of dataset types to list.")
	cmdGet.Flags().BoolP("recurse", "r", false, "Recurse to all levels.")
	cmdGet.Flags().Uint64P("depth", "d", 0, "Recursion depth limit, 0 for unlimited")

	root.AddCommand(
		cmdExists,
		cmdDestroy,
		cmdHolds,
		cmdSnapshot,
		cmdRollback,
		cmdCreate,
		cmdSend,
		cmdClone,
		cmdRename,
		cmdList,
		cmdGet,
	)
	if err := root.Execute(); err != nil {
		log.Fatal("root execute failed:", err)
	}
}

func help(cmd *cobra.Command, _ []string) {
	if err := cmd.Help(); err != nil {
		log.Fatal("help failed:", err)
	}
}
<commit_msg>Use public API GetDataset in command<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
	cobra "github.com/spf13/cobra"
)

type handler func(*cobra.Command, []string) error

func genCommand(use, short string, fn handler) *cobra.Command {
	run := func(cmd *cobra.Command, args []string) {
		if err := fn(cmd, args); err != nil {
			log.Fatal(err)
		} else {
			log.Debug("success")
		}
	}
	return &cobra.Command{
		Use:   use,
		Short: short,
		Run:   run,
	}

}

func main() {
	dummyPersistentPreRun := func(cmd *cobra.Command, args []string) {
		// Use PersistentPreRun here to replace the root one
	}
	root := &cobra.Command{
		Use:  "gozfs",
		Long: "gozfs is the cli for testing the go zfs ioctl api",
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			if cmd.Use == "gozfs" {
				return
			}
			name, err := cmd.Flags().GetString("name")
			if err != nil {
				log.Fatal(err)
			}
			if name == "" {
				log.Fatal("missing name")
			}
		},
		Run: help,
	}
	root.PersistentFlags().StringP("name", "n", "", "dataset name")

	cmdExists := genCommand("exists", "Test for dataset existence.",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			return exists(name)
		})

	cmdDestroy := genCommand("destroy", "Destroys a dataset or volume.",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			recursive, _ := cmd.Flags().GetBool("recursive")
			recursiveClones, _ := cmd.Flags().GetBool("recursiveclones")
			forceUnmount, _ := cmd.Flags().GetBool("forceunmount")
			deferDestroy, _ := cmd.Flags().GetBool("defer")

			d, err := GetDataset(name)
			if err != nil {
				return err
			}

			opts := &DestroyOptions{
				Recursive:       recursive,
				RecursiveClones: recursiveClones,
				ForceUnmount:    forceUnmount,
				Defer:           deferDestroy,
			}
			return d.Destroy(opts)
		})
	cmdDestroy.Flags().BoolP("defer", "d", false, "defer destroy")
	cmdDestroy.Flags().BoolP("recursive", "r", false, "recursively destroy datasets")
	cmdDestroy.Flags().BoolP("recursiveclones", "c", false, "recursively destroy clones")
	cmdDestroy.Flags().BoolP("forceunmount", "f", false, "force unmount")

	cmdHolds := genCommand("holds", "Retrieve list of user holds on the specified snapshot.",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			holds, err := holds(name)
			if err == nil {
				fmt.Println(holds)
			}
			return err
		})

	cmdSnapshot := genCommand("snapshot", "Creates a snapshot of a dataset or volume",
		func(cmd *cobra.Command, snaps []string) error {
			recursive, _ := cmd.Flags().GetBool("recursive")
			name, _ := cmd.Flags().GetString("name")
			nameParts := strings.Split(name, "@")
			if len(nameParts) != 2 {
				log.Fatal("invalid snapshot name")
			}

			var props map[string]string
			propJSON, _ := cmd.Flags().GetString("props")
			if err := json.Unmarshal([]byte(propJSON), &props); err != nil {
				log.Fatal("bad prop json")
			}

			d, err := GetDataset(nameParts[0])
			if err != nil {
				return err
			}

			return d.Snapshot(nameParts[1], recursive)
		},
	)
	cmdSnapshot.Flags().StringP("props", "p", "{}", "snapshot properties")
	cmdSnapshot.Flags().BoolP("recursive", "r", false, "recurisvely create snapshots")

	cmdRollback := genCommand("rollback", "Rollback this filesystem or volume to its most recent snapshot.",
		func(cmd *cobra.Command, args []string) error {
			destroyMoreRecent, _ := cmd.Flags().GetBool("destroyrecent")
			name, _ := cmd.Flags().GetString("name")

			d, err := GetDataset(name)
			if err != nil {
				return err
			}

			return d.Rollback(destroyMoreRecent)
		},
	)
	cmdRollback.Flags().BoolP("destroyrecent", "r", false, "destroy more recent snapshots and their clones")

	cmdCreate := genCommand("create", "Create a ZFS dataset or volume",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			volsize, _ := cmd.Flags().GetUint64("volsize")
			createTypeName, _ := cmd.Flags().GetString("type")

			propJSON, _ := cmd.Flags().GetString("props")
			var props map[string]interface{}
			if err := json.Unmarshal([]byte(propJSON), &props); err != nil {
				log.Fatal("bad prop json")
			}

			if createTypeName == "zvol" {
				if volblocksize, ok := props["volblocksize"]; ok {
					if volblocksizeFloat, ok := volblocksize.(float64); ok {
						props["volblocksize"] = uint64(volblocksizeFloat)
					}
				}

				_, err := CreateVolume(name, volsize, props)
				return err
			} else {
				_, err := CreateFilesystem(name, props)
				return err
			}
		},
	)
	cmdCreate.Flags().StringP("type", "t", "zfs", "zfs or zvol")
	cmdCreate.Flags().Uint64P("volsize", "s", 0, "volume size")
	cmdCreate.Flags().StringP("props", "p", "{}", "create properties")

	cmdSend := genCommand("send", "Generate a send stream from a snapshot",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			output, _ := cmd.Flags().GetString("output")

			outputFD := os.Stdout.Fd()
			if output != "/dev/stdout" {
				outputFile, err := os.Create(output)
				if err != nil {
					return err
				}
				defer outputFile.Close()

				outputFD = outputFile.Fd()
			} else {
				// If sending on stdout, don't log anything else unless there's
				// an error
				log.SetLevel(log.ErrorLevel)
			}

			d, err := GetDataset(name)
			if err != nil {
				return err
			}

			return d.Send(outputFD)
		},
	)
	cmdSend.Flags().StringP("output", "o", "/dev/stdout", "output file")

	cmdClone := genCommand("clone", "Creates a clone from a snapshot",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			origin, _ := cmd.Flags().GetString("origin")
			if origin == "" {
				log.Fatal("missing origin snapshot name")
			}

			var props map[string]interface{}
			propJSON, _ := cmd.Flags().GetString("props")
			if err := json.Unmarshal([]byte(propJSON), &props); err != nil {
				log.Fatal("bad prop json")
			}

			d, err := GetDataset(origin)
			if err != nil {
				return err
			}
			_, err = d.Clone(name, props)
			return err
		},
	)
	cmdClone.Flags().StringP("origin", "o", "", "name of origin snapshot")
	cmdClone.Flags().StringP("props", "p", "{}", "snapshot properties")

	cmdRename := genCommand("rename", "Rename a dataset",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			newName, _ := cmd.Flags().GetString("newname")
			recursive, _ := cmd.Flags().GetBool("recursive")

			failedName, err := rename(name, newName, recursive)
			if failedName != "" {
				log.Error(failedName)
			}
			return err
		},
	)
	cmdRename.Flags().StringP("newname", "o", "", "new name of dataset")
	cmdRename.Flags().BoolP("recursive", "r", false, "recursively rename snapshots")

	cmdList := genCommand("list", "List filesystems, volumes, snapshots and bookmarks.",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")
			dsType, _ := cmd.Flags().GetString("type")
			if dsType == "" {
				dsType = "all"
			}

			var datasets []*Dataset
			var err error
			switch dsType {
			case "all":
				datasets, err = Datasets(name)
			case DatasetFilesystem:
				datasets, err = Filesystems(name)
			case DatasetSnapshot:
				datasets, err = Snapshots(name)
			case DatasetVolume:
				datasets, err = Volumes(name)
			}
			if err != nil {
				return err
			}
			out, err := json.MarshalIndent(datasets, "", "  ")
			if err != nil {
				return err
			}
			fmt.Printf("%s\n", out)
			return nil
		})
	cmdList.PersistentPreRun = dummyPersistentPreRun
	cmdList.Flags().StringP("type", "t", "all", strings.Join([]string{"all", DatasetFilesystem, DatasetVolume, DatasetSnapshot}, ","))

	cmdGet := genCommand("get", "Get dataset properties",
		func(cmd *cobra.Command, args []string) error {
			name, _ := cmd.Flags().GetString("name")

			ds, err := GetDataset(name)
			if err != nil {
				return err
			}

			out, err := json.MarshalIndent(ds, "", "  ")
			if err != nil {
				return err
			}
			fmt.Printf("%s\n", out)
			return nil
		})

	root.AddCommand(
		cmdExists,
		cmdDestroy,
		cmdHolds,
		cmdSnapshot,
		cmdRollback,
		cmdCreate,
		cmdSend,
		cmdClone,
		cmdRename,
		cmdList,
		cmdGet,
	)
	if err := root.Execute(); err != nil {
		log.Fatal("root execute failed:", err)
	}
}

func help(cmd *cobra.Command, _ []string) {
	if err := cmd.Help(); err != nil {
		log.Fatal("help failed:", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"syscall"
	"time"
	"bytes"

	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
)

const VERSIONS_PREFIX = ".sys.v#."
var GLOBAL_FLAGS *cliFlags

type cliFlags struct {
	dbUsername string
	dbPassword string
	dbHost     string
	dbPort     int
	dbName     string
	dryRun     bool
	eosMGMURL  string
	debug bool
	userPrefix string
}

func parseFlags() *cliFlags {
	flags := &cliFlags{}
	flag.StringVar(&flags.dbUsername, "username", "", "The username to connect to the db")
	flag.StringVar(&flags.dbPassword, "password", "", "The password to connect to the db")
	flag.StringVar(&flags.dbHost, "host", "", "The host of the db")
	flag.IntVar(&flags.dbPort, "port", 0, "The port of the db")
	flag.StringVar(&flags.dbName, "dbname", "", "The name of the database")
	flag.BoolVar(&flags.dryRun, "dryrun", true, "With dry run enbaled the changes are not commited to the db")
	flag.StringVar(&flags.eosMGMURL, "eosmgmurl", "root://eospps-slave.cern.ch", "The EOS MGM URL")
	flag.StringVar(&flags.userPrefix, "userPrefix", "/eos/scratch/user/", "The path under users reside")
	flag.BoolVar(&flags.debug, "debug", false, "Print debug information")
	flag.Parse()
	GLOBAL_FLAGS = flags
	return flags
}

type shareInfo struct {
	ID          int64          `db:"id"`
	ShareType   int            `db:"share_type"`
	ShareWith   sql.NullString `db:"share_with"`
	UIDOwner    string         `db:"uid_owner"`
	Parent      sql.NullInt64  `db:"parent"`
	ItemType    sql.NullString `db:"item_type"`
	ItemSource  sql.NullString `db:"item_source"`
	ItemTarget  sql.NullString `db:"item_target"`
	FileSource  sql.NullInt64  `db:"file_source"`
	FileTarget  sql.NullString `db:"file_target"`
	Permissions string         `db:"permissions"`
	STime       int            `db:"stime"`
	Accepted    int            `db:"accepted"`
	Expiration  time.Time      `db:"expiration"`
	Token       sql.NullString `db:"token"`
	MailSend    int            `db:"mail_send"`
}

type sqlDriver struct {
	db *sqlx.DB
}

func newSQLDriver(flags *cliFlags) (*sqlDriver, error) {
	dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", flags.dbUsername, flags.dbPassword, flags.dbHost, flags.dbPort, flags.dbName)
	d, err := sqlx.Connect("mysql", dsn)
	if err != nil {
		return nil, err
	}
	return &sqlDriver{db: d}, nil
}
func (d *sqlDriver) getAllShares() ([]shareInfo, error) {
	var entries []shareInfo
	err := d.db.Select(&entries, "SELECT id,share_type,item_source,item_target,file_source,file_target from oc_share where share_type=3 and item_type='file' ORDER BY id;")
	if err != nil {
		return nil, err
	}
	return entries, nil
}

type Metadata struct {
	Inode int64
	Path  string
	UID   string
	GID   string
}

func (d *sqlDriver) executeCMD(cmd *exec.Cmd) (string, string, error) {
	outBuf := &bytes.Buffer{}
	errBuf := &bytes.Buffer{}
	cmd.Stdout = outBuf
	cmd.Stderr = errBuf
	err := cmd.Run()
	if GLOBAL_FLAGS.debug {
		fmt.Printf("DEBUG CMD: %+v ERR:%s", cmd,  err)
	}
	return outBuf.String(), errBuf.String(), err
}

// getMetadataFromEos returns the metadata fo the file/folder given the inode
func (d *sqlDriver) getMetadataFromEOS(ID int64) (*Metadata, error) {
	cmd := exec.Command("/usr/bin/eos", "-r", "0", "0", "file", "info", fmt.Sprintf("inode:%d", ID), "-m")
	stdout, _, err := d.executeCMD(cmd)
	if err != nil {
		return nil, err
	}
	return d.parseFileInfo(stdout)
}

// getMetadataFromEosPath returns the metadata fo the file/folder given the eos path
func (d *sqlDriver) getMetadataFromEOSPath(path string) (*Metadata, error) {
	cmd := exec.Command("/usr/bin/eos", "-r", "0", "0", "file", "info", path, "-m")
	stdout, _, err := d.executeCMD(cmd)
	if err != nil {
		return nil, err
	}
	return d.parseFileInfo(stdout)
}

// getVersionsFolderMetadata returns the metadata associated to the versions folder of the file with the inode passed
// if the versions folder does not exists it will try to create it.
func (d *sqlDriver) getVersionsFolderMetadata(fileMeta *Metadata) (*Metadata, error) {
	// obtain the versions folder path from the file path
	dirName := path.Dir(fileMeta.Path)
	baseName := path.Base(fileMeta.Path)
	versionsPath := path.Join(dirName, VERSIONS_PREFIX+baseName)
	versionsMeta, err := d.getMetadataFromEOSPath(versionsPath)
	if err != nil {
		// if versions folder does not exists (eos err code == 2) trigger creation of this folder
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				if status.ExitStatus() == 2 {
					err = d.createVersionsFolder(fileMeta)
					if err != nil {
						return nil, err
					}

					// if we are talking to the slave the creation of the versions folder may not has been replicated, so we retry ever
					maxRetries := 5
					err := fmt.Errorf("Version not created yet")
					var versionsMeta *Metadata

					for maxRetries > 0 && err != nil {
						_versionsMeta, _err := d.getMetadataFromEOSPath(versionsPath)
						err = _err
						versionsMeta = _versionsMeta
						maxRetries--

					}
					if err != nil {
						return nil, err
					}
					return versionsMeta, nil
				}
			}
			return nil, err
		}
		return nil, err
	}
	return versionsMeta, nil
}

func (d *sqlDriver) parseFileInfo(raw string) (*Metadata, error) {
	kv := make(map[string]string)
	partsBySpace := strings.Split(raw, " ") // we have [keylength.file=14 file=/eos/pps/proc/ container=3 ...}
	for _, p := range partsBySpace {
		partsByEqual := strings.Split(p, "=") // we have kv pairs like [ keylength.file 14]
		if len(partsByEqual) == 2 {
			kv[partsByEqual[0]] = partsByEqual[1]
		}
	}
	// fix eos path because the kv pair file=path could contains whitespace and the whitespace is the pair separator. Not very smart :(
	fileLength := kv["keylength.file"]
	fileLengthInt64, err := strconv.ParseInt(fileLength, 10, 64)
	if err != nil {
		return nil, err
	}
	startIndex := int64(14) + int64(len(fileLength)) + 7
	kv["file"] = raw[startIndex : startIndex+fileLengthInt64]

	inodeInt64, err := strconv.ParseInt(kv["ino"], 10, 64)
	if err != nil {
		return nil, err
	}
	m := &Metadata{Inode: inodeInt64, Path: kv["file"], UID: kv["uid"], GID: kv["gid"]}
	return m, nil
}
func (d *sqlDriver) createVersionsFolder(fileMeta *Metadata) error {
	cmd := exec.Command("/usr/bin/eos", "-r", fileMeta.UID, fileMeta.GID, "cp", fileMeta.Path, fileMeta.Path)
	_, stderr, err := d.executeCMD(cmd)
	if err != nil {
		fmt.Fprintln(os.Stderr, stderr)
		return err
	}
	return nil
}
func (d *sqlDriver) updateShareTable(shareInfo *shareInfo, versionsMeta *Metadata) error {
	query := "UPDATE oc_share SET item_source=?,item_target=?,file_source=?,file_target=? WHERE id=?"
	stmt, err := d.db.Prepare(query)
	if err != nil {
		return err
	}
	defer stmt.Close()
	result, err := stmt.Exec(fmt.Sprintf("%d",versionsMeta.Inode), "/" + fmt.Sprintf("%d", versionsMeta.Inode), versionsMeta.Inode, "/" + path.Base(versionsMeta.Path), shareInfo.ID)
	if err != nil {
		return err
	}
	numRowsAffected, err := result.RowsAffected()
	if err != nil  {
		return err
	}
	if numRowsAffected == 0 || numRowsAffected > 1  {
		return fmt.Errorf("Cannot updated share because share id %d does not exists anymore", shareInfo.ID)
	}
	return nil
}
func main() {
	flags := parseFlags()
	os.Setenv("EOS_MGM_URL", flags.eosMGMURL)
	d, err := newSQLDriver(flags)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	shares, err := d.getAllShares()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Cannot get all shares because ", err)
		os.Exit(1)
	}
	if len(shares) == 0 {
		fmt.Fprintln(os.Stderr, "oc_share table does not contain public share files")
		os.Exit(1)
	}

	for _, s := range shares {
		meta, err := d.getMetadataFromEOS(s.FileSource.Int64)
		// TODO: filter already versions folder or ones that are not under your homedir like recycle files
		if strings.Contains(meta.Path, VERSIONS_PREFIX) {
			continue
		}
		if !strings.HasPrefix(meta.Path, GLOBAL_FLAGS.userPrefix) {
			continue
		}
		if err != nil {
			//fmt.Fprintln(os.Stderr, err)
		} else {
			fmt.Printf("info:file id:%d share_type:%d item_source:%s item_target:%s file_source:%d file_target:%s eospath:%s uid:%s gid:%s\n", s.ID, s.ShareType, s.ItemSource.String, s.ItemTarget.String, s.FileSource.Int64, s.FileTarget.String, strconv.Quote(meta.Path), meta.UID, meta.GID)
			versionsMeta, err := d.getVersionsFolderMetadata(meta)
			if err != nil {
				//fmt.Fprintln(os.Stderr, err)
			} else {
				fmt.Printf("info:versionfolder id:%d path:%s\n", versionsMeta.Inode, versionsMeta.Path)
			}
			err = d.updateShareTable(&s, versionsMeta)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
			} else {

			}
		}
	}
}
<commit_msg>Added dry run support to not commit changes to DB<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"syscall"
	"time"

	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
)

const VERSIONS_PREFIX = ".sys.v#."

var GLOBAL_FLAGS *cliFlags

type cliFlags struct {
	dbUsername string
	dbPassword string
	dbHost     string
	dbPort     int
	dbName     string
	dryRun     bool
	eosMGMURL  string
	debug      bool
	userPrefix string
}

func parseFlags() *cliFlags {
	flags := &cliFlags{}
	flag.StringVar(&flags.dbUsername, "username", "", "The username to connect to the db")
	flag.StringVar(&flags.dbPassword, "password", "", "The password to connect to the db")
	flag.StringVar(&flags.dbHost, "host", "", "The host of the db")
	flag.IntVar(&flags.dbPort, "port", 0, "The port of the db")
	flag.StringVar(&flags.dbName, "dbname", "", "The name of the database")
	flag.BoolVar(&flags.dryRun, "dryrun", false, "With dry run enbaled the changes are not commited to the db")
	flag.StringVar(&flags.eosMGMURL, "eosmgmurl", "root://eospps-slave.cern.ch", "The EOS MGM URL")
	flag.StringVar(&flags.userPrefix, "userprefix", "/eos/scratch/user/", "The path under users reside")
	flag.BoolVar(&flags.debug, "debug", false, "Print debug information")
	flag.Parse()
	GLOBAL_FLAGS = flags
	return flags
}

type shareInfo struct {
	ID          int64          `db:"id"`
	ShareType   int            `db:"share_type"`
	ShareWith   sql.NullString `db:"share_with"`
	UIDOwner    string         `db:"uid_owner"`
	Parent      sql.NullInt64  `db:"parent"`
	ItemType    sql.NullString `db:"item_type"`
	ItemSource  sql.NullString `db:"item_source"`
	ItemTarget  sql.NullString `db:"item_target"`
	FileSource  sql.NullInt64  `db:"file_source"`
	FileTarget  sql.NullString `db:"file_target"`
	Permissions string         `db:"permissions"`
	STime       int            `db:"stime"`
	Accepted    int            `db:"accepted"`
	Expiration  time.Time      `db:"expiration"`
	Token       sql.NullString `db:"token"`
	MailSend    int            `db:"mail_send"`
}

type sqlDriver struct {
	db *sqlx.DB
}

func newSQLDriver(flags *cliFlags) (*sqlDriver, error) {
	dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", flags.dbUsername, flags.dbPassword, flags.dbHost, flags.dbPort, flags.dbName)
	d, err := sqlx.Connect("mysql", dsn)
	if err != nil {
		return nil, err
	}
	return &sqlDriver{db: d}, nil
}
func (d *sqlDriver) getAllShares() ([]shareInfo, error) {
	var entries []shareInfo
	err := d.db.Select(&entries, "SELECT id,share_type,item_source,item_target,file_source,file_target from oc_share where share_type=3 and item_type='file' ORDER BY id;")
	if err != nil {
		return nil, err
	}
	return entries, nil
}

type Metadata struct {
	Inode int64
	Path  string
	UID   string
	GID   string
}

func (d *sqlDriver) executeCMD(cmd *exec.Cmd) (string, string, error) {
	outBuf := &bytes.Buffer{}
	errBuf := &bytes.Buffer{}
	cmd.Stdout = outBuf
	cmd.Stderr = errBuf
	err := cmd.Run()
	if GLOBAL_FLAGS.debug {
		fmt.Printf("DEBUG CMD: %+v ERR:%s", cmd, err)
	}
	return outBuf.String(), errBuf.String(), err
}

// getMetadataFromEos returns the metadata fo the file/folder given the inode
func (d *sqlDriver) getMetadataFromEOS(ID int64) (*Metadata, error) {
	cmd := exec.Command("/usr/bin/eos", "-r", "0", "0", "file", "info", fmt.Sprintf("inode:%d", ID), "-m")
	stdout, _, err := d.executeCMD(cmd)
	if err != nil {
		return nil, err
	}
	return d.parseFileInfo(stdout)
}

// getMetadataFromEosPath returns the metadata fo the file/folder given the eos path
func (d *sqlDriver) getMetadataFromEOSPath(path string) (*Metadata, error) {
	cmd := exec.Command("/usr/bin/eos", "-r", "0", "0", "file", "info", path, "-m")
	stdout, _, err := d.executeCMD(cmd)
	if err != nil {
		return nil, err
	}
	return d.parseFileInfo(stdout)
}

// getVersionsFolderMetadata returns the metadata associated to the versions folder of the file with the inode passed
// if the versions folder does not exists it will try to create it.
func (d *sqlDriver) getVersionsFolderMetadata(fileMeta *Metadata) (*Metadata, error) {
	// obtain the versions folder path from the file path
	dirName := path.Dir(fileMeta.Path)
	baseName := path.Base(fileMeta.Path)
	versionsPath := path.Join(dirName, VERSIONS_PREFIX+baseName)
	versionsMeta, err := d.getMetadataFromEOSPath(versionsPath)
	if err != nil {
		// if versions folder does not exists (eos err code == 2) trigger creation of this folder
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				if status.ExitStatus() == 2 {
					err = d.createVersionsFolder(fileMeta)
					if err != nil {
						return nil, err
					}

					// if we are talking to the slave the creation of the versions folder may not has been replicated, so we retry ever
					maxRetries := 5
					err := fmt.Errorf("Version not created yet")
					var versionsMeta *Metadata

					for maxRetries > 0 && err != nil {
						_versionsMeta, _err := d.getMetadataFromEOSPath(versionsPath)
						err = _err
						versionsMeta = _versionsMeta
						maxRetries--

					}
					if err != nil {
						return nil, err
					}
					return versionsMeta, nil
				}
			}
			return nil, err
		}
		return nil, err
	}
	return versionsMeta, nil
}

func (d *sqlDriver) parseFileInfo(raw string) (*Metadata, error) {
	kv := make(map[string]string)
	partsBySpace := strings.Split(raw, " ") // we have [keylength.file=14 file=/eos/pps/proc/ container=3 ...}
	for _, p := range partsBySpace {
		partsByEqual := strings.Split(p, "=") // we have kv pairs like [ keylength.file 14]
		if len(partsByEqual) == 2 {
			kv[partsByEqual[0]] = partsByEqual[1]
		}
	}
	// fix eos path because the kv pair file=path could contains whitespace and the whitespace is the pair separator. Not very smart :(
	fileLength := kv["keylength.file"]
	fileLengthInt64, err := strconv.ParseInt(fileLength, 10, 64)
	if err != nil {
		return nil, err
	}
	startIndex := int64(14) + int64(len(fileLength)) + 7
	kv["file"] = raw[startIndex : startIndex+fileLengthInt64]

	inodeInt64, err := strconv.ParseInt(kv["ino"], 10, 64)
	if err != nil {
		return nil, err
	}
	m := &Metadata{Inode: inodeInt64, Path: kv["file"], UID: kv["uid"], GID: kv["gid"]}
	return m, nil
}
func (d *sqlDriver) createVersionsFolder(fileMeta *Metadata) error {
	cmd := exec.Command("/usr/bin/eos", "-r", fileMeta.UID, fileMeta.GID, "cp", fileMeta.Path, fileMeta.Path)
	_, stderr, err := d.executeCMD(cmd)
	if err != nil {
		fmt.Fprintln(os.Stderr, stderr)
		return err
	}
	return nil
}
func (d *sqlDriver) updateShareTable(shareInfo *shareInfo, versionsMeta *Metadata) error {
	fmt.Printf("RECORD: %d UPDATE oc_share SET item_source=%s,item_target=%s,file_source=%d,file_target=%s WHERE id=%d\n", shareInfo.ID, fmt.Sprintf("%d", versionsMeta.Inode), "/"+fmt.Sprintf("%d", versionsMeta.Inode), versionsMeta.Inode, "/"+path.Base(versionsMeta.Path), shareInfo.ID)
	if GLOBAL_FLAGS.dryRun {
		return nil
	}
	query := "UPDATE oc_share SET item_source=?,item_target=?,file_source=?,file_target=? WHERE id=?"
	stmt, err := d.db.Prepare(query)
	if err != nil {
		return err
	}
	defer stmt.Close()
	result, err := stmt.Exec(fmt.Sprintf("%d", versionsMeta.Inode), "/"+fmt.Sprintf("%d", versionsMeta.Inode), versionsMeta.Inode, "/"+path.Base(versionsMeta.Path), shareInfo.ID)
	if err != nil {
		return err
	}
	numRowsAffected, err := result.RowsAffected()
	if err != nil {
		return err
	}
	if numRowsAffected == 0 || numRowsAffected > 1 {
		return fmt.Errorf("Cannot updated share because share id %d does not exists anymore", shareInfo.ID)
	}
	return nil
}
func main() {
	flags := parseFlags()
	os.Setenv("EOS_MGM_URL", flags.eosMGMURL)
	d, err := newSQLDriver(flags)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	shares, err := d.getAllShares()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Cannot get all shares because ", err)
		os.Exit(1)
	}
	if len(shares) == 0 {
		fmt.Fprintln(os.Stderr, "oc_share table does not contain public share files")
		os.Exit(1)
	}

	for _, s := range shares {
		meta, err := d.getMetadataFromEOS(s.FileSource.Int64)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
		} else {
			fmt.Printf("RECORD: %d info:file id:%d share_type:%d item_source:%s item_target:%s file_source:%d file_target:%s eospath:%s uid:%s gid:%s\n", s.ID, s.ID, s.ShareType, s.ItemSource.String, s.ItemTarget.String, s.FileSource.Int64, s.FileTarget.String, strconv.Quote(meta.Path), meta.UID, meta.GID)
			if strings.Contains(meta.Path, VERSIONS_PREFIX) {
				fmt.Printf("RECORD: %d ALREADY POINTS TO THE VERSION FOLDER\n", s.ID)
				continue
			}
			if !strings.HasPrefix(meta.Path, GLOBAL_FLAGS.userPrefix) {
				fmt.Printf("RECORD: %d FILE NOT UNDER HOME DIRECTORY\n", s.ID)
				continue
			}
			versionsMeta, err := d.getVersionsFolderMetadata(meta)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
			} else {
				fmt.Printf("RECORD: %d info:versionfolder id:%d path:%s\n", s.ID, versionsMeta.Inode, versionsMeta.Path)
			}
			err = d.updateShareTable(&s, versionsMeta)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
			}
		}
	}
	fmt.Printf("Sucess. Dry run: %t\n", GLOBAL_FLAGS.dryRun)
	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import(
	"fmt"
	"log"
	"net/http"
	"os"
	"gofire/web"
)

const StandardPort = "8080"

func main(){

	port := os.Getenv("PORT")

	if port == ""{
		port = StandardPort
	}

	http.HandleFunc("/", web.IndexHandler)
	log.Println("IndexHandler registered")

	http.HandleFunc("/login", web.LoginHandler)
	log.Println("LoginHandler registered")

	log.Printf("Server started on port: :%s", port)
	err := http.ListenAndServe(fmt.Sprintf(":%s", port),nil)
	panic(err)
}
<commit_msg>corrected spelling<commit_after>package main

import(
	"fmt"
	"log"
	"net/http"
	"os"
	"gofire/web"
)

const StandardPort = "8080"

func main(){

	port := os.Getenv("PORT")

	if port == ""{
		port = StandardPort
	}

	http.HandleFunc("/", web.IndexHandler)
	log.Println("IndexHandler registered")

	http.HandleFunc("/login", web.LoginHandler)
	log.Println("LoginHandler registered")

	http.HandleFunc("/chat", web.ChatHandler)
	log.Println("ChatHandler registered")

	log.Printf("Server started on port: :%s", port)
	err := http.ListenAndServe(fmt.Sprintf(":%s", port),nil)
	panic(err)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"errors"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/go-redis/cache"
	"github.com/go-redis/redis"
	"github.com/gofn/gofn"
	"github.com/gofn/gofn/provision"
	"github.com/gorilla/mux"
	"github.com/urfave/negroni"
	msgpack "gopkg.in/vmihailenco/msgpack.v2"
)

const (
	// coverQMax is the maximum number of coverage run to be executed simultaneously
	coverQMax = 0
	// coverQName is the Redis channel name where the requests are queued
	coverQName = "coverqueue"

	// inProgrsKey is the redis HSet key in which all repo + tags which are currently being run
	// are saved
	inProgrsKey = "cover-in-progress"

	// DefaultTag is the Go version to run the tests with when no version
	// is specified
	DefaultTag = "golang-1.10"
)

var (
	// errLogger is the log instance with all the required flags set for error logging
	errLogger = log.New(os.Stderr, "Cover.Run ", log.LstdFlags|log.Lshortfile)

	// coverQCur is the current number of cover run requests being executed simultaneously
	coverQCur = int32(0)

	httpClient = &http.Client{
		// img.shields.io response time is very slow
		Timeout: 30 * time.Second,
	}

	// ErrImgUnSupported is the error returned when the Go version requested is
	// not in the supported list
	ErrImgUnSupported = errors.New("Unsupported Go version provided")
	// ErrRepoNotFound is the error returned when the repository does not exist
	ErrRepoNotFound = errors.New("Repository not found")
	// ErrUnknown is the error returned when an unidentified error is encountered
	ErrUnknown = errors.New("Unknown error occurred")
	// ErrQueueFull is the error returned when the cover run queueu is full
	ErrQueueFull = errors.New("Cover run queue is full")
	// ErrCovInPrgrs is the error returned when the repo coverage test is in progress
	ErrCovInPrgrs = errors.New("Cover run is in progress")

	redisRing = redis.NewRing(&redis.RingOptions{
		Addrs: map[string]string{
			"server1": "redis:6379",
		},
	})
	redisCodec = &cache.Codec{
		Redis: redisRing,

		Marshal: func(v interface{}) ([]byte, error) {
			return msgpack.Marshal(v)
		},
		Unmarshal: func(b []byte, v interface{}) error {
			return msgpack.Unmarshal(b, v)
		},
	}
	redisClient = redis.NewClient(&redis.Options{
		Addr:         "redis:6379",
		ReadTimeout:  time.Second * 2,
		DialTimeout:  time.Second * 5,
		WriteTimeout: time.Second * 5,
		PoolTimeout:  time.Second * 120,
	})

	qLock = sync.Mutex{}

	repoTmpl = template.Must(template.ParseFiles("./templates/layout.tmpl", "./templates/repo.tmpl"))
	homeTmpl = template.Must(template.ParseFiles("./templates/layout.tmpl", "./templates/home.tmpl"))
)

// imageSupported returns true if the given Go version is supported
func imageSupported(tag string) bool {
	switch tag {
	case "golang-1.10",
		"golang-1.9",
		"golang-1.8":
		return true
	}
	return false
}

// repoExists checks if the given repository exists (works only if HTTP request returns 200)
func repoExists(repo string) (bool, error) {
	resp, err := httpClient.Get(fmt.Sprintf("https://%s", repo))
	if err != nil {
		return false, err
	}

	if resp.StatusCode == http.StatusNotFound {
		return false, ErrRepoNotFound
	}

	if resp.StatusCode > 399 {
		return false, ErrUnknown
	}
	return true, nil
}

// run runs the custom script to get the coverage details; using gofn
func run(imageRepoName, dockerTag, repo string) (string, string, error) {
	_, err := repoExists(repo)
	if err != nil {
		return "", "", err
	}

	buildOpts := &provision.BuildOptions{
		DoNotUsePrefixImageName: true,
		ImageName:               strings.ToLower(fmt.Sprintf("%s:%s", imageRepoName, dockerTag)),
		StdIN:                   fmt.Sprintf("sh /run.sh %s", repo),
	}

	// 5 minutes timeout
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
	defer cancel()

	StdOut, StdErr, err := gofn.Run(ctx, buildOpts, &provision.ContainerOptions{})
	if err != nil {
		errLogger.Println(err)
	}

	return StdOut, StdErr, err
}

// Object struct holds all the details of a repository
type Object struct {
	Repo   string
	Tag    string
	Cover  string
	Output bool
}

// repoFullName generates a name by combining the Go tag
func repoFullName(repo, tag string) string {
	return fmt.Sprintf("%s:%s", repo, tag)
}

// repoTagFromFullName returns the repo name and Go tag, given the generated full name
func repoTagFromFullName(msg string) (string, string) {
	parts := strings.Split(msg, ":")
	if len(parts) == 2 {
		return parts[0], parts[1]
	}
	return "", ""
}

// addToQ pushes a new cover run request to the Redis channel
func addToQ(repo, tag string) error {
	qLock.Lock()
	err := redisClient.Publish(coverQName, repoFullName(repo, tag)).Err()
	qLock.Unlock()
	return err
}

// repoCoverStatus returns true if a repository + tag cover run is in progress
func repoCoverStatus(repo, tag string) (bool, error) {
	// Check if cover run is already in progress for the given repo and tag
	err := redisRing.HGet(inProgrsKey, repoFullName(repo, tag)).Err()
	if err == nil {
		return true, nil
	}
	return false, err
}

// setInProgress sets the repo + tag as in progress by adding to inPrgorsKey
func setInProgress(repo, tag string) error {
	err := redisRing.HSet(inProgrsKey, repoFullName(repo, tag), "y").Err()
	if err != nil {
		errLogger.Println(err)
	}
	return err
}

// unsetInProgress unsets the repo + tag from inprogress status
func unsetInProgress(repo, tag string) error {
	err := redisRing.HDel(inProgrsKey, repoFullName(repo, tag)).Err()
	if err != nil {
		errLogger.Println(err)
	}
	return err
}

// cover evaluates the coverage of a repository
// - Before starting evaluation, it sets the repo's status as in progress
// - Removes the inprogress status of a repo after it's done
func cover(repo, tag string) error {
	atomic.AddInt32(&coverQCur, 1)

	err := setInProgress(repo, tag)
	if err != nil {
		errLogger.Println(err)
	}

	StdOut, StdErr, err := run("avelino/cover.run", tag, repo)
	if err != nil {
		errLogger.Println(err)
		err := unsetInProgress(repo, tag)
		if err != nil {
			errLogger.Println(err)
		}
		if coverQCur > -1 {
			atomic.AddInt32(&coverQCur, -1)
		}
		return err
	}

	err = unsetInProgress(repo, tag)
	if err != nil {
		errLogger.Println(err)
	}

	obj := &Object{
		Repo:   repo,
		Tag:    tag,
		Cover:  StdErr,
		Output: false,
	}

	stdOut := strings.Trim(StdOut, " \n")
	if stdOut != "" {
		obj.Cover = stdOut
		obj.Output = true
	}

	err = redisCodec.Set(&cache.Item{
		Key:        repoFullName(repo, tag),
		Object:     obj,
		Expiration: time.Hour,
	})
	if err != nil {
		errLogger.Println(err)
	}

	// reduce the simultaneous process number by 1
	if coverQCur > -1 {
		atomic.AddInt32(&coverQCur, -1)
	}
	return nil
}

// repoCover returns code coverage details for the given repository and Go version
// - It checks if the coverage details is available in cache or not
// - It checks if the cover run is in progress or not
// - It checks if cover can be run simultaneously, if not request is pushed to Q
func repoCover(repo, imageTag string) (*Object, error) {
	obj := &Object{
		Repo: repo,
		Tag:  imageTag,
	}

	if redisCodec.Get(repoFullName(repo, imageTag), &obj) == nil {
		return obj, nil
	}

	if !imageSupported(imageTag) {
		obj.Cover = fmt.Sprintf("Sorry, docker image not found, avelino/cover.run:%s, see Supported languages: https://github.com/avelino/cover.run#supported", imageTag)
		return obj, ErrImgUnSupported
	}

	if ok, _ := repoCoverStatus(repo, imageTag); ok {
		return obj, ErrCovInPrgrs
	}

	// Checking if limit of simultaneous processing has reached or not
	if coverQCur >= coverQMax {
		err := addToQ(repo, imageTag)
		if err != nil {
			errLogger.Println(err)
		}
		return obj, ErrQueueFull
	}

	go cover(repo, imageTag)

	return obj, ErrCovInPrgrs
}

type Repository struct {
	Repo  string
	Tag   string
	Cover string
}

func repoLatest() ([]*Repository, error) {
	repos := make([]*Repository, 0)
	keys, _, err := redisRing.Scan(0, "*", 10).Result()
	if err != nil {
		errLogger.Println(err)
		return repos, err
	}

	var obj Object
	for _, key := range keys {
		if len(repos) == 5 {
			return repos, nil
		}
		if err := redisCodec.Get(key, &obj); err == nil {
			if obj.Output {
				repos = append(repos, &Repository{obj.Repo, obj.Tag, obj.Cover})
			}
		}
	}
	return repos, nil
}

// subscribe subscribes to the Redis channel
func subscribe(qname string) {
	pubsub, err := redisClient.Subscribe(qname)
	defer pubsub.Close()
	if err != nil {
		errLogger.Println(err)
		return
	}

	for {
		msg, err := pubsub.ReceiveMessage()
		if err != nil {
			errLogger.Println(err)
			return
		}
		repo, tag := repoTagFromFullName(msg.Payload)
		if ok, _ := repoCoverStatus(repo, tag); !ok {
			cover(repo, tag)
		}
	}
}

func main() {
	n := negroni.Classic()
	r := mux.NewRouter()
	r.HandleFunc("/", Handler)
	r.HandleFunc("/go/{repo:.*}.json", HandlerRepoJSON)
	r.HandleFunc("/go/{repo:.*}.svg", HandlerRepoSVG)
	r.HandleFunc("/go/{repo:.*}", HandlerRepo)
	r.PathPrefix("/assets").Handler(
		http.StripPrefix("/assets", http.FileServer(http.Dir("./assets/"))))
	n.UseHandler(r)
	go subscribe("coverQueue")
	n.Run(":3000")
}
<commit_msg>removed unwanted logs. removed subscribe does not exist when recieve message fails<commit_after>package main

import (
	"context"
	"errors"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/go-redis/cache"
	"github.com/go-redis/redis"
	"github.com/gofn/gofn"
	"github.com/gofn/gofn/provision"
	"github.com/gorilla/mux"
	"github.com/urfave/negroni"
	msgpack "gopkg.in/vmihailenco/msgpack.v2"
)

const (
	// coverQMax is the maximum number of coverage run to be executed simultaneously
	coverQMax = 0
	// coverQName is the Redis channel name where the requests are queued
	coverQName = "coverqueue"

	// inProgrsKey is the redis HSet key in which all repo + tags which are currently being run
	// are saved
	inProgrsKey = "cover-in-progress"

	// DefaultTag is the Go version to run the tests with when no version
	// is specified
	DefaultTag = "golang-1.10"
)

var (
	// errLogger is the log instance with all the required flags set for error logging
	errLogger = log.New(os.Stderr, "Cover.Run ", log.LstdFlags|log.Lshortfile)

	// coverQCur is the current number of cover run requests being executed simultaneously
	coverQCur = int32(0)

	httpClient = &http.Client{
		// img.shields.io response time is very slow
		Timeout: 30 * time.Second,
	}

	// ErrImgUnSupported is the error returned when the Go version requested is
	// not in the supported list
	ErrImgUnSupported = errors.New("Unsupported Go version provided")
	// ErrRepoNotFound is the error returned when the repository does not exist
	ErrRepoNotFound = errors.New("Repository not found")
	// ErrUnknown is the error returned when an unidentified error is encountered
	ErrUnknown = errors.New("Unknown error occurred")
	// ErrQueueFull is the error returned when the cover run queueu is full
	ErrQueueFull = errors.New("Cover run queue is full")
	// ErrCovInPrgrs is the error returned when the repo coverage test is in progress
	ErrCovInPrgrs = errors.New("Cover run is in progress")

	redisRing = redis.NewRing(&redis.RingOptions{
		Addrs: map[string]string{
			"server1": "redis:6379",
		},
	})
	redisCodec = &cache.Codec{
		Redis: redisRing,

		Marshal: func(v interface{}) ([]byte, error) {
			return msgpack.Marshal(v)
		},
		Unmarshal: func(b []byte, v interface{}) error {
			return msgpack.Unmarshal(b, v)
		},
	}
	redisClient = redis.NewClient(&redis.Options{
		Addr:         "redis:6379",
		ReadTimeout:  time.Second * 2,
		DialTimeout:  time.Second * 5,
		WriteTimeout: time.Second * 5,
		PoolTimeout:  time.Second * 120,
	})

	qLock = sync.Mutex{}

	repoTmpl = template.Must(template.ParseFiles("./templates/layout.tmpl", "./templates/repo.tmpl"))
	homeTmpl = template.Must(template.ParseFiles("./templates/layout.tmpl", "./templates/home.tmpl"))
)

// imageSupported returns true if the given Go version is supported
func imageSupported(tag string) bool {
	switch tag {
	case "golang-1.10",
		"golang-1.9",
		"golang-1.8":
		return true
	}
	return false
}

// repoExists checks if the given repository exists (works only if HTTP request returns 200)
func repoExists(repo string) (bool, error) {
	resp, err := httpClient.Get(fmt.Sprintf("https://%s", repo))
	if err != nil {
		return false, err
	}

	if resp.StatusCode == http.StatusNotFound {
		return false, ErrRepoNotFound
	}

	if resp.StatusCode > 399 {
		return false, ErrUnknown
	}
	return true, nil
}

// run runs the custom script to get the coverage details; using gofn
func run(imageRepoName, dockerTag, repo string) (string, string, error) {
	_, err := repoExists(repo)
	if err != nil {
		return "", "", err
	}

	buildOpts := &provision.BuildOptions{
		DoNotUsePrefixImageName: true,
		ImageName:               strings.ToLower(fmt.Sprintf("%s:%s", imageRepoName, dockerTag)),
		StdIN:                   fmt.Sprintf("sh /run.sh %s", repo),
	}

	// 5 minutes timeout
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
	defer cancel()

	StdOut, StdErr, err := gofn.Run(ctx, buildOpts, &provision.ContainerOptions{})
	if err != nil {
		errLogger.Println(err)
	}

	return StdOut, StdErr, err
}

// Object struct holds all the details of a repository
type Object struct {
	Repo   string
	Tag    string
	Cover  string
	Output bool
}

// repoFullName generates a name by combining the Go tag
func repoFullName(repo, tag string) string {
	return fmt.Sprintf("%s:%s", repo, tag)
}

// repoTagFromFullName returns the repo name and Go tag, given the generated full name
func repoTagFromFullName(msg string) (string, string) {
	parts := strings.Split(msg, ":")
	if len(parts) == 2 {
		return parts[0], parts[1]
	}
	return "", ""
}

// addToQ pushes a new cover run request to the Redis channel
func addToQ(repo, tag string) error {
	qLock.Lock()
	err := redisClient.Publish(coverQName, repoFullName(repo, tag)).Err()
	qLock.Unlock()
	return err
}

// repoCoverStatus returns true if a repository + tag cover run is in progress
func repoCoverStatus(repo, tag string) (bool, error) {
	// Check if cover run is already in progress for the given repo and tag
	err := redisRing.HGet(inProgrsKey, repoFullName(repo, tag)).Err()
	if err == nil {
		return true, nil
	} else {
		errLogger.Println(err)
	}
	return false, err
}

// setInProgress sets the repo + tag as in progress by adding to inPrgorsKey
func setInProgress(repo, tag string) error {
	err := redisRing.HSet(inProgrsKey, repoFullName(repo, tag), "y").Err()
	if err != nil {
		errLogger.Println(err)
	}
	return err
}

// unsetInProgress unsets the repo + tag from inprogress status
func unsetInProgress(repo, tag string) error {
	err := redisRing.HDel(inProgrsKey, repoFullName(repo, tag)).Err()
	if err != nil {
		errLogger.Println(err)
	}
	return err
}

// cover evaluates the coverage of a repository
// - Before starting evaluation, it sets the repo's status as in progress
// - Removes the inprogress status of a repo after it's done
func cover(repo, tag string) error {
	atomic.AddInt32(&coverQCur, 1)

	setInProgress(repo, tag)

	StdOut, StdErr, err := run("avelino/cover.run", tag, repo)
	if err != nil {
		errLogger.Println(err)
		unsetInProgress(repo, tag)
		if coverQCur > -1 {
			atomic.AddInt32(&coverQCur, -1)
		}
		return err
	}

	unsetInProgress(repo, tag)

	obj := &Object{
		Repo:   repo,
		Tag:    tag,
		Cover:  StdErr,
		Output: false,
	}

	stdOut := strings.Trim(StdOut, " \n")
	if stdOut != "" {
		obj.Cover = stdOut
		obj.Output = true
	}

	err = redisCodec.Set(&cache.Item{
		Key:        repoFullName(repo, tag),
		Object:     obj,
		Expiration: time.Hour,
	})
	if err != nil {
		errLogger.Println(err)
	}

	// reduce the simultaneous process number by 1
	if coverQCur > -1 {
		atomic.AddInt32(&coverQCur, -1)
	}
	return nil
}

// repoCover returns code coverage details for the given repository and Go version
// - It checks if the coverage details is available in cache or not
// - It checks if the cover run is in progress or not
// - It checks if cover can be run simultaneously, if not request is pushed to Q
func repoCover(repo, imageTag string) (*Object, error) {
	obj := &Object{
		Repo: repo,
		Tag:  imageTag,
	}

	if redisCodec.Get(repoFullName(repo, imageTag), &obj) == nil {
		return obj, nil
	}

	if !imageSupported(imageTag) {
		obj.Cover = fmt.Sprintf("Sorry, docker image not found, avelino/cover.run:%s, see Supported languages: https://github.com/avelino/cover.run#supported", imageTag)
		return obj, ErrImgUnSupported
	}

	if ok, _ := repoCoverStatus(repo, imageTag); ok {
		return obj, ErrCovInPrgrs
	}

	// Checking if limit of simultaneous processing has reached or not
	if coverQCur >= coverQMax {
		err := addToQ(repo, imageTag)
		if err != nil {
			errLogger.Println(err)
		}
		return obj, ErrQueueFull
	}

	go cover(repo, imageTag)

	return obj, ErrCovInPrgrs
}

type Repository struct {
	Repo  string
	Tag   string
	Cover string
}

func repoLatest() ([]*Repository, error) {
	repos := make([]*Repository, 0)
	keys, _, err := redisRing.Scan(0, "*", 10).Result()
	if err != nil {
		errLogger.Println(err)
		return repos, err
	}

	var obj Object
	for _, key := range keys {
		if len(repos) == 5 {
			return repos, nil
		}
		if err := redisCodec.Get(key, &obj); err == nil {
			if obj.Output {
				repos = append(repos, &Repository{obj.Repo, obj.Tag, obj.Cover})
			}
		}
	}
	return repos, nil
}

// subscribe subscribes to the Redis channel
func subscribe(qname string) {
	pubsub, err := redisClient.Subscribe(qname)
	defer pubsub.Close()
	if err != nil {
		errLogger.Println(err)
		return
	}

	for {
		msg, err := pubsub.ReceiveMessage()
		if err != nil {
			errLogger.Println(err)
		}
		repo, tag := repoTagFromFullName(msg.Payload)
		if ok, _ := repoCoverStatus(repo, tag); !ok {
			cover(repo, tag)
		}
	}
}

func main() {
	n := negroni.Classic()
	r := mux.NewRouter()
	r.HandleFunc("/", Handler)
	r.HandleFunc("/go/{repo:.*}.json", HandlerRepoJSON)
	r.HandleFunc("/go/{repo:.*}.svg", HandlerRepoSVG)
	r.HandleFunc("/go/{repo:.*}", HandlerRepo)
	r.PathPrefix("/assets").Handler(
		http.StripPrefix("/assets", http.FileServer(http.Dir("./assets/"))))
	n.UseHandler(r)
	go subscribe("coverQueue")
	n.Run(":3000")
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/pkg/errors"
	"github.com/pkg/profile"
	"github.com/sirupsen/logrus"
	archipelago "github.com/verath/archipelago/lib"
)

const (
	httpReadTimeout  = 10 * time.Second
	httpWriteTimeout = 10 * time.Second
)

func main() {
	var (
		debug             bool
		serveStatic       bool
		skipWSOriginCheck bool
		serverAddr        string
		profileMode       string
	)
	flag.BoolVar(&debug, "debug", false, "Set to true to log debug messages.")
	flag.BoolVar(&serveStatic, "serveStatic", false, "Enable serving of static assets.")
	flag.BoolVar(&skipWSOriginCheck, "skipWSOriginCheck", false, "")
	flag.StringVar(&serverAddr, "addr", ":8080", "TCP address for the http server to listen on.")
	flag.StringVar(&profileMode, "profile", "", "Enable profiling mode, one of [cpu, mem, mutex, block]")
	flag.Parse()

	logger := logrus.New()
	logger.Formatter = &logrus.TextFormatter{}
	if debug {
		logger.Level = logrus.DebugLevel
		logger.Info("Debug logging enabled")
	}
	if serveStatic {
		logger.Info("Serving static resources.")
	}
	if skipWSOriginCheck {
		logger.Info("Skipping WebSocket origin check.")
	}
	if profileMode != "" {
		defer startProfiling(profileMode)()
	}

	archipelagoServer, err := archipelago.New(logger, skipWSOriginCheck)
	if err != nil {
		logger.Fatalf("Error creating game: %+v", err)
	}

	http.Handle("/ws", archipelagoServer.WebsocketHandler())
	http.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("OK"))
	})
	if serveStatic {
		staticPath := "./web/dist"
		http.Handle("/", http.FileServer(http.Dir(staticPath)))
	}
	httpServer := &http.Server{
		Addr:         serverAddr,
		ReadTimeout:  httpReadTimeout,
		WriteTimeout: httpWriteTimeout,
	}

	ctx, cancel := context.WithCancel(lifetimeContext(logger))
	errCh := make(chan error)
	go func() {
		err := archipelagoServer.Run(ctx)
		errCh <- errors.Wrap(err, "archipelagoServer error")
	}()
	go func() {
		err := runHTTPServer(ctx, httpServer)
		errCh <- errors.Wrap(err, "httpServer error")
	}()
	logger.Infof("Archipelago backend started: '%s'", serverAddr)
	err = <-errCh
	cancel()
	<-errCh
	if errors.Cause(err) == context.Canceled {
		logger.Debugf("Error caught in main: %v", err)
	} else {
		logger.Fatalf("Error caught in main: %+v", err)
	}
	logger.Info("Archipelago backend stopped.")
}

// startProfiling starts profiling for the given profileMode. See
// pkg/profile. Returns a function that should be called when profiling
// should be stopped.
func startProfiling(profileMode string) func() {
	switch profileMode {
	case "cpu":
		return profile.Start(profile.NoShutdownHook, profile.ProfilePath("."), profile.CPUProfile).Stop
	case "mem":
		return profile.Start(profile.NoShutdownHook, profile.ProfilePath("."), profile.MemProfile).Stop
	case "mutex":
		return profile.Start(profile.NoShutdownHook, profile.ProfilePath("."), profile.MutexProfile).Stop
	case "block":
		return profile.Start(profile.NoShutdownHook, profile.ProfilePath("."), profile.BlockProfile).Stop
	default:
		return func() {}
	}
}

// lifetimeContext returns a context that is cancelled on the first SIGINT, SIGTERM,
// or SIGKILL signal received. The application is force closed if more than one
// signal is received.
func lifetimeContext(logger *logrus.Logger) context.Context {
	ctx, cancel := context.WithCancel(context.Background())
	stopSigs := []os.Signal{os.Interrupt, os.Kill, syscall.SIGTERM}
	stopCh := make(chan os.Signal, len(stopSigs))
	signal.Notify(stopCh, stopSigs...)
	go func() {
		<-stopCh
		logger.Info("Caught interrupt, shutting down")
		cancel()
		<-stopCh
		logger.Fatal("Caught second interrupt, force closing")
	}()
	return ctx
}

// runHTTPServer starts and runs the given HTTP server until either an error
// occurs or the context is cancelled.
func runHTTPServer(ctx context.Context, server *http.Server) error {
	errCh := make(chan error)
	go func() {
		err := server.ListenAndServe()
		errCh <- errors.Wrap(err, "ListenAndServe error")
	}()
	select {
	case err := <-errCh:
		return err
	case <-ctx.Done():
		server.Close()
		return ctx.Err()
	}
}
<commit_msg>main: minor cleanup<commit_after>package main

import (
	"context"
	"flag"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/pkg/errors"
	"github.com/pkg/profile"
	"github.com/sirupsen/logrus"
	archipelago "github.com/verath/archipelago/lib"
)

const (
	httpReadTimeout  = 10 * time.Second
	httpWriteTimeout = 10 * time.Second
)

func main() {
	var (
		debug             bool
		serveStatic       bool
		skipWSOriginCheck bool
		httpAddr          string
		profileMode       string
	)
	flag.BoolVar(&debug, "debug", false, "Set to true to log debug messages.")
	flag.BoolVar(&serveStatic, "serveStatic", false, "Enable serving of static assets.")
	flag.BoolVar(&skipWSOriginCheck, "skipWSOriginCheck", false, "")
	flag.StringVar(&httpAddr, "addr", ":8080", "TCP address for the http server to listen on.")
	flag.StringVar(&profileMode, "profile", "", "Enable profiling mode, one of [cpu, mem, mutex, block]")
	flag.Parse()
	logger := logrus.New()
	logger.Formatter = &logrus.TextFormatter{}
	// Flags.
	if debug {
		logger.Level = logrus.DebugLevel
		logger.Info("Debug logging enabled")
	}
	if serveStatic {
		logger.Info("Serving static resources.")
	}
	if skipWSOriginCheck {
		logger.Info("Skipping WebSocket origin check.")
	}
	logger.Infof("HTTP server will listen on '%v'.", httpAddr)
	if profileMode != "" {
		defer startProfiling(profileMode)()
	}
	// Setup archipelago server instance.
	archipelagoServer, err := archipelago.New(logger, skipWSOriginCheck)
	if err != nil {
		logger.Fatalf("Error creating archipelago: %+v", err)
	}

	http.Handle("/ws", archipelagoServer.WebsocketHandler())
	// HTTP util routes.
	http.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("OK"))
	})
	if serveStatic {
		staticPath := "./web/dist"
		http.Handle("/", http.FileServer(http.Dir(staticPath)))
	}
	// Setup toplevel HTTP server.
	httpServer := &http.Server{
		Addr:         httpAddr,
		ReadTimeout:  httpReadTimeout,
		WriteTimeout: httpWriteTimeout,
	}
	// Run archipelago server, HTTP Server.
	err = runBackend(logger, archipelagoServer, httpServer)
	if errors.Cause(err) == context.Canceled {
		logger.Debugf("Error caught in main: %v", err)
	} else {
		logger.Fatalf("Error caught in main: %+v", err)
	}
	logger.Info("Archipelago backend stopped.")
}

// startProfiling starts profiling for the given profileMode. See
// pkg/profile. Returns a function that should be called when profiling
// should be stopped.
func startProfiling(profileMode string) func() {
	switch profileMode {
	case "cpu":
		return profile.Start(profile.NoShutdownHook, profile.ProfilePath("."), profile.CPUProfile).Stop
	case "mem":
		return profile.Start(profile.NoShutdownHook, profile.ProfilePath("."), profile.MemProfile).Stop
	case "mutex":
		return profile.Start(profile.NoShutdownHook, profile.ProfilePath("."), profile.MutexProfile).Stop
	case "block":
		return profile.Start(profile.NoShutdownHook, profile.ProfilePath("."), profile.BlockProfile).Stop
	default:
		return func() {}
	}
}

// lifetimeContext returns a context that is cancelled on the first SIGINT, SIGTERM,
// or SIGKILL signal received. The application is force closed if more than one
// signal is received.
func lifetimeContext(logger *logrus.Logger) context.Context {
	ctx, cancel := context.WithCancel(context.Background())
	stopSigs := []os.Signal{os.Interrupt, os.Kill, syscall.SIGTERM}
	stopCh := make(chan os.Signal, len(stopSigs))
	signal.Notify(stopCh, stopSigs...)
	go func() {
		<-stopCh
		logger.Info("Caught interrupt, shutting down")
		cancel()
		<-stopCh
		logger.Fatal("Caught second interrupt, force closing")
	}()
	return ctx
}

func runBackend(logger *logrus.Logger, archipelagoServer *archipelago.Server, httpServer *http.Server) error {
	ctx, cancel := context.WithCancel(lifetimeContext(logger))
	errCh := make(chan error)
	go func() {
		err := archipelagoServer.Run(ctx)
		errCh <- errors.Wrap(err, "archipelagoServer error")
	}()
	go func() {
		err := runHTTPServer(ctx, httpServer)
		errCh <- errors.Wrap(err, "httpServer error")
	}()
	logger.Info("Archipelago backend started.")
	err := <-errCh
	cancel()
	<-errCh
	return err
}

// runHTTPServer starts and runs the given HTTP server until either an error
// occurs or the context is cancelled.
func runHTTPServer(ctx context.Context, server *http.Server) error {
	errCh := make(chan error)
	go func() {
		err := server.ListenAndServe()
		errCh <- errors.Wrap(err, "ListenAndServe error")
	}()
	select {
	case err := <-errCh:
		return err
	case <-ctx.Done():
		server.Close()
		return ctx.Err()
	}
}
<|endoftext|>


<commit_before>package journal

import (
	"fmt"
	"log/syslog"
	"os"
	"testing"
	"time"
)

func TestJournalFollow(t *testing.T) {
	r, err := NewJournalReader(JournalReaderConfig{
		Since: time.Duration(-15) * time.Second,
		Matches: []Match{
			Match{
				Field: SD_JOURNAL_FIELD_SYSTEMD_UNIT,
				Value: "NetworkManager.service",
			},
		},
	})

	if err != nil {
		t.Fatalf("Error opening journal: %s", err)
	}

	if r == nil {
		t.Fatal("Got a nil reader")
	}

	defer r.Close()

	// start writing some test entries
	done := make(chan struct{}, 1)
	go func() {
		j, err := NewJournal()
		defer j.Close()

	writer:
		for {
			select {
			case <-done:
				break writer
			default:
				err = j.Print(int(syslog.LOG_INFO), fmt.Sprintf("test message %s", time.Now()))

				if err != nil {
					t.Fatalf("Error writing to journal: %s\n", err)
				}

				time.Sleep(time.Duration(1) * time.Second)
			}
		}
	}()

	// and follow the reader synchronously
	timeout := time.Duration(5) * time.Second
	err = r.Follow(time.After(timeout), os.Stdout)

	// shut down the test writer
	close(done)

	if err != nil {
		t.Fatalf("Error during follow: %s", err)
	}
}
<commit_msg>journal: cleanup test, use Print() instead of dropped j.Print()<commit_after>package journal

import (
	"os"
	"testing"
	"time"
)

func TestJournalFollow(t *testing.T) {
	r, err := NewJournalReader(JournalReaderConfig{
		Since: time.Duration(-15) * time.Second,
		Matches: []Match{
			{
				Field: SD_JOURNAL_FIELD_SYSTEMD_UNIT,
				Value: "NetworkManager.service",
			},
		},
	})

	if err != nil {
		t.Fatalf("Error opening journal: %s", err)
	}

	if r == nil {
		t.Fatal("Got a nil reader")
	}

	defer r.Close()

	// start writing some test entries
	done := make(chan struct{}, 1)
	go func() {
		j, err := NewJournal()
		defer j.Close()

		for {
			select {
			case <-done:
				return
			default:
				if err = Print(PriInfo, "test message %s", time.Now()); err != nil {
					t.Fatalf("Error writing to journal: %s", err)
				}

				time.Sleep(time.Second)
			}
		}
	}()

	// and follow the reader synchronously
	timeout := time.Duration(5) * time.Second
	err = r.Follow(time.After(timeout), os.Stdout)

	// shut down the test writer
	close(done)

	if err != nil {
		t.Fatalf("Error during follow: %s", err)
	}
}
<|endoftext|>


<commit_before>package broker_acceptance_test

import (
	"encoding/json"
	"fmt"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Common service tests", func() {

	Context("Shareable services", func() {

		shareableServices := map[string]bool{
			"elasticsearch": true,
			"influxdb":      true,
			"mysql":         true,
			"postgres":      true,
			"redis":         true,
			"aws-s3-bucket": true,
			"cdn-route":     false,
		}

		It("is service shareable", func() {

			retrieveServicesCommand := cf.Cf("curl", "/v3/service_offerings")

			Expect(retrieveServicesCommand.Wait(testConfig.DefaultTimeoutDuration())).To(Exit(0))

			var servicesCommandResp struct {
				Pagination struct {
					TotalResults int `json:"total_results"`
				}
				Resources []struct {
					Name          string `json:"name"`
					Description   string `json:"description"`
					BrokerCatalog struct {
						Metadata struct {
							Shareable bool `json:"shareable"`
						}
					} `json:"broker_catalog"`
				}
			}

			err := json.Unmarshal(retrieveServicesCommand.Buffer().Contents(), &servicesCommandResp)
			Expect(err).NotTo(HaveOccurred())

			var message string
			//during the run of the pipeline we are running few acceptance tests in parallel
			//this can bring a total number of services in the CF instance to greater than 7 (the normal count)
			//hence, while checking for 'shareable' services - we need to filter out the fake services created
			//by other tests.
			// fake service can be identified by having a value 'fake service' in desciption field.
			fakeServicesCount := 0
			for _, service := range servicesCommandResp.Resources {

				if service.Description == "fake service" {
					fakeServicesCount++
					continue
				}

				message = fmt.Sprintf("verifying that %s backing service is shareable", service.Name)
				By(message)
				if shareableServices[service.Name] {
					Expect(service.BrokerCatalog.Metadata.Shareable).NotTo(BeNil(),
						"Expected %s to have 'shareable' parameter", service.Name)

					Expect(service.BrokerCatalog.Metadata.Shareable).To(BeTrue(),
						"Expected %s to be shareable - i.e.: 'shareable' parameter set to 'true'", service.Name)
				} else {
					Expect(service.BrokerCatalog.Metadata.Shareable).To(BeFalse(),
						"Expected %s NOT to have 'shareable' parameter or to be set to 'false'", service.Name)
				}
			}

			Expect(servicesCommandResp.Pagination.TotalResults-fakeServicesCount).To(BeNumerically("==", len(shareableServices)), "the amount of services doesn't match")
		})
	})
})
<commit_msg>app-autoscaler: add to shared services test<commit_after>package broker_acceptance_test

import (
	"encoding/json"
	"fmt"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Common service tests", func() {

	Context("Shareable services", func() {

		shareableServices := map[string]bool{
			"autoscaler":    false,
			"elasticsearch": true,
			"influxdb":      true,
			"mysql":         true,
			"postgres":      true,
			"redis":         true,
			"aws-s3-bucket": true,
			"cdn-route":     false,
		}

		It("is service shareable", func() {

			retrieveServicesCommand := cf.Cf("curl", "/v3/service_offerings")

			Expect(retrieveServicesCommand.Wait(testConfig.DefaultTimeoutDuration())).To(Exit(0))

			var servicesCommandResp struct {
				Pagination struct {
					TotalResults int `json:"total_results"`
				}
				Resources []struct {
					Name          string `json:"name"`
					Description   string `json:"description"`
					BrokerCatalog struct {
						Metadata struct {
							Shareable bool `json:"shareable"`
						}
					} `json:"broker_catalog"`
				}
			}

			err := json.Unmarshal(retrieveServicesCommand.Buffer().Contents(), &servicesCommandResp)
			Expect(err).NotTo(HaveOccurred())

			var message string
			//during the run of the pipeline we are running few acceptance tests in parallel
			//this can bring a total number of services in the CF instance to greater than 7 (the normal count)
			//hence, while checking for 'shareable' services - we need to filter out the fake services created
			//by other tests.
			// fake service can be identified by having a value 'fake service' in desciption field.
			fakeServicesCount := 0
			for _, service := range servicesCommandResp.Resources {

				if service.Description == "fake service" {
					fakeServicesCount++
					continue
				}

				message = fmt.Sprintf("verifying that %s backing service is shareable", service.Name)
				By(message)
				if shareableServices[service.Name] {
					Expect(service.BrokerCatalog.Metadata.Shareable).NotTo(BeNil(),
						"Expected %s to have 'shareable' parameter", service.Name)

					Expect(service.BrokerCatalog.Metadata.Shareable).To(BeTrue(),
						"Expected %s to be shareable - i.e.: 'shareable' parameter set to 'true'", service.Name)
				} else {
					Expect(service.BrokerCatalog.Metadata.Shareable).To(BeFalse(),
						"Expected %s NOT to have 'shareable' parameter or to be set to 'false'", service.Name)
				}
			}

			Expect(servicesCommandResp.Pagination.TotalResults-fakeServicesCount).To(BeNumerically("==", len(shareableServices)), "the amount of services doesn't match")
		})
	})
})
<|endoftext|>


<commit_before>package main

import "testing"

func TestMagicSquare(t *testing.T) {
	row1 := []int{8, 1, 6}
	row2 := []int{3, 5, 7}
	row3 := []int{4, 9, 2}
	expected := [][]int{row1, row2, row3}
	result := magicSquare(3)

	if len(result) != 3 {
		t.Error("result is not long enough got: %v", result)
	}

	for i := range expected {
		if len(result[i]) != len(expected[i]) {
			t.Error("result is not long enough got: %v", result)
		}
		for j := range expected[i] {
			if expected[i][j] != result[i][j] {
				t.Error("Expected:\n %v for magic square got %v", expected, result)
			}
		}
	}
}
<commit_msg>little cleanup on the test<commit_after>package main

import "testing"

func TestMagicSquare(t *testing.T) {
	expected := [][]int{
		[]int{8, 1, 6},
		[]int{3, 5, 7},
		[]int{4, 9, 2},
	}

	result := magicSquare(3)

	if len(result) != 3 {
		t.Error("result is not long enough got: %v", result)
	}

	for i := range expected {
		if len(result[i]) != len(expected[i]) {
			t.Error("result is not long enough got: %v", result)
		}
		for j := range expected[i] {
			if expected[i][j] != result[i][j] {
				t.Error("Expected:\n %v for magic square got %v", expected, result)
			}
		}
	}
}
<|endoftext|>


<commit_before>package native

import (
	"time"

	"github.com/CyCoreSystems/ari"
)

type nativeBridge struct {
	conn          *Conn
	subscriber    ari.Subscriber
	playback      ari.Playback
	liveRecording ari.LiveRecording
}

func (b *nativeBridge) Playback() ari.Playback {
	return b.playback
}

func (b *nativeBridge) Create(id string, t string, name string) (bh *ari.BridgeHandle, err error) {

	type request struct {
		ID   string `json:"bridgeId,omitempty"`
		Type string `json:"type,omitempty"`
		Name string `json:"name,omitempty"`
	}

	req := request{id, t, name}
	var bd ari.BridgeData

	err = Post(b.conn, "/bridges/"+id, &req, &bd)
	if err != nil {
		return
	}

	bh = b.Get(bd.ID)
	return
}

func (b *nativeBridge) Get(id string) *ari.BridgeHandle {
	return ari.NewBridgeHandle(id, b)
}

func (b *nativeBridge) List() (bx []*ari.BridgeHandle, err error) {
	var bridges = []struct {
		ID string `json:"id"`
	}{}

	err = Get(b.conn, "/bridges", &bridges)
	for _, i := range bridges {
		bx = append(bx, b.Get(i.ID))
	}
	return
}

// Data returns the details of a bridge
// Equivalent to Get /bridges/{bridgeId}
func (b *nativeBridge) Data(id string) (bd ari.BridgeData, err error) {
	err = Get(b.conn, "/bridges/"+id, &bd)
	return
}

// AddChannel adds a channel to a bridge
// Equivalent to Post /bridges/{id}/addChannel
func (b *nativeBridge) AddChannel(bridgeID string, channelID string) (err error) {

	type request struct {
		ChannelID string `json:"channel"`
		Role      string `json:"role,omitempty"`
	}

	req := request{channelID, ""}
	err = Post(b.conn, "/bridges/"+bridgeID+"/addChannel", nil, &req)
	return
}

// RemoveChannel removes the specified channel from a bridge
// Equivalent to Post /bridges/{id}/removeChannel
func (b *nativeBridge) RemoveChannel(id string, channelID string) (err error) {
	req := struct {
		ChannelID string `json:"channel"`
	}{
		ChannelID: channelID,
	}

	//pass request
	err = Post(b.conn, "/bridges/"+id+"/removeChannel", nil, &req)
	return
}

// BridgeDelete shuts down a bridge. If any channels are in this bridge,
// they will be removed and resume whatever they were doing beforehand.
// This means that the channels themselves are not deleted.
// Equivalent to DELETE /bridges/{id}
func (b *nativeBridge) Delete(id string) (err error) {
	err = Delete(b.conn, "/bridges/"+id, nil, "")
	return
}

func (b *nativeBridge) Play(id string, playbackID string, mediaURI string) (ph *ari.PlaybackHandle, err error) {
	resp := make(map[string]interface{})
	type request struct {
		Media string `json:"media"`
	}
	req := request{mediaURI}
	err = Post(b.conn, "/bridges/"+id+"/play/"+playbackID, &resp, &req)
	ph = b.playback.Get(playbackID)
	return
}

func (b *nativeBridge) Record(id string, name string, opts *ari.RecordingOptions) (rh *ari.LiveRecordingHandle, err error) {

	if opts == nil {
		opts = &ari.RecordingOptions{}
	}

	resp := make(map[string]interface{})
	type request struct {
		Name        string `json:"name"`
		Format      string `json:"format"`
		MaxDuration int    `json:"maxDurationSeconds"`
		MaxSilence  int    `json:"maxSilenceSeconds"`
		IfExists    string `json:"ifExists,omitempty"`
		Beep        bool   `json:"beep"`
		TerminateOn string `json:"terminateOn,omitempty"`
	}
	req := request{
		Name:        name,
		Format:      opts.Format,
		MaxDuration: int(opts.MaxDuration / time.Second),
		MaxSilence:  int(opts.MaxSilence / time.Second),
		IfExists:    opts.Exists,
		Beep:        opts.Beep,
		TerminateOn: opts.Terminate,
	}
	err = Post(b.conn, "/bridges/"+id+"/record", &resp, &req)
	if err != nil {
		rh = b.liveRecording.Get(name)
	}
	return
}

func (b *nativeBridge) Subscribe(id string, n ...string) ari.Subscription {
	var ns nativeSubscription

	ns.events = make(chan ari.Event, 10)
	ns.closeChan = make(chan struct{})

	bridgeHandle := b.Get(id)

	go func() {
		sub := b.subscriber.Subscribe(n...)
		defer sub.Cancel()
		for {

			select {
			case <-ns.closeChan:
				ns.closeChan = nil
				return
			case evt := <-sub.Events():
				//TODO: do we want to send in events on the bridge for a specific channel?
				if bridgeHandle.Match(evt) {
					ns.events <- evt
				}
			}
		}
	}()

	return &ns
}
<commit_msg>Fix bridgehandle on create<commit_after>package native

import (
	"time"

	"github.com/CyCoreSystems/ari"
)

type nativeBridge struct {
	conn          *Conn
	subscriber    ari.Subscriber
	playback      ari.Playback
	liveRecording ari.LiveRecording
}

func (b *nativeBridge) Playback() ari.Playback {
	return b.playback
}

func (b *nativeBridge) Create(id string, t string, name string) (bh *ari.BridgeHandle, err error) {

	req := struct {
		ID   string `json:"bridgeId,omitempty"`
		Type string `json:"type,omitempty"`
		Name string `json:"name,omitempty"`
	}{
		ID:   id,
		Type: t,
		Name: name,
	}

	err = Post(b.conn, "/bridges/"+id, &req, nil)
	if err != nil {
		return
	}

	bh = b.Get(id)
	return
}

func (b *nativeBridge) Get(id string) *ari.BridgeHandle {
	return ari.NewBridgeHandle(id, b)
}

func (b *nativeBridge) List() (bx []*ari.BridgeHandle, err error) {
	var bridges = []struct {
		ID string `json:"id"`
	}{}

	err = Get(b.conn, "/bridges", &bridges)
	for _, i := range bridges {
		bx = append(bx, b.Get(i.ID))
	}
	return
}

// Data returns the details of a bridge
// Equivalent to Get /bridges/{bridgeId}
func (b *nativeBridge) Data(id string) (bd ari.BridgeData, err error) {
	err = Get(b.conn, "/bridges/"+id, &bd)
	return
}

// AddChannel adds a channel to a bridge
// Equivalent to Post /bridges/{id}/addChannel
func (b *nativeBridge) AddChannel(bridgeID string, channelID string) (err error) {

	type request struct {
		ChannelID string `json:"channel"`
		Role      string `json:"role,omitempty"`
	}

	req := request{channelID, ""}
	err = Post(b.conn, "/bridges/"+bridgeID+"/addChannel", nil, &req)
	return
}

// RemoveChannel removes the specified channel from a bridge
// Equivalent to Post /bridges/{id}/removeChannel
func (b *nativeBridge) RemoveChannel(id string, channelID string) (err error) {
	req := struct {
		ChannelID string `json:"channel"`
	}{
		ChannelID: channelID,
	}

	//pass request
	err = Post(b.conn, "/bridges/"+id+"/removeChannel", nil, &req)
	return
}

// BridgeDelete shuts down a bridge. If any channels are in this bridge,
// they will be removed and resume whatever they were doing beforehand.
// This means that the channels themselves are not deleted.
// Equivalent to DELETE /bridges/{id}
func (b *nativeBridge) Delete(id string) (err error) {
	err = Delete(b.conn, "/bridges/"+id, nil, "")
	return
}

func (b *nativeBridge) Play(id string, playbackID string, mediaURI string) (ph *ari.PlaybackHandle, err error) {
	resp := make(map[string]interface{})
	type request struct {
		Media string `json:"media"`
	}
	req := request{mediaURI}
	err = Post(b.conn, "/bridges/"+id+"/play/"+playbackID, &resp, &req)
	ph = b.playback.Get(playbackID)
	return
}

func (b *nativeBridge) Record(id string, name string, opts *ari.RecordingOptions) (rh *ari.LiveRecordingHandle, err error) {

	if opts == nil {
		opts = &ari.RecordingOptions{}
	}

	resp := make(map[string]interface{})
	type request struct {
		Name        string `json:"name"`
		Format      string `json:"format"`
		MaxDuration int    `json:"maxDurationSeconds"`
		MaxSilence  int    `json:"maxSilenceSeconds"`
		IfExists    string `json:"ifExists,omitempty"`
		Beep        bool   `json:"beep"`
		TerminateOn string `json:"terminateOn,omitempty"`
	}
	req := request{
		Name:        name,
		Format:      opts.Format,
		MaxDuration: int(opts.MaxDuration / time.Second),
		MaxSilence:  int(opts.MaxSilence / time.Second),
		IfExists:    opts.Exists,
		Beep:        opts.Beep,
		TerminateOn: opts.Terminate,
	}
	err = Post(b.conn, "/bridges/"+id+"/record", &resp, &req)
	if err != nil {
		rh = b.liveRecording.Get(name)
	}
	return
}

func (b *nativeBridge) Subscribe(id string, n ...string) ari.Subscription {
	var ns nativeSubscription

	ns.events = make(chan ari.Event, 10)
	ns.closeChan = make(chan struct{})

	bridgeHandle := b.Get(id)

	go func() {
		sub := b.subscriber.Subscribe(n...)
		defer sub.Cancel()
		for {

			select {
			case <-ns.closeChan:
				ns.closeChan = nil
				return
			case evt := <-sub.Events():
				//TODO: do we want to send in events on the bridge for a specific channel?
				if bridgeHandle.Match(evt) {
					ns.events <- evt
				}
			}
		}
	}()

	return &ns
}
<|endoftext|>