<commit_before>package main

/*
* Link Shortener, with a Redis backend.
*
* Released under and MIT License, please see the LICENSE.md file.
*
* John Nye
*
 */
import (
	"flag"
	"./utils"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"net/http"
	"github.com/garyburd/redigo/redis"
)

var host = flag.String("h", "localhost", "Bind address to listen on")
var base = flag.String("b", "http://localhost/", "Base URL for the shortener")
var port = flag.String("p", "8080", "Port you want to listen on, defaults to 8080")
var maxConnections = flag.Int("c", 512, "The maximum number of active connections")

const domain = "http://localhost:8080/"

type Data struct {
	Original  string
	Short     string
	FullShort string
	HitCount  int
}

//var collection []Data

func handler(w http.ResponseWriter, r *http.Request) {

	type NewURL struct {
		URL string
	}
	var url NewURL

	create, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if len(create) == 0 {
		var domain Data
		domain = getLongURL(r.URL.Path[1:])
		fmt.Println(domain)
		if len(domain.Original) > 0 {
			http.Redirect(w, r, domain.Original, http.StatusFound)
			return
		}
		http.Redirect(w, r, domain.Original, http.StatusNotFound)
		return
	}

	err = json.Unmarshal(create, &url)
	if err != nil {
		log.Print(err)
		http.Error(w, err.Error(), 500)
		return
	}

	conn, err := redis.Dial("tcp", ":6379")

	search := strings.Join([]string{"*||", url.URL},"")

	n, err := redis.Strings(conn.Do("KEYS", search));

	var newItem Data
	if len(n) < 1{
		newItem = createShortURL(url.URL)
	}else{
		parts :=  strings.Split(n[0], "||")

		newItem.Short = parts[0]
		newItem.Original = parts[1]
		newItem.FullShort = strings.Join([]string{*base, parts[0]}, "")
		newCount, err := redis.Int(conn.Do("HGET", n[0], "count"))
		if err == nil {
			//TODO..
		}
		newItem.HitCount = newCount
	}

	if err != nil {
		log.Print(err)
		http.Error(w, err.Error(), 500)
		return
	}

	//collection = append(collection, newItem)

	output, err := json.Marshal(newItem)

	if err != nil {
		log.Print(err)
		http.Error(w, err.Error(), 500)
		return
	}
	fmt.Fprintf(w, "%s", output)
}

func createShortURL(url string) Data {

	conn, err := redis.Dial("tcp", ":6379")
	var d Data
	count, err := redis.Int(conn.Do("INCR", "global:size"))
	if err != nil {
		log.Print(err)
		return d
	}
	encodedVar := base62.EncodeInt(int64(count))
	key := strings.Join([]string{encodedVar, url}, "||")
	conn.Send("MULTI")
	conn.Send("HSET", key, "count", 0)
	_, err2 := conn.Do("EXEC")

	if err2 != nil {
		log.Print(err)
		return d
	}

	d.Original = url
	d.HitCount = 0
	d.Short = encodedVar
	d.FullShort = strings.Join([]string{*base, encodedVar}, "")

	return d
}

func getLongURL(short string) Data {
	var d Data

	conn, err := redis.Dial("tcp", ":6379")

	search := strings.Join([]string{short, "||*"},"")
	fmt.Println(search)
	n, err := redis.Strings(conn.Do("KEYS", search));

	if err != nil {
		fmt.Println("Errors")
		log.Print(err)
		return d
	}

	if len(n) < 1{
		//Return an error

	}else{
		parts :=  strings.Split(n[0], "||")

		d.Short = parts[0]
		d.Original = parts[1]
		d.FullShort = strings.Join([]string{*base, parts[0]}, "")
		newCount, err := redis.Int(conn.Do("HINCRBY", n[0], "count",1))
		if err == nil {
			//TODO..
		}
		d.HitCount = newCount
		return d
	}

	return d
}

func main() {
	flag.Parse()
	http.HandleFunc("/", handler)
	err := http.ListenAndServe(*host+":"+*port, nil)
	if err != nil{
		fmt.Println(err)
	}
}
<commit_msg>dynamic Redis Address!<commit_after>package main

/*
* Link Shortener, with a Redis backend.
*
* Released under and MIT License, please see the LICENSE.md file.
*
* John Nye
*
 */
import (
	"flag"
	"./utils"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"net/http"
	"github.com/garyburd/redigo/redis"
)

var host = flag.String("h", "localhost", "Bind address to listen on")
var base = flag.String("b", "http://localhost/", "Base URL for the shortener")
var port = flag.String("p", "8080", "Port you want to listen on, defaults to 8080")
var maxConnections = flag.Int("c", 512, "The maximum number of active connections")
var redis = flag.String("r", "localhost:6379", "Redis Address, defaults to localhost:6379")

type Data struct {
	Original  string
	Short     string
	FullShort string
	HitCount  int
}

//var collection []Data

func handler(w http.ResponseWriter, r *http.Request) {

	type NewURL struct {
		URL string
	}
	var url NewURL

	create, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if len(create) == 0 {
		var domain Data
		domain = getLongURL(r.URL.Path[1:])
		fmt.Println(domain)
		if len(domain.Original) > 0 {
			http.Redirect(w, r, domain.Original, http.StatusFound)
			return
		}
		http.Redirect(w, r, domain.Original, http.StatusNotFound)
		return
	}

	err = json.Unmarshal(create, &url)
	if err != nil {
		log.Print(err)
		http.Error(w, err.Error(), 500)
		return
	}

	conn, err := redis.Dial("tcp", *redis)

	search := strings.Join([]string{"*||", url.URL},"")

	n, err := redis.Strings(conn.Do("KEYS", search));

	var newItem Data
	if len(n) < 1{
		newItem = createShortURL(url.URL)
	}else{
		parts :=  strings.Split(n[0], "||")

		newItem.Short = parts[0]
		newItem.Original = parts[1]
		newItem.FullShort = strings.Join([]string{*base, parts[0]}, "")
		newCount, err := redis.Int(conn.Do("HGET", n[0], "count"))
		if err == nil {
			//TODO..
		}
		newItem.HitCount = newCount
	}

	if err != nil {
		log.Print(err)
		http.Error(w, err.Error(), 500)
		return
	}

	//collection = append(collection, newItem)

	output, err := json.Marshal(newItem)

	if err != nil {
		log.Print(err)
		http.Error(w, err.Error(), 500)
		return
	}
	fmt.Fprintf(w, "%s", output)
}

func createShortURL(url string) Data {

	conn, err := redis.Dial("tcp", *redis)
	var d Data
	count, err := redis.Int(conn.Do("INCR", "global:size"))
	if err != nil {
		log.Print(err)
		return d
	}
	encodedVar := base62.EncodeInt(int64(count))
	key := strings.Join([]string{encodedVar, url}, "||")
	conn.Send("MULTI")
	conn.Send("HSET", key, "count", 0)
	_, err2 := conn.Do("EXEC")

	if err2 != nil {
		log.Print(err)
		return d
	}

	d.Original = url
	d.HitCount = 0
	d.Short = encodedVar
	d.FullShort = strings.Join([]string{*base, encodedVar}, "")

	return d
}

func getLongURL(short string) Data {
	var d Data

	conn, err := redis.Dial("tcp", *redis)

	search := strings.Join([]string{short, "||*"},"")
	fmt.Println(search)
	n, err := redis.Strings(conn.Do("KEYS", search));

	if err != nil {
		fmt.Println("Errors")
		log.Print(err)
		return d
	}

	if len(n) < 1{
		//Return an error

	}else{
		parts :=  strings.Split(n[0], "||")

		d.Short = parts[0]
		d.Original = parts[1]
		d.FullShort = strings.Join([]string{*base, parts[0]}, "")
		newCount, err := redis.Int(conn.Do("HINCRBY", n[0], "count",1))
		if err == nil {
			//TODO..
		}
		d.HitCount = newCount
		return d
	}

	return d
}

func main() {
	flag.Parse()
	http.HandleFunc("/", handler)
	err := http.ListenAndServe(*host+":"+*port, nil)
	if err != nil{
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>package downloader

import (
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/juju-core/log"
	"launchpad.net/tomb"
	"net/http"
	"os"
)

// Status represents the status of a completed download.
type Status struct {
	// URL holds the downloaded URL.
	URL string
	// Err describes any error encountered downloading the tools.
	Err error
	// File holds the file that the URL has downloaded to.
	File *os.File
}

// Downloader can download an archived directory from the network.
type Downloader struct {
	current *downloadOne
	done    chan Status
}

// New returns a new Downloader instance.
// Nothing will be downloaded until Start is called.
func New() *Downloader {
	return &Downloader{
		done: make(chan Status),
	}
}

// Start requests that the given URL be downloaded and written
// to the given Writer.
//
// If Start is called while another download is already in progress, the
// previous download will be cancelled.
func (d *Downloader) Start(url string) {
	if d.current != nil {
		d.Stop()
	}
	d.current = &downloadOne{
		url:  url,
		done: d.done,
	}
	go d.current.run()
}

// Stop stops any download that's in progress.
func (d *Downloader) Stop() {
	if d.current != nil {
		d.current.stop()
		d.current = nil
	}
}

// Done returns a channel that receives a value when
// some tools have been successfully downloaded and installed.
func (d *Downloader) Done() <-chan Status {
	return d.done
}

// downloadOne runs a single download attempt.
type downloadOne struct {
	tomb tomb.Tomb
	done chan Status
	url  string
	w io.Writer
}

func (d *downloadOne) stop() error {
	d.tomb.Kill(nil)
	return d.tomb.Wait()
}

func (d *downloadOne) run() {
	defer d.tomb.Done()
	file, err := d.download()
	if err != nil {
		err = fmt.Errorf("cannot download %q: %v", d.url, err)
	}
	status := Status{
		URL: d.url,
		File: file,
		Err: err,
	}
	// If we have been interrupted while downloading
	// then don't try to send the status.
	// This is to make tests easier - when we can interrupt
	// downloads, this can go away.
	select {
	case <-d.tomb.Dying():
		return
	default:
	}
	select {
	case d.done <- status:
	case <-d.tomb.Dying():
	}
}

func (d *downloadOne) download() (file *os.File, err error) {
	tmpFile, err := ioutil.TempFile("", "juju-download-")
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			tmpFile.Close()
			if err := os.Remove(tmpFile.Name()); err != nil {
				log.Printf("downloader: cannot remove temporary file: %v", err)
			}
		}
	}()
	resp, err := http.Get(d.url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("bad http response %v", resp.Status)
	}
	// TODO(rog) make the Copy operation interruptible.
	_, err = io.Copy(tmpFile, resp.Body)
	if err != nil {
		return nil, err
	}
	_, err = tmpFile.Seek(0, 0)
	if err != nil {
		return nil, err
	}
	return tmpFile, nil
}
<commit_msg>downloader: adjust comments<commit_after>package downloader

import (
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/juju-core/log"
	"launchpad.net/tomb"
	"net/http"
	"os"
)

// Status represents the status of a completed download.
// It is the receiver's responsibility to close and remove
// the file.
type Status struct {
	// URL holds the downloaded URL.
	URL string
	// Err describes any error encountered while downloading.
	Err error
	// File holds the file that the URL has downloaded to.
	File *os.File
}

// Downloader can download a file from the network.
type Downloader struct {
	current *downloadOne
	done    chan Status
}

// New returns a new Downloader instance.
// Nothing will be downloaded until Start is called.
func New() *Downloader {
	return &Downloader{
		done: make(chan Status),
	}
}

// Start requests that the given URL be downloaded and written
// to the given Writer.
//
// If Start is called while another download is already in progress, the
// previous download will be cancelled.
func (d *Downloader) Start(url string) {
	if d.current != nil {
		d.Stop()
	}
	d.current = &downloadOne{
		url:  url,
		done: d.done,
	}
	go d.current.run()
}

// Stop stops any download that's in progress.
func (d *Downloader) Stop() {
	if d.current != nil {
		d.current.stop()
		d.current = nil
	}
}

// Done returns a channel that receives a value when
// a file has been successfully downloaded and installed.
func (d *Downloader) Done() <-chan Status {
	return d.done
}

// downloadOne runs a single download attempt.
type downloadOne struct {
	tomb tomb.Tomb
	done chan Status
	url  string
	w io.Writer
}

func (d *downloadOne) stop() error {
	d.tomb.Kill(nil)
	return d.tomb.Wait()
}

func (d *downloadOne) run() {
	defer d.tomb.Done()
	file, err := d.download()
	if err != nil {
		err = fmt.Errorf("cannot download %q: %v", d.url, err)
	}
	status := Status{
		URL: d.url,
		File: file,
		Err: err,
	}
	// If we have been interrupted while downloading
	// then don't try to send the status.
	// This is to make tests easier - when we can interrupt
	// downloads, this can go away.
	select {
	case <-d.tomb.Dying():
		return
	default:
	}
	select {
	case d.done <- status:
	case <-d.tomb.Dying():
	}
}

func (d *downloadOne) download() (file *os.File, err error) {
	tmpFile, err := ioutil.TempFile("", "juju-download-")
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			tmpFile.Close()
			if err := os.Remove(tmpFile.Name()); err != nil {
				log.Printf("downloader: cannot remove temporary file: %v", err)
			}
		}
	}()
	// TODO(rog) make the Get operation interruptible.
	resp, err := http.Get(d.url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("bad http response %v", resp.Status)
	}
	_, err = io.Copy(tmpFile, resp.Body)
	if err != nil {
		return nil, err
	}
	_, err = tmpFile.Seek(0, 0)
	if err != nil {
		return nil, err
	}
	return tmpFile, nil
}
<|endoftext|>


<commit_before>// 2014, Lauri Peltomäki
// Licensed CC0, so go nuts.
package stack

import "errors"

var emptyStackError = errors.New("Attempting to Pop an empty stack")

type Stack struct {
	s []interface{}
}

func (s *Stack) Push(a interface{}) {
	s.s = append(s.s, a)
}

func (s *Stack) PopE() (interface{}, error) {
	if len(s.s) == 0 {
		return nil, emptyStackError
	}
	last := s.s[len(s.s)-1]
	s.s = s.s[:len(s.s)-1]
	return last, nil
}

func (s *Stack) Pop() interface{} {
	rv, err := s.PopE()
	if err != nil {
		return nil
	}
	return rv
}

func (s *Stack) Dup() error {
	rv, err := s.PopE()
	if err != nil {
		return err
	}
	s.Push(rv)
	s.Push(rv)
	return nil
}

func (s *Stack) Drop() {
	s.Pop()
}

func (s *Stack) Swap() error {
	r2, err := s.PopE()
	r1, err := s.PopE()
	if err != nil {
		return err
	}
	s.Push(r2)
	s.Push(r1)
	return nil
}

func (s *Stack) Over() error {
	r2, err := s.PopE()
	r1, err := s.PopE()
	if err != nil {
		return err
	}
	s.Push(r1)
	s.Push(r2)
	s.Push(r1)
	return nil
}

func (s *Stack) Rot() error {
	r3, err := s.PopE()
	r2, err := s.PopE()
	r1, err := s.PopE()
	if err != nil {
		return err
	}
	s.Push(r2)
	s.Push(r3)
	s.Push(r1)
	return nil
}
<commit_msg>A few small changes `inspired` by golint.<commit_after>// Package stack provides a general stack with some basic stack operation methods.
package stack

import "errors"

var errEmptyStack = errors.New("attempting to Pop an empty stack")

type Stack struct {
	s []interface{}
}

func (s *Stack) Push(a interface{}) {
	s.s = append(s.s, a)
}

func (s *Stack) PopE() (interface{}, error) {
	if len(s.s) == 0 {
		return nil, errEmptyStack
	}
	last := s.s[len(s.s)-1]
	s.s = s.s[:len(s.s)-1]
	return last, nil
}

func (s *Stack) Pop() interface{} {
	rv, err := s.PopE()
	if err != nil {
		return nil
	}
	return rv
}

func (s *Stack) Dup() error {
	rv, err := s.PopE()
	if err != nil {
		return err
	}
	s.Push(rv)
	s.Push(rv)
	return nil
}

func (s *Stack) Drop() {
	s.Pop()
}

func (s *Stack) Swap() error {
	r2, err := s.PopE()
	r1, err := s.PopE()
	if err != nil {
		return err
	}
	s.Push(r2)
	s.Push(r1)
	return nil
}

func (s *Stack) Over() error {
	r2, err := s.PopE()
	r1, err := s.PopE()
	if err != nil {
		return err
	}
	s.Push(r1)
	s.Push(r2)
	s.Push(r1)
	return nil
}

func (s *Stack) Rot() error {
	r3, err := s.PopE()
	r2, err := s.PopE()
	r1, err := s.PopE()
	if err != nil {
		return err
	}
	s.Push(r2)
	s.Push(r3)
	s.Push(r1)
	return nil
}
<|endoftext|>


<commit_before>package queue

import (
	"bytes"
	"fmt"
	"sort"
)

type ConnectionStat struct {
	UnackedCount int
	Consumers    []string
}

func (stat ConnectionStat) String() string {
	return fmt.Sprintf("[unacked:%d consumers:%d]",
		stat.UnackedCount,
		len(stat.Consumers),
	)
}

type ConnectionStats map[string]ConnectionStat

type QueueStat struct {
	ReadyCount      int
	RejectedCount   int
	ConnectionStats ConnectionStats
}

func NewQueueStat(readyCount, rejectedCount int) QueueStat {
	return QueueStat{
		ReadyCount:      readyCount,
		RejectedCount:   rejectedCount,
		ConnectionStats: ConnectionStats{},
	}
}

func (stat QueueStat) String() string {
	return fmt.Sprintf("[ready:%d rejected:%d conn:%s",
		stat.ReadyCount,
		stat.RejectedCount,
		stat.ConnectionStats,
	)
}

func (stat QueueStat) UnackedCount() int {
	unacked := 0
	for _, connectionStat := range stat.ConnectionStats {
		unacked += connectionStat.UnackedCount
	}
	return unacked
}

func (stat QueueStat) ConsumerCount() int {
	consumer := 0
	for _, connectionStat := range stat.ConnectionStats {
		consumer += len(connectionStat.Consumers)
	}
	return consumer
}

type QueueStats map[string]QueueStat

func CollectStats(mainConnection *Connection) QueueStats {
	queueStats := QueueStats{}
	for _, queueName := range mainConnection.GetOpenQueues() {
		queue := mainConnection.openQueue(queueName)
		queueStats[queueName] = NewQueueStat(queue.ReadyCount(), queue.RejectedCount())
	}

	connectionNames := mainConnection.GetConnections()
	for _, connectionName := range connectionNames {
		connection := mainConnection.hijackConnection(connectionName)
		queueNames := connection.GetConsumingQueues()

		for _, queueName := range queueNames {
			queue := connection.openQueue(queueName)
			consumers := queue.GetConsumers()

			queueStats[queueName].ConnectionStats[connectionName] = ConnectionStat{
				UnackedCount: queue.UnackedCount(),
				Consumers:    consumers,
			}
		}
	}

	return queueStats
}

func (stats QueueStats) String() string {
	var buffer bytes.Buffer

	for queueName, queueStat := range stats {
		buffer.WriteString(fmt.Sprintf("    queue:%s ready:%d rejected:%d unacked:%d consumers:%d\n",
			queueName, queueStat.ReadyCount, queueStat.RejectedCount, queueStat.UnackedCount(), queueStat.ConsumerCount(),
		))

		for connectionName, connectionStat := range queueStat.ConnectionStats {
			buffer.WriteString(fmt.Sprintf("        connection:%s unacked:%d consumers:%d\n",
				connectionName, connectionStat.UnackedCount, len(connectionStat.Consumers),
			))
		}
	}

	return buffer.String()
}

func (stats QueueStats) GetHtml() string {
	var buffer bytes.Buffer
	buffer.WriteString(`<html><body><table style="font-family:monospace">`)
	buffer.WriteString(`<tr><td>` +
		`queue</td><td></td><td>` +
		`ready</td><td></td><td>` +
		`rejected</td><td></td><td style="color:lightgrey">` +
		`connection</td><td></td><td>` +
		`unacked</td><td></td><td>` +
		`consumers</td><td></td></tr>`,
	)

	for _, queueName := range stats.sortedNames() {
		queueStat := stats[queueName]
		buffer.WriteString(fmt.Sprintf(`<tr><td>`+
			`%s</td><td></td><td>`+
			`%d</td><td></td><td>`+
			`%d</td><td></td><td>`+
			`%s</td><td></td><td>`+
			`%d</td><td></td><td>`+
			`%d</td><td></td></tr>`,
			queueName, queueStat.ReadyCount, queueStat.RejectedCount, "", queueStat.UnackedCount(), queueStat.ConsumerCount(),
		))

		for _, connectionName := range queueStat.ConnectionStats.sortedNames() {
			connectionStat := queueStat.ConnectionStats[connectionName]
			buffer.WriteString(fmt.Sprintf(`<tr style="color:lightgrey"><td>`+
				`%s</td><td></td><td>`+
				`%s</td><td></td><td>`+
				`%s</td><td></td><td>`+
				`%s</td><td></td><td>`+
				`%d</td><td></td><td>`+
				`%d</td><td></td></tr>`,
				"", "", "", connectionName, connectionStat.UnackedCount, len(connectionStat.Consumers),
			))
		}
	}

	buffer.WriteString(`</table></body></html>`)
	return buffer.String()
}

func (stats ConnectionStats) sortedNames() []string {
	var keys []string
	for key := range stats {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}

func (stats QueueStats) sortedNames() []string {
	var keys []string
	for key := range stats {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}
<commit_msg>queue.Stats show which connections are active<commit_after>package queue

import (
	"bytes"
	"fmt"
	"sort"
)

type ConnectionStat struct {
	Active       bool
	UnackedCount int
	Consumers    []string
}

func (stat ConnectionStat) String() string {
	return fmt.Sprintf("[unacked:%d consumers:%d]",
		stat.UnackedCount,
		len(stat.Consumers),
	)
}

func (stat ConnectionStat) ActiveSign() string {
	if stat.Active {
		return "✓"
	}
	return "✗"
}

type ConnectionStats map[string]ConnectionStat

type QueueStat struct {
	ReadyCount      int
	RejectedCount   int
	ConnectionStats ConnectionStats
}

func NewQueueStat(readyCount, rejectedCount int) QueueStat {
	return QueueStat{
		ReadyCount:      readyCount,
		RejectedCount:   rejectedCount,
		ConnectionStats: ConnectionStats{},
	}
}

func (stat QueueStat) String() string {
	return fmt.Sprintf("[ready:%d rejected:%d conn:%s",
		stat.ReadyCount,
		stat.RejectedCount,
		stat.ConnectionStats,
	)
}

func (stat QueueStat) UnackedCount() int {
	unacked := 0
	for _, connectionStat := range stat.ConnectionStats {
		unacked += connectionStat.UnackedCount
	}
	return unacked
}

func (stat QueueStat) ConsumerCount() int {
	consumer := 0
	for _, connectionStat := range stat.ConnectionStats {
		consumer += len(connectionStat.Consumers)
	}
	return consumer
}

type QueueStats map[string]QueueStat

func CollectStats(mainConnection *Connection) QueueStats {
	queueStats := QueueStats{}
	for _, queueName := range mainConnection.GetOpenQueues() {
		queue := mainConnection.openQueue(queueName)
		queueStats[queueName] = NewQueueStat(queue.ReadyCount(), queue.RejectedCount())
	}

	connectionNames := mainConnection.GetConnections()
	for _, connectionName := range connectionNames {
		connection := mainConnection.hijackConnection(connectionName)
		queueNames := connection.GetConsumingQueues()

		for _, queueName := range queueNames {
			queue := connection.openQueue(queueName)
			consumers := queue.GetConsumers()

			queueStats[queueName].ConnectionStats[connectionName] = ConnectionStat{
				Active:       connection.Check(),
				UnackedCount: queue.UnackedCount(),
				Consumers:    consumers,
			}
		}
	}

	return queueStats
}

func (stats QueueStats) String() string {
	var buffer bytes.Buffer

	for queueName, queueStat := range stats {
		buffer.WriteString(fmt.Sprintf("    queue:%s ready:%d rejected:%d unacked:%d consumers:%d\n",
			queueName, queueStat.ReadyCount, queueStat.RejectedCount, queueStat.UnackedCount(), queueStat.ConsumerCount(),
		))

		for connectionName, connectionStat := range queueStat.ConnectionStats {
			buffer.WriteString(fmt.Sprintf("        connection:%s unacked:%d consumers:%d active:%t\n",
				connectionName, connectionStat.UnackedCount, len(connectionStat.Consumers), connectionStat.Active,
			))
		}
	}

	return buffer.String()
}

func (stats QueueStats) GetHtml() string {
	var buffer bytes.Buffer
	buffer.WriteString(`<html><body><table style="font-family:monospace">`)
	buffer.WriteString(`<tr><td>` +
		`queue</td><td></td><td>` +
		`ready</td><td></td><td>` +
		`rejected</td><td></td><td>` +
		`</td><td></td><td style="color:lightgrey">` +
		`connection</td><td></td><td>` +
		`unacked</td><td></td><td>` +
		`consumers</td><td></td></tr>`,
	)

	for _, queueName := range stats.sortedNames() {
		queueStat := stats[queueName]
		buffer.WriteString(fmt.Sprintf(`<tr><td>`+
			`%s</td><td></td><td>`+
			`%d</td><td></td><td>`+
			`%d</td><td></td><td>`+
			`%s</td><td></td><td>`+
			`%s</td><td></td><td>`+
			`%d</td><td></td><td>`+
			`%d</td><td></td></tr>`,
			queueName, queueStat.ReadyCount, queueStat.RejectedCount, "", "", queueStat.UnackedCount(), queueStat.ConsumerCount(),
		))

		for _, connectionName := range queueStat.ConnectionStats.sortedNames() {
			connectionStat := queueStat.ConnectionStats[connectionName]
			buffer.WriteString(fmt.Sprintf(`<tr style="color:lightgrey"><td>`+
				`%s</td><td></td><td>`+
				`%s</td><td></td><td>`+
				`%s</td><td></td><td>`+
				`%s</td><td></td><td>`+
				`%s</td><td></td><td>`+
				`%d</td><td></td><td>`+
				`%d</td><td></td></tr>`,
				"", "", "", connectionStat.ActiveSign(), connectionName, connectionStat.UnackedCount, len(connectionStat.Consumers),
			))
		}
	}

	buffer.WriteString(`</table></body></html>`)
	return buffer.String()
}

func (stats ConnectionStats) sortedNames() []string {
	var keys []string
	for key := range stats {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}

func (stats QueueStats) sortedNames() []string {
	var keys []string
	for key := range stats {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/cespare/subcmd"
)

var cmds = []subcmd.Command{
	{
		Name:        "summarize",
		Description: "Display summary statistics for a sequence of numbers",
		Do:          summarize,
	},
}

const version = "0.1.0"

func main() {
	log.SetFlags(0)
	if len(os.Args) >= 2 {
		switch os.Args[1] {
		case "-version", "--version", "-v":
			fmt.Println(version)
			return
		}
	}
	subcmd.Run(cmds)
}
<commit_msg>Version 0.1.1<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/cespare/subcmd"
)

var cmds = []subcmd.Command{
	{
		Name:        "summarize",
		Description: "Display summary statistics for a sequence of numbers",
		Do:          summarize,
	},
}

const version = "0.1.1"

func main() {
	log.SetFlags(0)
	if len(os.Args) >= 2 {
		switch os.Args[1] {
		case "-version", "--version", "-v":
			fmt.Println(version)
			return
		}
	}
	subcmd.Run(cmds)
}
<|endoftext|>


<commit_before>package storable

import (
	"errors"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	// ErrNonNewDocument non-new documents cannot be inserted
	ErrNonNewDocument = errors.New("Cannot insert a non new document.")
	// ErrNewDocument a new documents cannot be updated
	ErrNewDocument = errors.New("Cannot updated a new document.")
	// ErrEmptyQueryInRaw an empty query cannot be used on any *Raw method
	ErrEmptyQueryInRaw = errors.New("Empty queries are not allowed on raw ops.")
	// ErrEmptyID a document without Id cannot be used with Save method
	ErrEmptyID = errors.New("A document without id is not allowed.")
)

type Store struct {
	db         *mgo.Database
	collection string
}

// NewStore returns a new Store instance
func NewStore(db *mgo.Database, collection string) *Store {
	return &Store{
		db:         db,
		collection: collection,
	}
}

// Insert insert the given document in the collection, returns error if no-new
// document is given. The document id is setted if is empty.
func (s *Store) Insert(doc DocumentBase) error {
	if !doc.IsNew() {
		return ErrNonNewDocument
	}

	if len(doc.GetId()) == 0 {
		doc.SetId(bson.NewObjectId())
	}

	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	err := c.Insert(doc)
	if err == nil {
		doc.SetIsNew(false)
	}

	return err
}

// Update update the given document in the collection, returns error if a new
// document is given.
func (s *Store) Update(doc DocumentBase) error {
	if doc.IsNew() {
		return ErrNewDocument
	}

	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	return c.UpdateId(doc.GetId(), doc)
}

// Save insert or update the given document in the collection, a document with
// id should be provided. Upsert is used (http://godoc.org/gopkg.in/mgo.v2#Collection.Upsert)
func (s *Store) Save(doc DocumentBase) (updated bool, err error) {
	id := doc.GetId()
	if len(id) == 0 {
		return false, ErrEmptyID
	}

	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	inf, err := c.UpsertId(id, doc)
	if err != nil {
		return false, err
	}

	doc.SetIsNew(false)
	return inf.Updated > 0, nil
}

// Delete remove the document from the collection
func (s *Store) Delete(doc DocumentBase) error {
	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	return c.RemoveId(doc.GetId())
}

// Find executes the given query in the collection
func (s *Store) Find(q Query) (*ResultSet, error) {
	sess, c := s.getSessionAndCollection()
	mq := c.Find(q.GetCriteria())

	if !q.GetSort().IsEmpty() {
		mq.Sort(q.GetSort().ToList()...)
	}

	if q.GetSkip() != 0 {
		mq.Skip(q.GetSkip())
	}

	if q.GetLimit() != 0 {
		mq.Limit(q.GetLimit())
	}

	if !q.GetSelect().IsEmpty() {
		mq.Select(q.GetSelect().ToMap())
	}

	return &ResultSet{session: sess, mgoQuery: mq}, nil
}

// RawUpdate performes a direct update in the collection, update is wrapped on
// a $set operator. If a query without criteria is given EmptyQueryInRawErr is
// returned
func (s *Store) RawUpdate(query Query, update interface{}, multi bool) error {
	criteria := query.GetCriteria()
	if len(criteria) == 0 {
		return ErrEmptyQueryInRaw
	}

	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	var err error
	if multi {
		_, err = c.UpdateAll(criteria, bson.M{"$set": update})
	} else {
		err = c.Update(criteria, bson.M{"$set": update})
	}

	return err
}

// RawDelete performes a direct remove in the collection. If a query without
// criteria is given EmptyQueryInRawErr is returned
func (s *Store) RawDelete(query Query, multi bool) error {
	criteria := query.GetCriteria()
	if len(criteria) == 0 {
		return ErrEmptyQueryInRaw
	}

	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	var err error
	if multi {
		_, err = c.RemoveAll(criteria)
	} else {
		err = c.Remove(criteria)
	}

	return err
}

func (s *Store) getSessionAndCollection() (*mgo.Session, *mgo.Collection) {
	sess := s.db.Session.Clone()

	return sess, sess.DB(s.db.Name).C(s.collection)
}
<commit_msg>using session.Copy instead of session.Clone<commit_after>package storable

import (
	"errors"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	// ErrNonNewDocument non-new documents cannot be inserted
	ErrNonNewDocument = errors.New("Cannot insert a non new document.")
	// ErrNewDocument a new documents cannot be updated
	ErrNewDocument = errors.New("Cannot updated a new document.")
	// ErrEmptyQueryInRaw an empty query cannot be used on any *Raw method
	ErrEmptyQueryInRaw = errors.New("Empty queries are not allowed on raw ops.")
	// ErrEmptyID a document without Id cannot be used with Save method
	ErrEmptyID = errors.New("A document without id is not allowed.")
)

type Store struct {
	db         *mgo.Database
	collection string
}

// NewStore returns a new Store instance
func NewStore(db *mgo.Database, collection string) *Store {
	return &Store{
		db:         db,
		collection: collection,
	}
}

// Insert insert the given document in the collection, returns error if no-new
// document is given. The document id is setted if is empty.
func (s *Store) Insert(doc DocumentBase) error {
	if !doc.IsNew() {
		return ErrNonNewDocument
	}

	if len(doc.GetId()) == 0 {
		doc.SetId(bson.NewObjectId())
	}

	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	err := c.Insert(doc)
	if err == nil {
		doc.SetIsNew(false)
	}

	return err
}

// Update update the given document in the collection, returns error if a new
// document is given.
func (s *Store) Update(doc DocumentBase) error {
	if doc.IsNew() {
		return ErrNewDocument
	}

	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	return c.UpdateId(doc.GetId(), doc)
}

// Save insert or update the given document in the collection, a document with
// id should be provided. Upsert is used (http://godoc.org/gopkg.in/mgo.v2#Collection.Upsert)
func (s *Store) Save(doc DocumentBase) (updated bool, err error) {
	id := doc.GetId()
	if len(id) == 0 {
		return false, ErrEmptyID
	}

	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	inf, err := c.UpsertId(id, doc)
	if err != nil {
		return false, err
	}

	doc.SetIsNew(false)
	return inf.Updated > 0, nil
}

// Delete remove the document from the collection
func (s *Store) Delete(doc DocumentBase) error {
	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	return c.RemoveId(doc.GetId())
}

// Find executes the given query in the collection
func (s *Store) Find(q Query) (*ResultSet, error) {
	sess, c := s.getSessionAndCollection()
	mq := c.Find(q.GetCriteria())

	if !q.GetSort().IsEmpty() {
		mq.Sort(q.GetSort().ToList()...)
	}

	if q.GetSkip() != 0 {
		mq.Skip(q.GetSkip())
	}

	if q.GetLimit() != 0 {
		mq.Limit(q.GetLimit())
	}

	if !q.GetSelect().IsEmpty() {
		mq.Select(q.GetSelect().ToMap())
	}

	return &ResultSet{session: sess, mgoQuery: mq}, nil
}

// RawUpdate performes a direct update in the collection, update is wrapped on
// a $set operator. If a query without criteria is given EmptyQueryInRawErr is
// returned
func (s *Store) RawUpdate(query Query, update interface{}, multi bool) error {
	criteria := query.GetCriteria()
	if len(criteria) == 0 {
		return ErrEmptyQueryInRaw
	}

	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	var err error
	if multi {
		_, err = c.UpdateAll(criteria, bson.M{"$set": update})
	} else {
		err = c.Update(criteria, bson.M{"$set": update})
	}

	return err
}

// RawDelete performes a direct remove in the collection. If a query without
// criteria is given EmptyQueryInRawErr is returned
func (s *Store) RawDelete(query Query, multi bool) error {
	criteria := query.GetCriteria()
	if len(criteria) == 0 {
		return ErrEmptyQueryInRaw
	}

	sess, c := s.getSessionAndCollection()
	defer sess.Close()

	var err error
	if multi {
		_, err = c.RemoveAll(criteria)
	} else {
		err = c.Remove(criteria)
	}

	return err
}

func (s *Store) getSessionAndCollection() (*mgo.Session, *mgo.Collection) {
	sess := s.db.Session.Copy()

	return sess, sess.DB(s.db.Name).C(s.collection)
}
<|endoftext|>


<commit_before>package rkv

import (
	"bytes"
	"errors"
	"fmt"
	"math/rand"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/relab/raft"
	commonpb "github.com/relab/raft/raftpb"
	cmdpb "github.com/relab/rkv/cmdpb"
)

// TODO Context timeout should depend on heartbeat setting.

// ErrSessionExpired indicates that the client session doesn't exist or has
// already expired.
var ErrSessionExpired = errors.New("session expired")

// Store is a key-value store backed by a map.
type Store struct {
	sync.RWMutex

	store   map[string]string
	clients map[string]uint64
	waiting map[string]chan interface{}

	raft raft.Raft
}

type readRequest struct {
	key   string
	value chan<- string
}

// NewStore returns a newly initialized store.
func NewStore(raft raft.Raft) *Store {
	s := &Store{
		store:   make(map[string]string),
		clients: make(map[string]uint64),
		waiting: make(map[string]chan interface{}),
		raft:    raft,
	}

	go s.run()

	return s
}

func (s *Store) run() {
	for {
		select {
		case entries := <-s.raft.Committed():
			for _, entry := range entries {
				s.handleEntry(&entry)
			}
		}
	}
}

func (s *Store) handleEntry(entry *commonpb.Entry) {
	s.Lock()
	defer s.Unlock()

	switch entry.EntryType {
	case commonpb.EntryNormal:
		if bytes.Equal(raft.NOOP, entry.Data) {
			return
		}

		var cmd cmdpb.Cmd
		err := cmd.Unmarshal(entry.Data)

		if err != nil {
			// TODO Ignore malformed requests. Log?
			return
		}

		var res interface{}

		switch cmd.CmdType {
		case cmdpb.Register:
			id := fmt.Sprintf("%d", entry.Index)
			res = id
			if _, ok := s.clients[id]; ok {
				// Don't reset sequence number if we receive a
				// duplicate.
				break
			}
			s.clients[id] = 0
		case cmdpb.Insert:
			// TODO We need to return ErrSessionExpired, when the
			// session is expired. TODO If we see seq > oldseq + 1,
			// we need to deal with that, i.e, we might have stale
			// data?
			if oldseq := s.clients[cmd.ClientID]; oldseq >= cmd.Seq {
				// Already applied.
				break
			}

			s.clients[cmd.ClientID] = cmd.Seq
			s.store[cmd.Key] = cmd.Value
		}

		cmdID := cmdUID(&cmd)
		if ch, ok := s.waiting[cmdID]; ok {
			ch <- res
			delete(s.waiting, cmdID)
		}
	}
}

// randSeq is used to uniquely identify register command. There is a chance of a
// collision, but I'll take those odds any day.
func randSeq() uint64 {
	return uint64(rand.Uint32())<<32 + uint64(rand.Uint32())
}

func cmdUID(cmd *cmdpb.Cmd) string {
	if cmd.CmdType == cmdpb.Insert {
		return fmt.Sprintf("INSERT:%s%d", cmd.ClientID, cmd.Seq)
	}

	return fmt.Sprintf("REGISTER:%d", cmd.Seq)
}

func (s *Store) Register() (string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()

	cmd := cmdpb.Cmd{CmdType: cmdpb.Register, Seq: randSeq()}
	b, err := cmd.Marshal()

	if err != nil {
		return "", err
	}

	done := make(chan interface{}, 1)
	// TODO Store cmds under clientID, so they can be expired.
	s.Lock()
	s.waiting[cmdUID(&cmd)] = done
	s.Unlock()

	if err := s.raft.ProposeCmd(ctx, b); err != nil {
		return "", err
	}

	select {
	case res := <-done:
		return res.(string), nil
	case <-ctx.Done():
		return "", ctx.Err()
	}
}

// Lookup gets a value from the map given a key. If an error is returned, the
// client should retry the same request. Note that if the key doesn't exist, the
// empty string is returned.
func (s *Store) Lookup(key string, allowStale bool) (string, error) {
	// TODO Buffer reads to amortize cost.

	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()

	if !allowStale {
		err := s.raft.Read(ctx)

		if err != nil {
			return "", err
		}
	}

	value := make(chan string, 1)

	go func() {
		s.RLock()
		defer s.RUnlock()
		value <- s.store[key]
	}()

	select {
	case v := <-value:
		return v, nil
	case <-ctx.Done():
		return "", ctx.Err()
	}
}

// Insert inserts value in map given a key. If an error is returned, the client
// should retry the same request.
func (s *Store) Insert(key, value, id string, seq uint64) error {
	err := s.validate(id, seq)

	if err != nil {
		return err
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()

	// TODO Check if session is valid. Make it a function.

	cmd := cmdpb.Cmd{
		CmdType:  cmdpb.Insert,
		ClientID: id,
		Seq:      seq,
		Key:      key,
		Value:    value,
	}
	b, err := cmd.Marshal()

	if err != nil {
		return err
	}

	done := make(chan interface{}, 1)
	s.Lock()
	s.waiting[cmdUID(&cmd)] = done
	s.Unlock()

	if err := s.raft.ProposeCmd(ctx, b); err != nil {
		return err
	}

	select {
	case <-done:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (s *Store) validate(id string, seq uint64) error {
	s.RLock()
	defer s.RUnlock()

	if oldSeq, ok := s.clients[id]; !ok || oldSeq != seq-1 {
		return ErrSessionExpired
	}

	return nil
}
<commit_msg>store.go: Wait till client registration has been committed<commit_after>package rkv

import (
	"bytes"
	"errors"
	"fmt"
	"math/rand"
	"strconv"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/relab/raft"
	commonpb "github.com/relab/raft/raftpb"
	cmdpb "github.com/relab/rkv/cmdpb"
)

// TODO Context timeout should depend on heartbeat setting.

// ErrSessionExpired indicates that the client session doesn't exist or has
// already expired.
var ErrSessionExpired = errors.New("session expired")

// Store is a key-value store backed by a map.
type Store struct {
	sync.RWMutex

	store       map[string]string
	clients     map[string]uint64
	waitCmd     map[string]chan interface{}
	waitApplied map[uint64][]chan struct{}

	applied uint64

	raft raft.Raft
}

type readRequest struct {
	key   string
	value chan<- string
}

// NewStore returns a newly initialized store.
func NewStore(raft raft.Raft) *Store {
	s := &Store{
		store:       make(map[string]string),
		clients:     make(map[string]uint64),
		waitCmd:     make(map[string]chan interface{}),
		waitApplied: make(map[uint64][]chan struct{}),
		raft:        raft,
	}

	go s.run()

	return s
}

func (s *Store) run() {
	for {
		select {
		case entries := <-s.raft.Committed():
			for _, entry := range entries {
				s.handleEntry(&entry)
			}
		}
	}
}

func (s *Store) handleEntry(entry *commonpb.Entry) {
	s.Lock()
	defer s.Unlock()

	switch entry.EntryType {
	case commonpb.EntryNormal:
		if bytes.Equal(raft.NOOP, entry.Data) {
			return
		}

		var cmd cmdpb.Cmd
		err := cmd.Unmarshal(entry.Data)

		if err != nil {
			// TODO Ignore malformed requests. Log?
			return
		}

		var res interface{}

		switch cmd.CmdType {
		case cmdpb.Register:
			id := fmt.Sprintf("%d", entry.Index)
			res = id
			if _, ok := s.clients[id]; ok {
				// Don't reset sequence number if we receive a
				// duplicate.
				break
			}
			s.clients[id] = 0
		case cmdpb.Insert:
			// TODO We need to return ErrSessionExpired, when the
			// session is expired. TODO If we see seq > oldseq + 1,
			// we need to deal with that, i.e, we might have stale
			// data?
			if oldseq := s.clients[cmd.ClientID]; oldseq >= cmd.Seq {
				// Already applied.
				break
			}

			s.clients[cmd.ClientID] = cmd.Seq
			s.store[cmd.Key] = cmd.Value
		}

		cmdID := cmdUID(&cmd)
		if ch, ok := s.waitCmd[cmdID]; ok {
			ch <- res
			delete(s.waitCmd, cmdID)
		}
	}

	if waiting, ok := s.waitApplied[entry.Index]; ok {
		for _, ch := range waiting {
			close(ch)
		}

		delete(s.waitApplied, entry.Index)
	}

	s.applied = entry.Index
}

// randSeq is used to uniquely identify register command. There is a chance of a
// collision, but I'll take those odds any day.
func randSeq() uint64 {
	return uint64(rand.Uint32())<<32 + uint64(rand.Uint32())
}

func cmdUID(cmd *cmdpb.Cmd) string {
	if cmd.CmdType == cmdpb.Insert {
		return fmt.Sprintf("INSERT:%s%d", cmd.ClientID, cmd.Seq)
	}

	return fmt.Sprintf("REGISTER:%d", cmd.Seq)
}

func (s *Store) Register() (string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()

	cmd := cmdpb.Cmd{CmdType: cmdpb.Register, Seq: randSeq()}
	b, err := cmd.Marshal()

	if err != nil {
		return "", err
	}

	done := make(chan interface{}, 1)
	// TODO Store cmds under clientID, so they can be expired.
	s.Lock()
	s.waitCmd[cmdUID(&cmd)] = done
	s.Unlock()

	if err := s.raft.ProposeCmd(ctx, b); err != nil {
		return "", err
	}

	select {
	case res := <-done:
		return res.(string), nil
	case <-ctx.Done():
		return "", ctx.Err()
	}
}

// Lookup gets a value from the map given a key. If an error is returned, the
// client should retry the same request. Note that if the key doesn't exist, the
// empty string is returned.
func (s *Store) Lookup(key string, allowStale bool) (string, error) {
	// TODO Buffer reads to amortize cost.

	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()

	if !allowStale {
		err := s.raft.Read(ctx)

		if err != nil {
			return "", err
		}
	}

	value := make(chan string, 1)

	go func() {
		s.RLock()
		defer s.RUnlock()
		value <- s.store[key]
	}()

	select {
	case v := <-value:
		return v, nil
	case <-ctx.Done():
		return "", ctx.Err()
	}
}

// Insert inserts value in map given a key. If an error is returned, the client
// should retry the same request.
func (s *Store) Insert(key, value, id string, seq uint64) error {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()

	err := s.validate(ctx, id, seq)

	if err != nil {
		return err
	}

	cmd := cmdpb.Cmd{
		CmdType:  cmdpb.Insert,
		ClientID: id,
		Seq:      seq,
		Key:      key,
		Value:    value,
	}
	b, err := cmd.Marshal()

	if err != nil {
		return err
	}

	done := make(chan interface{}, 1)
	s.Lock()
	s.waitCmd[cmdUID(&cmd)] = done
	s.Unlock()

	if err := s.raft.ProposeCmd(ctx, b); err != nil {
		return err
	}

	select {
	case <-done:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (s *Store) validate(ctx context.Context, id string, seq uint64) error {
	minIndex, err := strconv.ParseUint(id, 10, 64)

	if err != nil {
		return err
	}

	s.Lock()
	ci := s.applied

	if ci < minIndex {
		done := make(chan struct{})

		s.waitApplied[minIndex] = append(s.waitApplied[minIndex], done)
		s.Unlock()

		select {
		case <-done:
		case <-ctx.Done():
			return ctx.Err()
		}
	} else {
		s.Unlock()
	}

	s.RLock()
	defer s.RUnlock()

	if oldSeq, ok := s.clients[id]; !ok || oldSeq != seq-1 {
		return ErrSessionExpired
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"strconv"
	"sir"
)

func floyd(start int) (int, int) {
	tortoise := f[start]
	hare := f[f[start]]

	for tortoise != hare {
		tortoise = f[tortoise]
		hare = f[f[hare]]
	}

	mu := 0
	tortoise = start

	for tortoise != hare {
		tortoise = f[tortoise]
		hare = f[hare]
		mu += 1
	}

	lam := 1
	hare = f[tortoise]

	for tortoise != hare {
		hare = f[hare]
		lam += 1
	}

	return mu, lam
}

func brent(start int) (int, int) {
	power := 1
	lam := 1

	tortoise := start
	hare := f[start]

	for tortoise != hare {
		if power == lam {
			tortoise = hare
			power *= 2
			lam = 0
		}

		hare = f[hare]
		lam += 1
	}

	mu := 0
	tortoise = start
	hare = start

	for i := 0; i < lam; i++ {
		hare = f[hare]
	}

	for tortoise != hare {
		tortoise = f[tortoise]
		hare = f[hare]
		mu += 1
	}

	return mu, lam
}

var f []int

func main() {
	if len(os.Args) < 1 {
		log.Fatal("not enough args")
	}

	max := 0

	for i := 1; i < len(os.Args); i++ {
		arg, err := strconv.Atoi(os.Args[i])

		sir.CheckError(err)

		if arg > max {
			max = arg
		}
	}

	if len(os.Args[1:]) < max {
		log.Fatal(max, " points to a location not in array")
	}

	log.Println(max)

	f = make([]int, 0)

	for i := 1; i < len(os.Args); i++ {
		arg, err := strconv.Atoi(os.Args[i])

		sir.CheckError(err)

		f = append(f, arg)
	}

	log.Println("\t", "[start pos]", "[cycle start pos]", "[cycle len]")

	for i := 0; i < len(f); i++ {
		fpos, flen := floyd(i)
		log.Println("floyd", i, "\t", fpos, "\t", flen)

		bpos, blen := brent(i)
		log.Println("brent", i, "\t", bpos, "\t", blen)

		if fpos != bpos || flen != blen {
			log.Fatal("[error]", "floyd and brent do not agree!")
		}
	}

	log.Println(f)
}
<commit_msg>fmtttttt<commit_after>package main

import (
	"log"
	"os"
	"sir"
	"strconv"
)

func floyd(start int) (int, int) {
	tortoise := f[start]
	hare := f[f[start]]

	for tortoise != hare {
		tortoise = f[tortoise]
		hare = f[f[hare]]
	}

	mu := 0
	tortoise = start

	for tortoise != hare {
		tortoise = f[tortoise]
		hare = f[hare]
		mu += 1
	}

	lam := 1
	hare = f[tortoise]

	for tortoise != hare {
		hare = f[hare]
		lam += 1
	}

	return mu, lam
}

func brent(start int) (int, int) {
	power := 1
	lam := 1

	tortoise := start
	hare := f[start]

	for tortoise != hare {
		if power == lam {
			tortoise = hare
			power *= 2
			lam = 0
		}

		hare = f[hare]
		lam += 1
	}

	mu := 0
	tortoise = start
	hare = start

	for i := 0; i < lam; i++ {
		hare = f[hare]
	}

	for tortoise != hare {
		tortoise = f[tortoise]
		hare = f[hare]
		mu += 1
	}

	return mu, lam
}

var f []int

func main() {
	if len(os.Args) < 1 {
		log.Fatal("not enough args")
	}

	max := 0

	for i := 1; i < len(os.Args); i++ {
		arg, err := strconv.Atoi(os.Args[i])

		sir.CheckError(err)

		if arg > max {
			max = arg
		}
	}

	if len(os.Args[1:]) < max {
		log.Fatal(max, " points to a location not in array")
	}

	log.Println(max)

	f = make([]int, 0)

	for i := 1; i < len(os.Args); i++ {
		arg, err := strconv.Atoi(os.Args[i])

		sir.CheckError(err)

		f = append(f, arg)
	}

	log.Println("\t", "[start pos]", "[cycle start pos]", "[cycle len]")

	for i := 0; i < len(f); i++ {
		fpos, flen := floyd(i)
		log.Println("floyd", i, "\t", fpos, "\t", flen)

		bpos, blen := brent(i)
		log.Println("brent", i, "\t", bpos, "\t", blen)

		if fpos != bpos || flen != blen {
			log.Fatal("[error]", "floyd and brent do not agree!")
		}
	}

	log.Println(f)
}
<|endoftext|>


<commit_before>package gomodel

import (
	"database/sql"
	"fmt"
	"reflect"
	"strconv"

	"github.com/cosiner/gohper/goutil"
	"github.com/cosiner/gohper/reflect2"
	"github.com/cosiner/gohper/slices"
	"github.com/cosiner/gohper/strings2"
)

type (
	// SQLBuilder build sql for model using fields and where fields
	SQLBuilder func(fields, whereFields uint64) string

	// Table represent information of type
	// contains field count, table name, field names, field offsets
	//
	// because count sql and limit select sql is both stored in the same cache,
	// you should not use a empty fields for limit select, that will conflict with
	// count sql and get the wrong sql statement.
	Table struct {
		Name string
		Num  uint64
		cache

		columns        []string
		prefix         string // Name + "."
		colsCache      map[uint64]Cols
		typedColsCache map[uint64]Cols
	}
)

// FieldsIdentity create signature from fields
func FieldsIdentity(sqlType SQLType, numField, fields, whereFields uint64) uint64 {
	return fields<<numField | whereFields | uint64(sqlType)
}

// Stmt get sql from cache container, if cache not exist, then create new
func (t *Table) Stmt(prepare Preparer, sqlType SQLType, fields, whereFields uint64, build SQLBuilder) (*sql.Stmt, error) {
	id := FieldsIdentity(sqlType, t.Num, fields, whereFields)

	sql_, stmt, err := t.cache.GetStmt(prepare, id)
	if err != nil {
		return nil, err
	}

	if stmt == nil {
		sql_ = build(fields, whereFields)
		sqlPrinter.Print(false, sql_)

		return t.cache.SetStmt(prepare, id, sql_)
	}

	sqlPrinter.Print(true, sql_)

	return stmt, nil
}

func (t *Table) StmtInsert(prepare Preparer, fields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, INSERT, fields, 0, t.SQLInsert)
}

func (t *Table) StmtUpdate(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, UPDATE, fields, whereFields, t.SQLUpdate)
}

func (t *Table) StmtDelete(prepare Preparer, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, DELETE, 0, whereFields, t.SQLDelete)
}

func (t *Table) StmtLimit(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, LIMIT, fields, whereFields, t.SQLLimit)
}

func (t *Table) StmtOne(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, ONE, fields, whereFields, t.SQLOne)
}

func (t *Table) StmtAll(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, ALL, fields, whereFields, t.SQLAll)
}

func (t *Table) StmtCount(prepare Preparer, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, LIMIT, 0, whereFields, t.SQLCount)
}

func (t *Table) StmtIncrBy(prepare Preparer, field, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, INCRBY, field, whereFields, t.SQLIncrBy)
}

// Stmt get sql from cache container, if cache not exist, then create new
func (t *Table) Prepare(prepare Preparer, sqlType SQLType, fields, whereFields uint64, build SQLBuilder) (*sql.Stmt, error) {
	id := FieldsIdentity(sqlType, t.Num, fields, whereFields)

	sql_, stmt, err := t.cache.PrepareSQL(prepare, id)
	if err != nil {
		return nil, err
	}

	if stmt == nil {
		sql_ = build(fields, whereFields)
		t.cache.SetSQL(id, sql_)
		sqlPrinter.Print(false, sql_)

		stmt, err = prepare.Prepare(sql_)
	} else {
		sqlPrinter.Print(true, sql_)
	}

	return stmt, err
}

func (t *Table) PrepareInsert(prepare Preparer, fields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, INSERT, fields, 0, t.SQLInsert)
}

func (t *Table) PrepareUpdate(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, UPDATE, fields, whereFields, t.SQLUpdate)
}

func (t *Table) PrepareDelete(prepare Preparer, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, DELETE, 0, whereFields, t.SQLDelete)
}

func (t *Table) PrepareLimit(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, LIMIT, fields, whereFields, t.SQLLimit)
}

func (t *Table) PrepareOne(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, ONE, fields, whereFields, t.SQLOne)
}

func (t *Table) PrepareAll(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, ALL, fields, whereFields, t.SQLAll)
}

func (t *Table) PrepareCount(prepare Preparer, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, LIMIT, 0, whereFields, t.SQLCount)
}

func (t *Table) PrepareIncrBy(prepare Preparer, field, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, INCRBY, field, whereFields, t.SQLIncrBy)
}

// InsertSQL create insert sql for given fields
func (t *Table) SQLInsert(fields, _ uint64) string {
	cols := t.Cols(fields)

	return fmt.Sprintf("INSERT INTO %s(%s) VALUES(%s)",
		t.Name,
		cols.String(),
		cols.OnlyParam())
}

// UpdateSQL create update sql for given fields
func (t *Table) SQLUpdate(fields, whereFields uint64) string {
	return fmt.Sprintf("UPDATE %s SET %s %s",
		t.Name,
		t.Cols(fields).Paramed(),
		t.Where(whereFields))
}

// DeleteSQL create delete sql for given fields
func (t *Table) SQLDelete(_, whereFields uint64) string {
	return fmt.Sprintf("DELETE FROM %s %s", t.Name, t.Where(whereFields))
}

// LimitSQL create select sql for given fields
func (t *Table) SQLLimit(fields, whereFields uint64) string {
	return fmt.Sprintf("SELECT %s FROM %s %s LIMIT ?, ?",
		t.Cols(fields),
		t.Name,
		t.Where(whereFields))
}

// LimitSQL create select sql for given fields
func (t *Table) SQLOne(fields, whereFields uint64) string {
	return fmt.Sprintf("SELECT %s FROM %s %s LIMIT 1",
		t.Cols(fields),
		t.Name,
		t.Where(whereFields))
}

// AllSQL create select sql for given fields
func (t *Table) SQLAll(fields, whereFields uint64) string {
	return fmt.Sprintf("SELECT %s FROM %s %s",
		t.Cols(fields),
		t.Name,
		t.Where(whereFields))
}

// SQLForCount create select count sql
func (t *Table) SQLCount(_, whereFields uint64) string {
	return fmt.Sprintf("SELECT COUNT(*) FROM %s %s",
		t.Name,
		t.Where(whereFields))
}

// SQLIncrBy create sql for increase/decrease field value
func (t *Table) SQLIncrBy(field, whereFields uint64) string {
	if n := NumFields(field); n != 1 {
		panic("IncrBy only allow update one field, but got " + strconv.Itoa(n))
	}

	col := t.Col(field)
	return fmt.Sprintf("UPDATE %s SET %s=%s+? %s",
		t.Name,
		col,
		col,
		t.Where(whereFields),
	)
}

// Where create where clause for given fields, the 'WHERE' word is included
func (t *Table) Where(fields uint64) string {
	cols := t.Cols(fields)
	if cols.Length() == 0 {
		return ""
	}

	return "WHERE " + cols.Join("=?", " AND ")
}

// TabWhere is slimilar with Where, but prepend a table name for each column
func (t *Table) TabWhere(fields uint64) string {
	cols := t.TabCols(fields)
	if cols.Length() != 0 {
		return ""
	}

	return "WHERE " + cols.Join("=?", " AND ")
}

// Cols return column names for given fields
// if fields is only one, return single column
// else return column slice
func (t *Table) Cols(fields uint64) Cols {
	cols := t.colsCache[fields]
	if cols == nil {
		cols = t.cols(fields, "")
		t.colsCache[fields] = cols
	}

	return cols
}

// Col return column name of field
func (t *Table) Col(field uint64) string {
	return t.Cols(field).String()
}

// TabCols return column names for given fields with type's table name as prefix
// like table.column
func (t *Table) TabCols(fields uint64) Cols {
	cols := t.typedColsCache[fields]
	if cols == nil {
		cols = t.cols(fields, t.prefix)
		t.typedColsCache[fields] = cols
	}

	return cols
}

// Col return column name of field
func (t *Table) TabCol(field uint64) string {
	return t.TabCols(field).String()
}

// cols get fields names, each field prepend a prefix string
// if fields count is 0, type emptyCols was returned
// if fields count is 1, type singleCols was returned
// otherwise, type cols was returned
func (t *Table) cols(fields uint64, prefix string) Cols {
	fieldNames := t.columns
	if colCount := NumFields(fields); colCount > 1 {
		names := make([]string, colCount)
		var index int
		for i, l := uint64(0), uint64(len(fieldNames)); i < l; i++ {
			if (1<<i)&fields != 0 {
				names[index] = prefix + fieldNames[i]
				index++
			}
		}

		return &cols{cols: names}
	} else if colCount == 1 {
		for i, l := uint64(0), uint64(len(fieldNames)); i < l; i++ {
			if (1<<i)&fields != 0 {
				return singleCol(prefix + fieldNames[i])
			}
		}
	}

	return _emptyCols
}

const (
	// _FIELD_TAG is tag name of database column
	_FIELD_TAG = "column"
)

// parse will first use field tag as column name, the tag key is 'column',
// if no tag specified, use field name's camel_case, disable a field by put 'notcol'
// in field tag
func parse(v Model, db *DB) *Table {
	typ := reflect2.IndirectType(v)
	num := typ.NumField()

	cols := make([]string, 0)
	for i := 0; i < num; i++ {
		field := typ.Field(i)
		col := field.Name
		// Exported + !(anonymous && structure)
		if goutil.IsExported(col) &&
			!(field.Anonymous &&
				field.Type.Kind() == reflect.Struct) {

			tagName := field.Tag.Get(_FIELD_TAG)
			if tagName == "-" {
				continue
			}
			if tagName != "" {
				col = tagName
			}

			cols = append(cols, strings2.ToSnake(col))
		}
	}

	if len(cols) > MAX_NUMFIELDS {
		panic(fmt.Sprint("can't register model with fields count over ", MAX_NUMFIELDS))
	}

	return &Table{
		Num:   uint64(num),
		Name:  v.Table(),
		cache: newCache(),

		columns:        slices.FitCapToLenForString(cols),
		prefix:         v.Table() + ".",
		colsCache:      make(map[uint64]Cols),
		typedColsCache: make(map[uint64]Cols),
	}
}
<commit_msg>fix bug for model fields count<commit_after>package gomodel

import (
	"database/sql"
	"fmt"
	"reflect"
	"strconv"

	"github.com/cosiner/gohper/goutil"
	"github.com/cosiner/gohper/reflect2"
	"github.com/cosiner/gohper/slices"
	"github.com/cosiner/gohper/strings2"
)

type (
	// SQLBuilder build sql for model using fields and where fields
	SQLBuilder func(fields, whereFields uint64) string

	// Table represent information of type
	// contains field count, table name, field names, field offsets
	//
	// because count sql and limit select sql is both stored in the same cache,
	// you should not use a empty fields for limit select, that will conflict with
	// count sql and get the wrong sql statement.
	Table struct {
		Name      string
		NumFields uint64
		cache

		columns        []string
		prefix         string // Name + "."
		colsCache      map[uint64]Cols
		typedColsCache map[uint64]Cols
	}
)

// FieldsIdentity create signature from fields
func FieldsIdentity(sqlType SQLType, numField, fields, whereFields uint64) uint64 {
	return fields<<numField | whereFields | uint64(sqlType)
}

// Stmt get sql from cache container, if cache not exist, then create new
func (t *Table) Stmt(prepare Preparer, sqlType SQLType, fields, whereFields uint64, build SQLBuilder) (*sql.Stmt, error) {
	id := FieldsIdentity(sqlType, t.NumFields, fields, whereFields)

	sql_, stmt, err := t.cache.GetStmt(prepare, id)
	if err != nil {
		return nil, err
	}

	if stmt == nil {
		sql_ = build(fields, whereFields)
		sqlPrinter.Print(false, sql_)

		return t.cache.SetStmt(prepare, id, sql_)
	}

	sqlPrinter.Print(true, sql_)

	return stmt, nil
}

func (t *Table) StmtInsert(prepare Preparer, fields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, INSERT, fields, 0, t.SQLInsert)
}

func (t *Table) StmtUpdate(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, UPDATE, fields, whereFields, t.SQLUpdate)
}

func (t *Table) StmtDelete(prepare Preparer, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, DELETE, 0, whereFields, t.SQLDelete)
}

func (t *Table) StmtLimit(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, LIMIT, fields, whereFields, t.SQLLimit)
}

func (t *Table) StmtOne(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, ONE, fields, whereFields, t.SQLOne)
}

func (t *Table) StmtAll(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, ALL, fields, whereFields, t.SQLAll)
}

func (t *Table) StmtCount(prepare Preparer, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, LIMIT, 0, whereFields, t.SQLCount)
}

func (t *Table) StmtIncrBy(prepare Preparer, field, whereFields uint64) (*sql.Stmt, error) {
	return t.Stmt(prepare, INCRBY, field, whereFields, t.SQLIncrBy)
}

// Stmt get sql from cache container, if cache not exist, then create new
func (t *Table) Prepare(prepare Preparer, sqlType SQLType, fields, whereFields uint64, build SQLBuilder) (*sql.Stmt, error) {
	id := FieldsIdentity(sqlType, t.NumFields, fields, whereFields)

	sql_, stmt, err := t.cache.PrepareSQL(prepare, id)
	if err != nil {
		return nil, err
	}

	if stmt == nil {
		sql_ = build(fields, whereFields)
		t.cache.SetSQL(id, sql_)
		sqlPrinter.Print(false, sql_)

		stmt, err = prepare.Prepare(sql_)
	} else {
		sqlPrinter.Print(true, sql_)
	}

	return stmt, err
}

func (t *Table) PrepareInsert(prepare Preparer, fields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, INSERT, fields, 0, t.SQLInsert)
}

func (t *Table) PrepareUpdate(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, UPDATE, fields, whereFields, t.SQLUpdate)
}

func (t *Table) PrepareDelete(prepare Preparer, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, DELETE, 0, whereFields, t.SQLDelete)
}

func (t *Table) PrepareLimit(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, LIMIT, fields, whereFields, t.SQLLimit)
}

func (t *Table) PrepareOne(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, ONE, fields, whereFields, t.SQLOne)
}

func (t *Table) PrepareAll(prepare Preparer, fields, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, ALL, fields, whereFields, t.SQLAll)
}

func (t *Table) PrepareCount(prepare Preparer, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, LIMIT, 0, whereFields, t.SQLCount)
}

func (t *Table) PrepareIncrBy(prepare Preparer, field, whereFields uint64) (*sql.Stmt, error) {
	return t.Prepare(prepare, INCRBY, field, whereFields, t.SQLIncrBy)
}

// InsertSQL create insert sql for given fields
func (t *Table) SQLInsert(fields, _ uint64) string {
	cols := t.Cols(fields)

	return fmt.Sprintf("INSERT INTO %s(%s) VALUES(%s)",
		t.Name,
		cols.String(),
		cols.OnlyParam())
}

// UpdateSQL create update sql for given fields
func (t *Table) SQLUpdate(fields, whereFields uint64) string {
	return fmt.Sprintf("UPDATE %s SET %s %s",
		t.Name,
		t.Cols(fields).Paramed(),
		t.Where(whereFields))
}

// DeleteSQL create delete sql for given fields
func (t *Table) SQLDelete(_, whereFields uint64) string {
	return fmt.Sprintf("DELETE FROM %s %s", t.Name, t.Where(whereFields))
}

// LimitSQL create select sql for given fields
func (t *Table) SQLLimit(fields, whereFields uint64) string {
	return fmt.Sprintf("SELECT %s FROM %s %s LIMIT ?, ?",
		t.Cols(fields),
		t.Name,
		t.Where(whereFields))
}

// LimitSQL create select sql for given fields
func (t *Table) SQLOne(fields, whereFields uint64) string {
	return fmt.Sprintf("SELECT %s FROM %s %s LIMIT 1",
		t.Cols(fields),
		t.Name,
		t.Where(whereFields))
}

// AllSQL create select sql for given fields
func (t *Table) SQLAll(fields, whereFields uint64) string {
	return fmt.Sprintf("SELECT %s FROM %s %s",
		t.Cols(fields),
		t.Name,
		t.Where(whereFields))
}

// SQLForCount create select count sql
func (t *Table) SQLCount(_, whereFields uint64) string {
	return fmt.Sprintf("SELECT COUNT(*) FROM %s %s",
		t.Name,
		t.Where(whereFields))
}

// SQLIncrBy create sql for increase/decrease field value
func (t *Table) SQLIncrBy(field, whereFields uint64) string {
	if n := NumFields(field); n != 1 {
		panic("IncrBy only allow update one field, but got " + strconv.Itoa(n))
	}

	col := t.Col(field)

	return fmt.Sprintf("UPDATE %s SET %s=%s+? %s",
		t.Name,
		col,
		col,
		t.Where(whereFields),
	)
}

// Where create where clause for given fields, the 'WHERE' word is included
func (t *Table) Where(fields uint64) string {
	cols := t.Cols(fields)
	if cols.Length() == 0 {
		return ""
	}

	return "WHERE " + cols.Join("=?", " AND ")
}

// TabWhere is slimilar with Where, but prepend a table name for each column
func (t *Table) TabWhere(fields uint64) string {
	cols := t.TabCols(fields)
	if cols.Length() != 0 {
		return ""
	}

	return "WHERE " + cols.Join("=?", " AND ")
}

// Cols return column names for given fields
// if fields is only one, return single column
// else return column slice
func (t *Table) Cols(fields uint64) Cols {
	cols := t.colsCache[fields]
	if cols == nil {
		cols = t.cols(fields, "")
		t.colsCache[fields] = cols
	}

	return cols
}

// Col return column name of field
func (t *Table) Col(field uint64) string {
	return t.Cols(field).String()
}

// TabCols return column names for given fields with type's table name as prefix
// like table.column
func (t *Table) TabCols(fields uint64) Cols {
	cols := t.typedColsCache[fields]
	if cols == nil {
		cols = t.cols(fields, t.prefix)
		t.typedColsCache[fields] = cols
	}

	return cols
}

// Col return column name of field
func (t *Table) TabCol(field uint64) string {
	return t.TabCols(field).String()
}

// cols get fields names, each field prepend a prefix string
// if fields count is 0, type emptyCols was returned
// if fields count is 1, type singleCols was returned
// otherwise, type cols was returned
func (t *Table) cols(fields uint64, prefix string) Cols {
	fieldNames := t.columns
	if colCount := NumFields(fields); colCount > 1 {
		names := make([]string, colCount)
		var index int
		for i, l := uint64(0), uint64(len(fieldNames)); i < l; i++ {
			if (1<<i)&fields != 0 {
				names[index] = prefix + fieldNames[i]
				index++
			}
		}

		return &cols{cols: names}
	} else if colCount == 1 {
		for i, l := uint64(0), uint64(len(fieldNames)); i < l; i++ {
			if (1<<i)&fields != 0 {
				return singleCol(prefix + fieldNames[i])
			}
		}
	}

	return _emptyCols
}

const (
	// _FIELD_TAG is tag name of database column
	_FIELD_TAG = "column"
)

// parse will first use field tag as column name, the tag key is 'column',
// if no tag specified, use field name's camel_case, disable a field by put 'notcol'
// in field tag
func parse(v Model, db *DB) *Table {
	typ := reflect2.IndirectType(v)
	num := typ.NumField()

	cols := make([]string, 0)
	for i := 0; i < num; i++ {
		field := typ.Field(i)
		col := field.Name
		// Exported + !(anonymous && structure)
		if goutil.IsExported(col) &&
			!(field.Anonymous &&
				field.Type.Kind() == reflect.Struct) {

			tagName := field.Tag.Get(_FIELD_TAG)
			if tagName == "-" {
				continue
			}
			if tagName != "" {
				col = tagName
			}

			cols = append(cols, strings2.ToSnake(col))
		}
	}

	if len(cols) > MAX_NUMFIELDS {
		panic(fmt.Sprint("can't register model with fields count over ", MAX_NUMFIELDS))
	}

	cols = slices.FitCapToLenForString(cols)
	return &Table{
		NumFields: uint64(len(cols)),
		Name:      v.Table(),
		cache:     newCache(),

		columns:        cols,
		prefix:         v.Table() + ".",
		colsCache:      make(map[uint64]Cols),
		typedColsCache: make(map[uint64]Cols),
	}
}
<|endoftext|>


<commit_before>package slack

import (
	"context"
	"errors"
	"log"
	"net/url"
	"os"
	"fmt"
)

var logger stdLogger // A logger that can be set by consumers
/*
  Added as a var so that we can change this for testing purposes
*/
var SLACK_API string = "https://slack.com/api/"
var SLACK_WEB_API_FORMAT string = "https://%s.slack.com/api/users.admin.%s?t=%s"

type SlackResponse struct {
	Ok    bool   `json:"ok"`
	Error string `json:"error"`
}

type AuthTestResponse struct {
	URL    string `json:"url"`
	Team   string `json:"team"`
	User   string `json:"user"`
	TeamID string `json:"team_id"`
	UserID string `json:"user_id"`
}

type authTestResponseFull struct {
	SlackResponse
	AuthTestResponse
}

type Client struct {
	config struct {
		token string
	}
	info  Info
	debug bool
}

// stdLogger is a logger interface compatible with both stdlib and some
// 3rd party loggers such as logrus.
type stdLogger interface {
	Print(...interface{})
	Printf(string, ...interface{})
	Println(...interface{})

	Fatal(...interface{})
	Fatalf(string, ...interface{})
	Fatalln(...interface{})

	Panic(...interface{})
	Panicf(string, ...interface{})
	Panicln(...interface{})
}

// SetLogger let's library users supply a logger, so that api debugging
// can be logged along with the application's debugging info.
func SetLogger(l stdLogger) {
	logger = l
}

// New creates new Client.
func New(token string) *Client {
	s := &Client{}
	s.config.token = token
	return s
}

// AuthTest tests if the user is able to do authenticated requests or not
func (api *Client) AuthTest() (response *AuthTestResponse, error error) {
	return api.AuthTestContext(context.Background())
}

// AuthTestContext tests if the user is able to do authenticated requests or not with a custom context
func (api *Client) AuthTestContext(ctx context.Context) (response *AuthTestResponse, error error) {
	responseFull := &authTestResponseFull{}
	err := post(ctx, "auth.test", url.Values{"token": {api.config.token}}, responseFull, api.debug)
	if err != nil {
		return nil, err
	}
	if !responseFull.Ok {
		return nil, errors.New(responseFull.Error)
	}
	return &responseFull.AuthTestResponse, nil
}

// SetDebug switches the api into debug mode
// When in debug mode, it logs various info about what its doing
// If you ever use this in production, don't call SetDebug(true)
func (api *Client) SetDebug(debug bool) {
	api.debug = debug
	if debug && logger == nil {
		logger = log.New(os.Stdout, "nlopes/slack", log.LstdFlags|log.Lshortfile)
	}
}

func (api *Client) Debugf(format string, v ...interface{}) {
	if api.debug {
		logger.Output(2,fmt.Sprintf(format, v...))
	}
}

func (api *Client) Debugln(v ...interface{}) {
	if api.debug {
		logger.Output(2, fmt.Sprintln(v...))
	}
}
<commit_msg>nlopes/slack#232 - added missing Output method<commit_after>package slack

import (
	"context"
	"errors"
	"fmt"
	"log"
	"net/url"
	"os"
)

var logger stdLogger // A logger that can be set by consumers
/*
  Added as a var so that we can change this for testing purposes
*/
var SLACK_API string = "https://slack.com/api/"
var SLACK_WEB_API_FORMAT string = "https://%s.slack.com/api/users.admin.%s?t=%s"

type SlackResponse struct {
	Ok    bool   `json:"ok"`
	Error string `json:"error"`
}

type AuthTestResponse struct {
	URL    string `json:"url"`
	Team   string `json:"team"`
	User   string `json:"user"`
	TeamID string `json:"team_id"`
	UserID string `json:"user_id"`
}

type authTestResponseFull struct {
	SlackResponse
	AuthTestResponse
}

type Client struct {
	config struct {
		token string
	}
	info  Info
	debug bool
}

// stdLogger is a logger interface compatible with both stdlib and some
// 3rd party loggers such as logrus.
type stdLogger interface {
	Print(...interface{})
	Printf(string, ...interface{})
	Println(...interface{})

	Fatal(...interface{})
	Fatalf(string, ...interface{})
	Fatalln(...interface{})

	Panic(...interface{})
	Panicf(string, ...interface{})
	Panicln(...interface{})

	Output(int, string) error
}

// SetLogger let's library users supply a logger, so that api debugging
// can be logged along with the application's debugging info.
func SetLogger(l stdLogger) {
	logger = l
}

// New creates new Client.
func New(token string) *Client {
	s := &Client{}
	s.config.token = token
	return s
}

// AuthTest tests if the user is able to do authenticated requests or not
func (api *Client) AuthTest() (response *AuthTestResponse, error error) {
	return api.AuthTestContext(context.Background())
}

// AuthTestContext tests if the user is able to do authenticated requests or not with a custom context
func (api *Client) AuthTestContext(ctx context.Context) (response *AuthTestResponse, error error) {
	responseFull := &authTestResponseFull{}
	err := post(ctx, "auth.test", url.Values{"token": {api.config.token}}, responseFull, api.debug)
	if err != nil {
		return nil, err
	}
	if !responseFull.Ok {
		return nil, errors.New(responseFull.Error)
	}
	return &responseFull.AuthTestResponse, nil
}

// SetDebug switches the api into debug mode
// When in debug mode, it logs various info about what its doing
// If you ever use this in production, don't call SetDebug(true)
func (api *Client) SetDebug(debug bool) {
	api.debug = debug
	if debug && logger == nil {
		logger = log.New(os.Stdout, "nlopes/slack", log.LstdFlags|log.Lshortfile)
	}
}

func (api *Client) Debugf(format string, v ...interface{}) {
	if api.debug {
		logger.Output(2, fmt.Sprintf(format, v...))
	}
}

func (api *Client) Debugln(v ...interface{}) {
	if api.debug {
		logger.Output(2, fmt.Sprintln(v...))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/pkg/errors"
)

// Load a backup from the database
func backupLoadByName(s *state.State, project, name string) (*backup, error) {
	// Get the backup database record
	args, err := s.Cluster.ContainerGetBackup(project, name)
	if err != nil {
		return nil, errors.Wrap(err, "Load backup from database")
	}

	// Load the container it belongs to
	c, err := containerLoadById(s, args.ContainerID)
	if err != nil {
		return nil, errors.Wrap(err, "Load container from database")
	}

	// Return the backup struct
	return &backup{
		state:            s,
		container:        c,
		id:               args.ID,
		name:             name,
		creationDate:     args.CreationDate,
		expiryDate:       args.ExpiryDate,
		containerOnly:    args.ContainerOnly,
		optimizedStorage: args.OptimizedStorage,
	}, nil
}

// Create a new backup
func backupCreate(s *state.State, args db.ContainerBackupArgs, sourceContainer container) error {
	// Create the database entry
	err := s.Cluster.ContainerBackupCreate(args)
	if err != nil {
		if err == db.ErrAlreadyDefined {
			return fmt.Errorf("backup '%s' already exists", args.Name)
		}

		return errors.Wrap(err, "Insert backup info into database")
	}

	// Get the backup struct
	b, err := backupLoadByName(s, sourceContainer.Project(), args.Name)
	if err != nil {
		return errors.Wrap(err, "Load backup object")
	}

	// Now create the empty snapshot
	err = sourceContainer.Storage().ContainerBackupCreate(*b, sourceContainer)
	if err != nil {
		s.Cluster.ContainerBackupRemove(args.Name)
		return errors.Wrap(err, "Backup storage")
	}

	return nil
}

// backup represents a container backup
type backup struct {
	state     *state.State
	container container

	// Properties
	id               int
	name             string
	creationDate     time.Time
	expiryDate       time.Time
	containerOnly    bool
	optimizedStorage bool
}

type backupInfo struct {
	Project         string   `json:"project" yaml:"project"`
	Name            string   `json:"name" yaml:"name"`
	Backend         string   `json:"backend" yaml:"backend"`
	Privileged      bool     `json:"privileged" yaml:"privileged"`
	Pool            string   `json:"pool" yaml:"pool"`
	Snapshots       []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
	HasBinaryFormat bool     `json:"-" yaml:"-"`
}

// Rename renames a container backup
func (b *backup) Rename(newName string) error {
	oldBackupPath := shared.VarPath("backups", b.name)
	newBackupPath := shared.VarPath("backups", newName)

	// Create the new backup path
	backupsPath := shared.VarPath("backups", b.container.Name())
	if !shared.PathExists(backupsPath) {
		err := os.MkdirAll(backupsPath, 0700)
		if err != nil {
			return err
		}
	}

	// Rename the backup directory
	err := os.Rename(oldBackupPath, newBackupPath)
	if err != nil {
		return err
	}

	// Check if we can remove the container directory
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Rename the database record
	err = b.state.Cluster.ContainerBackupRename(b.name, newName)
	if err != nil {
		return err
	}

	return nil
}

// Delete removes a container backup
func (b *backup) Delete() error {
	return doBackupDelete(b.state, b.name, b.container.Name())
}

func (b *backup) Render() *api.ContainerBackup {
	return &api.ContainerBackup{
		Name:             strings.SplitN(b.name, "/", 2)[1],
		CreationDate:     b.creationDate,
		ExpiryDate:       b.expiryDate,
		ContainerOnly:    b.containerOnly,
		OptimizedStorage: b.optimizedStorage,
	}
}

func backupGetInfo(r io.ReadSeeker) (*backupInfo, error) {
	var tr *tar.Reader
	result := backupInfo{}
	hasBinaryFormat := false
	hasIndexFile := false

	// Extract
	r.Seek(0, 0)
	_, _, unpacker, err := shared.DetectCompressionFile(r)
	if err != nil {
		return nil, err
	}
	r.Seek(0, 0)

	if unpacker == nil {
		return nil, fmt.Errorf("Unsupported backup compression")
	}

	if len(unpacker) > 0 {
		var buf bytes.Buffer

		err := shared.RunCommandWithFds(r, &buf, unpacker[0], unpacker[1:]...)
		if err != nil {
			return nil, err
		}

		tr = tar.NewReader(&buf)
	} else {
		tr = tar.NewReader(r)
	}

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break // End of archive
		}
		if err != nil {
			return nil, err
		}

		if hdr.Name == "backup/index.yaml" {
			err = yaml.NewDecoder(tr).Decode(&result)
			if err != nil {
				return nil, err
			}

			hasIndexFile = true
		}

		if hdr.Name == "backup/container.bin" {
			hasBinaryFormat = true
		}
	}

	if !hasIndexFile {
		return nil, fmt.Errorf("Backup is missing index.yaml")
	}

	result.HasBinaryFormat = hasBinaryFormat
	return &result, nil
}

// fixBackupStoragePool changes the pool information in the backup.yaml. This
// is done only if the provided pool doesn't exist. In this case, the pool of
// the default profile will be used.
func backupFixStoragePool(c *db.Cluster, b backupInfo) error {
	// Get the default profile
	_, profile, err := c.ProfileGet("default", "default")
	if err != nil {
		return err
	}

	_, v, err := shared.GetRootDiskDevice(profile.Devices)
	if err != nil {
		return err
	}

	// Get the default's profile pool
	_, pool, err := c.StoragePoolGet(v["pool"])
	if err != nil {
		return err
	}

	f := func(path string) error {
		// Read in the backup.yaml file.
		backup, err := slurpBackupFile(path)
		if err != nil {
			return err
		}

		// Change the pool in the backup.yaml
		backup.Pool = pool
		backup.Container.Devices["root"]["pool"] = "default"

		file, err := os.Create(path)
		if err != nil {
			return err
		}
		defer file.Close()

		data, err := yaml.Marshal(&backup)
		if err != nil {
			return err
		}

		_, err = file.Write(data)
		if err != nil {
			return err
		}

		return nil
	}

	err = f(shared.VarPath("storage-pools", pool.Name, "containers", b.Name, "backup.yaml"))
	if err != nil {
		return err
	}

	for _, snap := range b.Snapshots {
		err = f(shared.VarPath("storage-pools", pool.Name, "containers-snapshots", b.Name, snap,
			"backup.yaml"))
		if err != nil {
			return err
		}
	}
	return nil
}

func backupCreateTarball(s *state.State, path string, backup backup) error {
	container := backup.container

	// Create the index
	pool, err := container.StoragePool()
	if err != nil {
		return err
	}

	indexFile := backupInfo{
		Name:       container.Name(),
		Backend:    container.Storage().GetStorageTypeName(),
		Privileged: container.IsPrivileged(),
		Pool:       pool,
		Snapshots:  []string{},
	}

	if !backup.containerOnly {
		snaps, err := container.Snapshots()
		if err != nil {
			return err
		}

		for _, snap := range snaps {
			_, snapName, _ := containerGetParentAndSnapshotName(snap.Name())
			indexFile.Snapshots = append(indexFile.Snapshots, snapName)
		}
	}

	data, err := yaml.Marshal(&indexFile)
	if err != nil {
		return err
	}

	file, err := os.Create(filepath.Join(path, "index.yaml"))
	if err != nil {
		return err
	}

	_, err = file.Write(data)
	file.Close()
	if err != nil {
		return err
	}

	// Create the target path if needed
	backupsPath := shared.VarPath("backups", backup.container.Name())
	if !shared.PathExists(backupsPath) {
		err := os.MkdirAll(backupsPath, 0700)
		if err != nil {
			return err
		}
	}

	// Create the tarball
	backupPath := shared.VarPath("backups", backup.name)
	args := []string{"-cf", backupPath, "--xattrs", "-C", path, "--transform", "s,^./,backup/,", "."}
	_, err = shared.RunCommand("tar", args...)
	if err != nil {
		return err
	}

	// Compress it
	compress, err := cluster.ConfigGetString(s.Cluster, "backups.compression_algorithm")
	if err != nil {
		return err
	}

	if compress != "none" {
		infile, err := os.Open(backupPath)
		if err != nil {
			return err
		}
		defer infile.Close()

		compressed, err := os.Create(backupPath + ".compressed")
		if err != nil {
			return err
		}
		defer compressed.Close()

		err = compressFile(compress, infile, compressed)
		if err != nil {
			return err
		}

		err = os.Remove(backupPath)
		if err != nil {
			return err
		}

		err = os.Rename(compressed.Name(), backupPath)
		if err != nil {
			return err
		}
	}

	// Set permissions
	err = os.Chmod(backupPath, 0600)
	if err != nil {
		return err
	}

	return nil
}

func pruneExpiredContainerBackupsTask(d *Daemon) (task.Func, task.Schedule) {
	f := func(ctx context.Context) {
		opRun := func(op *operation) error {
			return pruneExpiredContainerBackups(ctx, d)
		}

		op, err := operationCreate(d.cluster, "", operationClassTask, db.OperationBackupsExpire, nil, nil, opRun, nil, nil)
		if err != nil {
			logger.Error("Failed to start expired backups operation", log.Ctx{"err": err})
			return
		}

		logger.Info("Pruning expired container backups")
		_, err = op.Run()
		if err != nil {
			logger.Error("Failed to expire backups", log.Ctx{"err": err})
		}
		logger.Info("Done pruning expired container backups")
	}

	f(context.Background())

	first := true
	schedule := func() (time.Duration, error) {
		interval := time.Hour

		if first {
			first = false
			return interval, task.ErrSkip
		}

		return interval, nil
	}

	return f, schedule
}

func pruneExpiredContainerBackups(ctx context.Context, d *Daemon) error {
	// Get the list of expired backups.
	backups, err := d.cluster.ContainerBackupsGetExpired()
	if err != nil {
		return errors.Wrap(err, "Unable to retrieve the list of expired container backups")
	}

	for _, backup := range backups {
		containerName, _, _ := containerGetParentAndSnapshotName(backup)
		err := doBackupDelete(d.State(), backup, containerName)
		if err != nil {
			return errors.Wrapf(err, "Error deleting container backup %s", backup)
		}
	}

	return nil
}

func doBackupDelete(s *state.State, backupName, containerName string) error {
	backupPath := shared.VarPath("backups", backupName)

	// Delete the on-disk data
	if shared.PathExists(backupPath) {
		err := os.RemoveAll(backupPath)
		if err != nil {
			return err
		}
	}

	// Check if we can remove the container directory
	backupsPath := shared.VarPath("backups", containerName)
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Remove the database record
	err := s.Cluster.ContainerBackupRemove(backupName)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>lxd/backup: Don't waste memory during unpack<commit_after>package main

import (
	"archive/tar"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/pkg/errors"
)

// Load a backup from the database
func backupLoadByName(s *state.State, project, name string) (*backup, error) {
	// Get the backup database record
	args, err := s.Cluster.ContainerGetBackup(project, name)
	if err != nil {
		return nil, errors.Wrap(err, "Load backup from database")
	}

	// Load the container it belongs to
	c, err := containerLoadById(s, args.ContainerID)
	if err != nil {
		return nil, errors.Wrap(err, "Load container from database")
	}

	// Return the backup struct
	return &backup{
		state:            s,
		container:        c,
		id:               args.ID,
		name:             name,
		creationDate:     args.CreationDate,
		expiryDate:       args.ExpiryDate,
		containerOnly:    args.ContainerOnly,
		optimizedStorage: args.OptimizedStorage,
	}, nil
}

// Create a new backup
func backupCreate(s *state.State, args db.ContainerBackupArgs, sourceContainer container) error {
	// Create the database entry
	err := s.Cluster.ContainerBackupCreate(args)
	if err != nil {
		if err == db.ErrAlreadyDefined {
			return fmt.Errorf("backup '%s' already exists", args.Name)
		}

		return errors.Wrap(err, "Insert backup info into database")
	}

	// Get the backup struct
	b, err := backupLoadByName(s, sourceContainer.Project(), args.Name)
	if err != nil {
		return errors.Wrap(err, "Load backup object")
	}

	// Now create the empty snapshot
	err = sourceContainer.Storage().ContainerBackupCreate(*b, sourceContainer)
	if err != nil {
		s.Cluster.ContainerBackupRemove(args.Name)
		return errors.Wrap(err, "Backup storage")
	}

	return nil
}

// backup represents a container backup
type backup struct {
	state     *state.State
	container container

	// Properties
	id               int
	name             string
	creationDate     time.Time
	expiryDate       time.Time
	containerOnly    bool
	optimizedStorage bool
}

type backupInfo struct {
	Project         string   `json:"project" yaml:"project"`
	Name            string   `json:"name" yaml:"name"`
	Backend         string   `json:"backend" yaml:"backend"`
	Privileged      bool     `json:"privileged" yaml:"privileged"`
	Pool            string   `json:"pool" yaml:"pool"`
	Snapshots       []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
	HasBinaryFormat bool     `json:"-" yaml:"-"`
}

// Rename renames a container backup
func (b *backup) Rename(newName string) error {
	oldBackupPath := shared.VarPath("backups", b.name)
	newBackupPath := shared.VarPath("backups", newName)

	// Create the new backup path
	backupsPath := shared.VarPath("backups", b.container.Name())
	if !shared.PathExists(backupsPath) {
		err := os.MkdirAll(backupsPath, 0700)
		if err != nil {
			return err
		}
	}

	// Rename the backup directory
	err := os.Rename(oldBackupPath, newBackupPath)
	if err != nil {
		return err
	}

	// Check if we can remove the container directory
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Rename the database record
	err = b.state.Cluster.ContainerBackupRename(b.name, newName)
	if err != nil {
		return err
	}

	return nil
}

// Delete removes a container backup
func (b *backup) Delete() error {
	return doBackupDelete(b.state, b.name, b.container.Name())
}

func (b *backup) Render() *api.ContainerBackup {
	return &api.ContainerBackup{
		Name:             strings.SplitN(b.name, "/", 2)[1],
		CreationDate:     b.creationDate,
		ExpiryDate:       b.expiryDate,
		ContainerOnly:    b.containerOnly,
		OptimizedStorage: b.optimizedStorage,
	}
}

func backupGetInfo(r io.ReadSeeker) (*backupInfo, error) {
	var tr *tar.Reader
	result := backupInfo{}
	hasBinaryFormat := false
	hasIndexFile := false

	// Extract
	r.Seek(0, 0)
	_, _, unpacker, err := shared.DetectCompressionFile(r)
	if err != nil {
		return nil, err
	}
	r.Seek(0, 0)

	if unpacker == nil {
		return nil, fmt.Errorf("Unsupported backup compression")
	}

	if len(unpacker) > 0 {
		cmd := exec.Command(unpacker[0], unpacker[1:]...)
		cmd.Stdin = r

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			return nil, err
		}
		defer stdout.Close()

		err = cmd.Start()
		if err != nil {
			return nil, err
		}
		defer cmd.Wait()

		tr = tar.NewReader(stdout)
	} else {
		tr = tar.NewReader(r)
	}

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break // End of archive
		}
		if err != nil {
			return nil, err
		}

		if hdr.Name == "backup/index.yaml" {
			err = yaml.NewDecoder(tr).Decode(&result)
			if err != nil {
				return nil, err
			}

			hasIndexFile = true
		}

		if hdr.Name == "backup/container.bin" {
			hasBinaryFormat = true
		}
	}

	if !hasIndexFile {
		return nil, fmt.Errorf("Backup is missing index.yaml")
	}

	result.HasBinaryFormat = hasBinaryFormat
	return &result, nil
}

// fixBackupStoragePool changes the pool information in the backup.yaml. This
// is done only if the provided pool doesn't exist. In this case, the pool of
// the default profile will be used.
func backupFixStoragePool(c *db.Cluster, b backupInfo) error {
	// Get the default profile
	_, profile, err := c.ProfileGet("default", "default")
	if err != nil {
		return err
	}

	_, v, err := shared.GetRootDiskDevice(profile.Devices)
	if err != nil {
		return err
	}

	// Get the default's profile pool
	_, pool, err := c.StoragePoolGet(v["pool"])
	if err != nil {
		return err
	}

	f := func(path string) error {
		// Read in the backup.yaml file.
		backup, err := slurpBackupFile(path)
		if err != nil {
			return err
		}

		// Change the pool in the backup.yaml
		backup.Pool = pool
		backup.Container.Devices["root"]["pool"] = "default"

		file, err := os.Create(path)
		if err != nil {
			return err
		}
		defer file.Close()

		data, err := yaml.Marshal(&backup)
		if err != nil {
			return err
		}

		_, err = file.Write(data)
		if err != nil {
			return err
		}

		return nil
	}

	err = f(shared.VarPath("storage-pools", pool.Name, "containers", b.Name, "backup.yaml"))
	if err != nil {
		return err
	}

	for _, snap := range b.Snapshots {
		err = f(shared.VarPath("storage-pools", pool.Name, "containers-snapshots", b.Name, snap,
			"backup.yaml"))
		if err != nil {
			return err
		}
	}
	return nil
}

func backupCreateTarball(s *state.State, path string, backup backup) error {
	container := backup.container

	// Create the index
	pool, err := container.StoragePool()
	if err != nil {
		return err
	}

	indexFile := backupInfo{
		Name:       container.Name(),
		Backend:    container.Storage().GetStorageTypeName(),
		Privileged: container.IsPrivileged(),
		Pool:       pool,
		Snapshots:  []string{},
	}

	if !backup.containerOnly {
		snaps, err := container.Snapshots()
		if err != nil {
			return err
		}

		for _, snap := range snaps {
			_, snapName, _ := containerGetParentAndSnapshotName(snap.Name())
			indexFile.Snapshots = append(indexFile.Snapshots, snapName)
		}
	}

	data, err := yaml.Marshal(&indexFile)
	if err != nil {
		return err
	}

	file, err := os.Create(filepath.Join(path, "index.yaml"))
	if err != nil {
		return err
	}

	_, err = file.Write(data)
	file.Close()
	if err != nil {
		return err
	}

	// Create the target path if needed
	backupsPath := shared.VarPath("backups", backup.container.Name())
	if !shared.PathExists(backupsPath) {
		err := os.MkdirAll(backupsPath, 0700)
		if err != nil {
			return err
		}
	}

	// Create the tarball
	backupPath := shared.VarPath("backups", backup.name)
	args := []string{"-cf", backupPath, "--xattrs", "-C", path, "--transform", "s,^./,backup/,", "."}
	_, err = shared.RunCommand("tar", args...)
	if err != nil {
		return err
	}

	// Compress it
	compress, err := cluster.ConfigGetString(s.Cluster, "backups.compression_algorithm")
	if err != nil {
		return err
	}

	if compress != "none" {
		infile, err := os.Open(backupPath)
		if err != nil {
			return err
		}
		defer infile.Close()

		compressed, err := os.Create(backupPath + ".compressed")
		if err != nil {
			return err
		}
		defer compressed.Close()

		err = compressFile(compress, infile, compressed)
		if err != nil {
			return err
		}

		err = os.Remove(backupPath)
		if err != nil {
			return err
		}

		err = os.Rename(compressed.Name(), backupPath)
		if err != nil {
			return err
		}
	}

	// Set permissions
	err = os.Chmod(backupPath, 0600)
	if err != nil {
		return err
	}

	return nil
}

func pruneExpiredContainerBackupsTask(d *Daemon) (task.Func, task.Schedule) {
	f := func(ctx context.Context) {
		opRun := func(op *operation) error {
			return pruneExpiredContainerBackups(ctx, d)
		}

		op, err := operationCreate(d.cluster, "", operationClassTask, db.OperationBackupsExpire, nil, nil, opRun, nil, nil)
		if err != nil {
			logger.Error("Failed to start expired backups operation", log.Ctx{"err": err})
			return
		}

		logger.Info("Pruning expired container backups")
		_, err = op.Run()
		if err != nil {
			logger.Error("Failed to expire backups", log.Ctx{"err": err})
		}
		logger.Info("Done pruning expired container backups")
	}

	f(context.Background())

	first := true
	schedule := func() (time.Duration, error) {
		interval := time.Hour

		if first {
			first = false
			return interval, task.ErrSkip
		}

		return interval, nil
	}

	return f, schedule
}

func pruneExpiredContainerBackups(ctx context.Context, d *Daemon) error {
	// Get the list of expired backups.
	backups, err := d.cluster.ContainerBackupsGetExpired()
	if err != nil {
		return errors.Wrap(err, "Unable to retrieve the list of expired container backups")
	}

	for _, backup := range backups {
		containerName, _, _ := containerGetParentAndSnapshotName(backup)
		err := doBackupDelete(d.State(), backup, containerName)
		if err != nil {
			return errors.Wrapf(err, "Error deleting container backup %s", backup)
		}
	}

	return nil
}

func doBackupDelete(s *state.State, backupName, containerName string) error {
	backupPath := shared.VarPath("backups", backupName)

	// Delete the on-disk data
	if shared.PathExists(backupPath) {
		err := os.RemoveAll(backupPath)
		if err != nil {
			return err
		}
	}

	// Check if we can remove the container directory
	backupsPath := shared.VarPath("backups", containerName)
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Remove the database record
	err := s.Cluster.ContainerBackupRemove(backupName)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var (
	countDirs  int
	countFiles int
)

func update(f os.FileInfo) {
	if f.IsDir() {
		countDirs++
	} else {
		countFiles++
	}
}

// readDir reads the files from the dir "s" recursively and checks if there are duplicated
func readDir(s string, depth int) {
	depth++

	files, err := ioutil.ReadDir(s)
	if err != nil {
		fmt.Printf("[-] Error reading %s: %s\n", s, err)
		return
	}
	if len(files) == 0 {
		return
	}

	for _, f := range files {
		if f.Name() == ".godedupe_ignore" {
			return
		}
	}

	for _, f := range files {
		path := filepath.Join(s, f.Name())
		file := File{
			path,
			f,
		}

		update(file.info)
		if !opt.quiet {
			fmt.Printf("[+] Analyzed: %v directories and %v files\r", countDirs, countFiles)
		}
		if !file.info.IsDir() {
			// Only scan for files of a given extension
			if opt.fileExt != "" && !strings.HasSuffix(file.info.Name(), opt.fileExt) {
			} else if opt.excludeEmptyFiles && file.info.Size() == 0 {
			} else if opt.excludeHiddenFiles && strings.HasPrefix(file.info.Name(), ".") {
			} else if !opt.followSymlinks && file.info.Mode()&os.ModeSymlink != 0 {
			} else {
				AddFile(file)
			}
		} else if opt.enableRecursion {
			if depth < opt.maxDepth || opt.maxDepth == -1 {
				readDir(path, depth)
			}
		}
	}
}

// Start the program with the targetDirs options. Options param is read only
func start() {
	// Set the global variable so readDir function can access to the options
	if len(opt.targetDirs) == 0 {
		fmt.Println("error: directory must be specified. See help.")
		return
	}

	for _, dir := range opt.targetDirs {
		if info, err := os.Stat(dir); !opt.quiet && err == nil && !info.IsDir() {
			fmt.Printf("[-] %s is not a valid directory. Removing", dir)
			// TODO: Delete target dir
		}
	}
	for _, dir := range opt.targetDirs {
		if !opt.quiet {
			fmt.Println("[+] Reading directory:", dir)
		}
		readDir(dir, 0)
	}
	if !opt.quiet {
		fmt.Printf("\n[+] Stage 1 / 3 completed\n")
	}
	ValidateDuplicatedFiles()
	ObtainReportData().DoReport()
}
<commit_msg>If a directory is not valid return an error<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var (
	countDirs  int
	countFiles int
)

func update(f os.FileInfo) {
	if f.IsDir() {
		countDirs++
	} else {
		countFiles++
	}
}

// readDir reads the files from the dir "s" recursively and checks if there are duplicated
func readDir(s string, depth int) {
	depth++

	files, err := ioutil.ReadDir(s)
	if err != nil {
		fmt.Printf("[-] Error reading %s: %s\n", s, err)
		return
	}
	if len(files) == 0 {
		return
	}

	for _, f := range files {
		if f.Name() == ".godedupe_ignore" {
			return
		}
	}

	for _, f := range files {
		path := filepath.Join(s, f.Name())
		file := File{
			path,
			f,
		}

		update(file.info)
		if !opt.quiet {
			fmt.Printf("[+] Analyzed: %v directories and %v files\r", countDirs, countFiles)
		}
		if !file.info.IsDir() {
			// Only scan for files of a given extension
			if opt.fileExt != "" && !strings.HasSuffix(file.info.Name(), opt.fileExt) {
			} else if opt.excludeEmptyFiles && file.info.Size() == 0 {
			} else if opt.excludeHiddenFiles && strings.HasPrefix(file.info.Name(), ".") {
			} else if !opt.followSymlinks && file.info.Mode()&os.ModeSymlink != 0 {
			} else {
				AddFile(file)
			}
		} else if opt.enableRecursion {
			if depth < opt.maxDepth || opt.maxDepth == -1 {
				readDir(path, depth)
			}
		}
	}
}

// Start the program with the targetDirs options. Options param is read only
func start() {
	// Set the global variable so readDir function can access to the options
	if len(opt.targetDirs) == 0 {
		fmt.Println("error: directory must be specified. See help.")
		return
	}

	for _, dir := range opt.targetDirs {
		if info, err := os.Stat(dir); err == nil && !info.IsDir() && !opt.quiet {
         // This should return an error to avoid hiding potential configuration errors
			fmt.Printf("[-] %s is not a valid directory", info.Name())
			return
		}
	}
	for _, dir := range opt.targetDirs {
		if !opt.quiet {
			fmt.Println("[+] Reading directory:", dir)
		}
		readDir(dir, 0)
	}
	if !opt.quiet {
		fmt.Printf("\n[+] Stage 1 / 3 completed\n")
	}
	ValidateDuplicatedFiles()
	ObtainReportData().DoReport()
}
<|endoftext|>