<commit_before>package fuse

import (
	"fmt"
	"log"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"syscall"
)

var _ = fmt.Println
var _ = log.Println

const (
	DebugDir = ".debug"
)

type getter func() []byte

// FileSystemDebug exposes a .debug directory, exposing files for
// which a read hooks into a callback.  This is useful for exporting
// metrics and debug information from the daemon.
type FileSystemDebug struct {
	sync.RWMutex
	callbacks map[string]getter
	WrappingFileSystem
}

func NewFileSystemDebug() *FileSystemDebug {
	return &FileSystemDebug{
		callbacks: make(map[string]getter),
	}
}

func (me *FileSystemDebug) Add(name string, callback getter) {
	me.RWMutex.Lock()
	defer me.RWMutex.Unlock()
	me.callbacks[name] = callback
}

func (me *FileSystemDebug) Open(path string, flags uint32) (fuseFile File, status Status) {

	content := me.getContent(path)
	if content != nil {
		return NewReadOnlyFile(content), OK
	}
	return me.Original.Open(path, flags)
}

func (me *FileSystemDebug) getContent(path string) []byte {
	comps := strings.Split(path, filepath.SeparatorString, -1)
	if comps[0] == DebugDir {
		me.RWMutex.RLock()
		defer me.RWMutex.RUnlock()
		f := me.callbacks[comps[1]]
		if f != nil {
			return f()
		}
	}
	return nil
}

func (me *FileSystemDebug) GetXAttr(name string, attr string) ([]byte, Status) {
	if strings.HasPrefix(name, DebugDir) {
		return nil, syscall.ENODATA
	}
	return me.Original.GetXAttr(name, attr)
}

func (me *FileSystemDebug) GetAttr(path string) (*Attr, Status) {
	if !strings.HasPrefix(path, DebugDir) {
		return me.Original.GetAttr(path)
	}
	if path == DebugDir {
		return &Attr{
			Mode: S_IFDIR | 0755,
		},OK
	}
	c := me.getContent(path)
	if c != nil {
		return &Attr{
			Mode: S_IFREG | 0644,
			Size: uint64(len(c)),
		},OK
	}
	return nil, ENOENT
}

func FloatMapToBytes(m map[string]float64) []byte {
	keys := make([]string, 0, len(m))
	for k, _ := range m {
		keys = append(keys, k)
	}

	sort.SortStrings(keys)

	var r []string
	for _, k := range keys {
		r = append(r, fmt.Sprintf("%v %v", k, m[k]))
	}
	return []byte(strings.Join(r, "\n"))
}

// Ugh - generics.
func IntMapToBytes(m map[string]int) []byte {
	keys := make([]string, 0, len(m))
	for k, _ := range m {
		keys = append(keys, k)
	}

	sort.SortStrings(keys)

	var r []string
	for _, k := range keys {
		r = append(r, fmt.Sprintf("%v %v", k, m[k]))
	}
	return []byte(strings.Join(r, "\n"))
}

func (me *FileSystemDebug) OpenDir(name string) (stream chan DirEntry, status Status) {
	if name == DebugDir {
		me.RWMutex.RLock()
		defer me.RWMutex.RUnlock()

		stream = make(chan DirEntry, len(me.callbacks))
		for k, _ := range me.callbacks {
			stream <- DirEntry{
				Name: k,
				Mode: S_IFREG,
			}
		}
		close(stream)
		return stream, OK
	}
	return me.Original.OpenDir(name)
}

func (me *FileSystemDebug) AddMountState(state *MountState) {
	me.Add("mountstate-latencies",
		func() []byte { return FloatMapToBytes(state.Latencies()) })
	me.Add("mountstate-opcounts",
		func() []byte { return IntMapToBytes(state.OperationCounts()) })
	me.Add("mountstate-bufferpool",
		func() []byte { return []byte(state.BufferPoolStats()) })
}

func (me *FileSystemDebug) AddFileSystemConnector(conn *FileSystemConnector) {
	me.Add("filesystemconnector-stats",
		func() []byte { return []byte(conn.Statistics()) })
}

func hotPaths(timing *TimingFileSystem) []byte {
	hot := timing.HotPaths("GetAttr")
	unique := len(hot)
	top := 20
	start := len(hot) - top
	if start < 0 {
		start = 0
	}
	return []byte(fmt.Sprintf("Unique GetAttr paths: %d\nTop %d GetAttr paths: %v",
		unique, top, hot[start:]))
}

func (me *FileSystemDebug) AddTimingFileSystem(tfs *TimingFileSystem) {
	me.Add("timingfs-latencies",
		func() []byte { return FloatMapToBytes(tfs.Latencies()) })
	me.Add("timingfs-opcounts",
		func() []byte { return IntMapToBytes(tfs.OperationCounts()) })
	me.Add("timingfs-hotpaths",
		func() []byte { return hotPaths(tfs) })
}

func (me *FileSystemDebug) AddRawTimingFileSystem(tfs *TimingRawFileSystem) {
	me.Add("rawtimingfs-latencies",
		func() []byte { return FloatMapToBytes(tfs.Latencies()) })
}
<commit_msg>Add TODO.<commit_after>package fuse

import (
	"fmt"
	"log"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"syscall"
)

var _ = fmt.Println
var _ = log.Println

const (
	DebugDir = ".debug"
)

type getter func() []byte

// FileSystemDebug exposes a .debug directory, exposing files for
// which a read hooks into a callback.  This is useful for exporting
// metrics and debug information from the daemon.
//
// TODO - should use in-process mount instead?
type FileSystemDebug struct {
	sync.RWMutex
	callbacks map[string]getter
	WrappingFileSystem
}

func NewFileSystemDebug() *FileSystemDebug {
	return &FileSystemDebug{
		callbacks: make(map[string]getter),
	}
}

func (me *FileSystemDebug) Add(name string, callback getter) {
	me.RWMutex.Lock()
	defer me.RWMutex.Unlock()
	me.callbacks[name] = callback
}

func (me *FileSystemDebug) Open(path string, flags uint32) (fuseFile File, status Status) {

	content := me.getContent(path)
	if content != nil {
		return NewReadOnlyFile(content), OK
	}
	return me.Original.Open(path, flags)
}

func (me *FileSystemDebug) getContent(path string) []byte {
	comps := strings.Split(path, filepath.SeparatorString, -1)
	if comps[0] == DebugDir {
		me.RWMutex.RLock()
		defer me.RWMutex.RUnlock()
		f := me.callbacks[comps[1]]
		if f != nil {
			return f()
		}
	}
	return nil
}

func (me *FileSystemDebug) GetXAttr(name string, attr string) ([]byte, Status) {
	if strings.HasPrefix(name, DebugDir) {
		return nil, syscall.ENODATA
	}
	return me.Original.GetXAttr(name, attr)
}

func (me *FileSystemDebug) GetAttr(path string) (*Attr, Status) {
	if !strings.HasPrefix(path, DebugDir) {
		return me.Original.GetAttr(path)
	}
	if path == DebugDir {
		return &Attr{
			Mode: S_IFDIR | 0755,
		},OK
	}
	c := me.getContent(path)
	if c != nil {
		return &Attr{
			Mode: S_IFREG | 0644,
			Size: uint64(len(c)),
		},OK
	}
	return nil, ENOENT
}

func FloatMapToBytes(m map[string]float64) []byte {
	keys := make([]string, 0, len(m))
	for k, _ := range m {
		keys = append(keys, k)
	}

	sort.SortStrings(keys)

	var r []string
	for _, k := range keys {
		r = append(r, fmt.Sprintf("%v %v", k, m[k]))
	}
	return []byte(strings.Join(r, "\n"))
}

// Ugh - generics.
func IntMapToBytes(m map[string]int) []byte {
	keys := make([]string, 0, len(m))
	for k, _ := range m {
		keys = append(keys, k)
	}

	sort.SortStrings(keys)

	var r []string
	for _, k := range keys {
		r = append(r, fmt.Sprintf("%v %v", k, m[k]))
	}
	return []byte(strings.Join(r, "\n"))
}

func (me *FileSystemDebug) OpenDir(name string) (stream chan DirEntry, status Status) {
	if name == DebugDir {
		me.RWMutex.RLock()
		defer me.RWMutex.RUnlock()

		stream = make(chan DirEntry, len(me.callbacks))
		for k, _ := range me.callbacks {
			stream <- DirEntry{
				Name: k,
				Mode: S_IFREG,
			}
		}
		close(stream)
		return stream, OK
	}
	return me.Original.OpenDir(name)
}

func (me *FileSystemDebug) AddMountState(state *MountState) {
	me.Add("mountstate-latencies",
		func() []byte { return FloatMapToBytes(state.Latencies()) })
	me.Add("mountstate-opcounts",
		func() []byte { return IntMapToBytes(state.OperationCounts()) })
	me.Add("mountstate-bufferpool",
		func() []byte { return []byte(state.BufferPoolStats()) })
}

func (me *FileSystemDebug) AddFileSystemConnector(conn *FileSystemConnector) {
	me.Add("filesystemconnector-stats",
		func() []byte { return []byte(conn.Statistics()) })
}

func hotPaths(timing *TimingFileSystem) []byte {
	hot := timing.HotPaths("GetAttr")
	unique := len(hot)
	top := 20
	start := len(hot) - top
	if start < 0 {
		start = 0
	}
	return []byte(fmt.Sprintf("Unique GetAttr paths: %d\nTop %d GetAttr paths: %v",
		unique, top, hot[start:]))
}

func (me *FileSystemDebug) AddTimingFileSystem(tfs *TimingFileSystem) {
	me.Add("timingfs-latencies",
		func() []byte { return FloatMapToBytes(tfs.Latencies()) })
	me.Add("timingfs-opcounts",
		func() []byte { return IntMapToBytes(tfs.OperationCounts()) })
	me.Add("timingfs-hotpaths",
		func() []byte { return hotPaths(tfs) })
}

func (me *FileSystemDebug) AddRawTimingFileSystem(tfs *TimingRawFileSystem) {
	me.Add("rawtimingfs-latencies",
		func() []byte { return FloatMapToBytes(tfs.Latencies()) })
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"mime"
	"net/http"
	"pkg/storage"
	"strconv"
	"strings"
	"time"
)

var (
	port        = flag.Int("port", 8080, "http listen port")
	chunkFolder = flag.String("dir", "/tmp", "data directory to store files")
	volumes     = flag.String("volumes", "0,1-3,4", "comma-separated list of volume ids or range of ids")
	publicUrl   = flag.String("publicUrl", "localhost:8080", "public url to serve data read")
	metaServer  = flag.String("mserver", "localhost:9333", "master directory server to store mappings")
	IsDebug     = flag.Bool("debug", false, "enable debug mode")
	pulse       = flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")

	store *storage.Store
)

func statusHandler(w http.ResponseWriter, r *http.Request) {
	writeJson(w, r, store.Status())
}
func addVolumeHandler(w http.ResponseWriter, r *http.Request) {
	store.AddVolume(r.FormValue("volume"))
	writeJson(w, r, store.Status())
}
func storeHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		GetHandler(w, r)
	case "DELETE":
		DeleteHandler(w, r)
	case "POST":
		PostHandler(w, r)
	}
}
func GetHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, ext := parseURLPath(r.URL.Path)
	volumeId, _ := strconv.ParseUint(vid, 10, 64)
	n.ParsePath(fid)

	if *IsDebug {
		log.Println("volume", volumeId, "reading", n)
	}
	cookie := n.Cookie
	count, e := store.Read(volumeId, n)
	if *IsDebug {
		log.Println("read bytes", count, "error", e)
	}
	if n.Cookie != cookie {
		log.Println("request with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		return
	}
	if ext != "" {
		w.Header().Set("Content-Type", mime.TypeByExtension(ext))
	}
	w.Write(n.Data)
}
func PostHandler(w http.ResponseWriter, r *http.Request) {
	vid, _, _ := parseURLPath(r.URL.Path)
	volumeId, e := strconv.ParseUint(vid, 10, 64)
	if e != nil {
		writeJson(w, r, e)
	} else {
		needle, ne := storage.NewNeedle(r)
		if ne != nil {
			writeJson(w, r, ne)
		} else {
			ret := store.Write(volumeId, needle)
			m := make(map[string]uint32)
			m["size"] = ret
			writeJson(w, r, m)
		}
	}
}
func DeleteHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, _ := parseURLPath(r.URL.Path)
	volumeId, _ := strconv.ParseUint(vid, 10, 64)
	n.ParsePath(fid)

	if *IsDebug {
		log.Println("deleting", n)
	}

	cookie := n.Cookie
	count, ok := store.Read(volumeId, n)

	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJson(w, r, m)
		return
	}

	if n.Cookie != cookie {
		log.Println("delete with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		return
	}

	n.Size = 0
	store.Delete(volumeId, n)
	m := make(map[string]uint32)
	m["size"] = uint32(count)
	writeJson(w, r, m)
}
func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) {
	w.Header().Set("Content-Type", "application/javascript")
	bytes, _ := json.Marshal(obj)
	callback := r.FormValue("callback")
	if callback == "" {
		w.Write(bytes)
	} else {
		w.Write([]uint8(callback))
		w.Write([]uint8("("))
		fmt.Fprint(w, string(bytes))
		w.Write([]uint8(")"))
	}
	//log.Println("JSON Response", string(bytes))
}
func parseURLPath(path string) (vid, fid, ext string) {
	sepIndex := strings.LastIndex(path, "/")
	commaIndex := strings.LastIndex(path[sepIndex:], ",")
	if commaIndex <= 0 {
		if "favicon.ico" != path[sepIndex+1:] {
			log.Println("unknown file id", path[sepIndex+1:])
		}
		return
	}
	dotIndex := strings.LastIndex(path[sepIndex:], ".")
	vid = path[sepIndex+1 : commaIndex]
	fid = path[commaIndex+1:]
	ext = ""
	if dotIndex > 0 {
		fid = path[commaIndex+1 : dotIndex]
		ext = path[dotIndex+1:]
	}
	return
}

func main() {
	flag.Parse()
	//TODO: now default to 1G, this value should come from server?
	store = storage.NewStore(*port, *publicUrl, *chunkFolder, *volumes)
	defer store.Close()
	http.HandleFunc("/", storeHandler)
	http.HandleFunc("/status", statusHandler)
	http.HandleFunc("/add_volume", addVolumeHandler)

	go func() {
		for {
			store.Join(*metaServer)
			time.Sleep(time.Duration(float32(*pulse*1e3)*(1+rand.Float32())) * time.Millisecond)
		}
	}()
	log.Println("store joined at", *metaServer)

	log.Println("Start storage service at http://127.0.0.1:"+strconv.Itoa(*port), "public url", *publicUrl)
	e := http.ListenAndServe(":"+strconv.Itoa(*port), nil)
	if e != nil {
		log.Fatalf("Fail to start:", e)
	}

}
<commit_msg>The extension ext should begin with a leading dot, as in ".html"<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"mime"
	"net/http"
	"pkg/storage"
	"strconv"
	"strings"
	"time"
)

var (
	port        = flag.Int("port", 8080, "http listen port")
	chunkFolder = flag.String("dir", "/tmp", "data directory to store files")
	volumes     = flag.String("volumes", "0,1-3,4", "comma-separated list of volume ids or range of ids")
	publicUrl   = flag.String("publicUrl", "localhost:8080", "public url to serve data read")
	metaServer  = flag.String("mserver", "localhost:9333", "master directory server to store mappings")
	IsDebug     = flag.Bool("debug", false, "enable debug mode")
	pulse       = flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")

	store *storage.Store
)

func statusHandler(w http.ResponseWriter, r *http.Request) {
	writeJson(w, r, store.Status())
}
func addVolumeHandler(w http.ResponseWriter, r *http.Request) {
	store.AddVolume(r.FormValue("volume"))
	writeJson(w, r, store.Status())
}
func storeHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		GetHandler(w, r)
	case "DELETE":
		DeleteHandler(w, r)
	case "POST":
		PostHandler(w, r)
	}
}
func GetHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, ext := parseURLPath(r.URL.Path)
	volumeId, _ := strconv.ParseUint(vid, 10, 64)
	n.ParsePath(fid)

	if *IsDebug {
		log.Println("volume", volumeId, "reading", n)
	}
	cookie := n.Cookie
	count, e := store.Read(volumeId, n)
	if *IsDebug {
		log.Println("read bytes", count, "error", e)
	}
	if n.Cookie != cookie {
		log.Println("request with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		return
	}
	if ext != "" {
		w.Header().Set("Content-Type", mime.TypeByExtension(ext))
	}
	w.Write(n.Data)
}
func PostHandler(w http.ResponseWriter, r *http.Request) {
	vid, _, _ := parseURLPath(r.URL.Path)
	volumeId, e := strconv.ParseUint(vid, 10, 64)
	if e != nil {
		writeJson(w, r, e)
	} else {
		needle, ne := storage.NewNeedle(r)
		if ne != nil {
			writeJson(w, r, ne)
		} else {
			ret := store.Write(volumeId, needle)
			m := make(map[string]uint32)
			m["size"] = ret
			writeJson(w, r, m)
		}
	}
}
func DeleteHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, _ := parseURLPath(r.URL.Path)
	volumeId, _ := strconv.ParseUint(vid, 10, 64)
	n.ParsePath(fid)

	if *IsDebug {
		log.Println("deleting", n)
	}

	cookie := n.Cookie
	count, ok := store.Read(volumeId, n)

	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJson(w, r, m)
		return
	}

	if n.Cookie != cookie {
		log.Println("delete with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		return
	}

	n.Size = 0
	store.Delete(volumeId, n)
	m := make(map[string]uint32)
	m["size"] = uint32(count)
	writeJson(w, r, m)
}
func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) {
	w.Header().Set("Content-Type", "application/javascript")
	bytes, _ := json.Marshal(obj)
	callback := r.FormValue("callback")
	if callback == "" {
		w.Write(bytes)
	} else {
		w.Write([]uint8(callback))
		w.Write([]uint8("("))
		fmt.Fprint(w, string(bytes))
		w.Write([]uint8(")"))
	}
	//log.Println("JSON Response", string(bytes))
}
func parseURLPath(path string) (vid, fid, ext string) {
	sepIndex := strings.LastIndex(path, "/")
	commaIndex := strings.LastIndex(path[sepIndex:], ",")
	if commaIndex <= 0 {
		if "favicon.ico" != path[sepIndex+1:] {
			log.Println("unknown file id", path[sepIndex+1:])
		}
		return
	}
	dotIndex := strings.LastIndex(path[sepIndex:], ".")
	vid = path[sepIndex+1 : commaIndex]
	fid = path[commaIndex+1:]
	ext = ""
	if dotIndex > 0 {
		fid = path[commaIndex+1 : dotIndex]
		ext = path[dotIndex:]
	}
	return
}

func main() {
	flag.Parse()
	//TODO: now default to 1G, this value should come from server?
	store = storage.NewStore(*port, *publicUrl, *chunkFolder, *volumes)
	defer store.Close()
	http.HandleFunc("/", storeHandler)
	http.HandleFunc("/status", statusHandler)
	http.HandleFunc("/add_volume", addVolumeHandler)

	go func() {
		for {
			store.Join(*metaServer)
			time.Sleep(time.Duration(float32(*pulse*1e3)*(1+rand.Float32())) * time.Millisecond)
		}
	}()
	log.Println("store joined at", *metaServer)

	log.Println("Start storage service at http://127.0.0.1:"+strconv.Itoa(*port), "public url", *publicUrl)
	e := http.ListenAndServe(":"+strconv.Itoa(*port), nil)
	if e != nil {
		log.Fatalf("Fail to start:", e)
	}

}
<|endoftext|>


<commit_before>package main

import "bytes"
import "sync"
import "testing"

type TestPipe struct {
	*bytes.Buffer
}

func (t *TestPipe) Close() (err error) {
	return
}

func TestRun(t *testing.T) {
	stdinbuf := &bytes.Buffer{}
	template := []string{"echo", "{{}}"}
	runner := NewRunner(template, "{{}}", "foobar", stdinbuf)

	ch, err := runner.Run()
	if err != nil {
		t.Errorf("runner.Run() returned an error: %s", err)
	}

	output := make([]string, 0)
	for outputline := range ch {
		output = append(output, outputline)
	}

	if len(output) != 1 {
		t.Errorf("output length wrong. expected: %d, actual: %d", 1, len(output))
	}

	if output[0] != "foobar\n" {
		t.Errorf("output wrong. expected: %q, actual: %q", "foobar", output[0])
	}
}

func TestStreamOutput(t *testing.T) {
	pipe := &TestPipe{bytes.NewBufferString("foo\nbar\n")}
	wg := &sync.WaitGroup{}
	runner := &Runner{}

	output := make([]string, 0)
	wg.Add(1)
	ch := runner.streamOutput(pipe, wg)
	for line := range ch {
		output = append(output, line)
	}
	wg.Wait()

	if len(output) != 2 {
		t.Errorf("streamed output length wrong. expected: %d, actual: %d", 2, len(output))
	}

	if output[0] != "foo\n" {
		t.Errorf("streamed output wrong. expected: %q, actual: %q", "foo\n", output[0])
	}

	if output[1] != "bar\n" {
		t.Errorf("streamed output wrong. expected: %q, actual: %q", "bar\n", output[0])
	}
}

func TestStreamOutputWithoutTrailingNewline(t *testing.T) {
	pipe := &TestPipe{bytes.NewBufferString("foo\nbar")}
	wg := &sync.WaitGroup{}
	runner := &Runner{}

	output := make([]string, 0)
	wg.Add(1)
	ch := runner.streamOutput(pipe, wg)

	for line := range ch {
		output = append(output, line)
	}
	wg.Wait()

	if len(output) != 2 {
		t.Errorf("streamed output length wrong. expected: %d, actual: %d", 2, len(output))
	}

	if output[0] != "foo\n" {
		t.Errorf("streamed output wrong. expected: %q, actual: %q", "foo\n", output[0])
	}

	if output[1] != "bar" {
		t.Errorf("streamed output wrong. expected: %q, actual: %q", "bar", output[0])
	}
}
<commit_msg>Add runner.Wait() call to tests<commit_after>package main

import "bytes"
import "sync"
import "testing"

type TestPipe struct {
	*bytes.Buffer
}

func (t *TestPipe) Close() (err error) {
	return
}

func TestRun(t *testing.T) {
	stdinbuf := &bytes.Buffer{}
	template := []string{"echo", "{{}}"}
	runner := NewRunner(template, "{{}}", "foobar", stdinbuf)

	ch, err := runner.Run()
	if err != nil {
		t.Errorf("runner.Run() returned an error: %s", err)
	}

	output := make([]string, 0)
	for outputline := range ch {
		output = append(output, outputline)
	}
	runner.Wait()

	if len(output) != 1 {
		t.Errorf("output length wrong. expected: %d, actual: %d", 1, len(output))
	}

	if output[0] != "foobar\n" {
		t.Errorf("output wrong. expected: %q, actual: %q", "foobar", output[0])
	}
}

func TestStreamOutput(t *testing.T) {
	pipe := &TestPipe{bytes.NewBufferString("foo\nbar\n")}
	wg := &sync.WaitGroup{}
	runner := &Runner{}

	output := make([]string, 0)
	wg.Add(1)
	ch := runner.streamOutput(pipe, wg)
	for line := range ch {
		output = append(output, line)
	}
	wg.Wait()

	if len(output) != 2 {
		t.Errorf("streamed output length wrong. expected: %d, actual: %d", 2, len(output))
	}

	if output[0] != "foo\n" {
		t.Errorf("streamed output wrong. expected: %q, actual: %q", "foo\n", output[0])
	}

	if output[1] != "bar\n" {
		t.Errorf("streamed output wrong. expected: %q, actual: %q", "bar\n", output[0])
	}
}

func TestStreamOutputWithoutTrailingNewline(t *testing.T) {
	pipe := &TestPipe{bytes.NewBufferString("foo\nbar")}
	wg := &sync.WaitGroup{}
	runner := &Runner{}

	output := make([]string, 0)
	wg.Add(1)
	ch := runner.streamOutput(pipe, wg)

	for line := range ch {
		output = append(output, line)
	}
	wg.Wait()

	if len(output) != 2 {
		t.Errorf("streamed output length wrong. expected: %d, actual: %d", 2, len(output))
	}

	if output[0] != "foo\n" {
		t.Errorf("streamed output wrong. expected: %q, actual: %q", "foo\n", output[0])
	}

	if output[1] != "bar" {
		t.Errorf("streamed output wrong. expected: %q, actual: %q", "bar", output[0])
	}
}
<|endoftext|>


<commit_before>package scrolls

import (
	"fmt"
	"io"
	"os"
	"strings"
)

type Logger struct {
	Stream  io.Writer
	context map[string]interface{}
}

func NewLogger(stream io.Writer) *Logger {
	return NewLoggerWithContext(stream, make(map[string]interface{}))
}

func NewLoggerWithContext(stream io.Writer, context map[string]interface{}) *Logger {
	if stream == nil {
		stream = os.Stdout
	}
	return &Logger{stream, context}
}

func (l *Logger) Log(data map[string]interface{}) {
	l.Stream.Write([]byte(l.buildLine(data)))
}

func (l *Logger) NewContext(data map[string]interface{}) *Logger {
	ctx := make(map[string]interface{})
	mergeMaps(l.context, ctx)
	mergeMaps(data, ctx)
	return NewLoggerWithContext(l.Stream, ctx)
}

func (l *Logger) AddContext(key string, value interface{}) {
	l.context[key] = value
}

func (l *Logger) DeleteContext(key string) {
	delete(l.context, key)
}

func (l *Logger) buildLine(data map[string]interface{}) string {
	merged := make(map[string]interface{})
	mergeMaps(l.context, merged)
	mergeMaps(data, merged)

	pieces := make([]string, len(merged))
	l.convertDataMap(merged, pieces)
	return strings.Join(pieces, space)
}

func (l *Logger) convertDataMap(data map[string]interface{}, pieces []string) {
	index := 0
	for key, value := range data {
		pieces[index] = fmt.Sprintf("%s=%s", key, value)
		index = index + 1
	}
}

func mergeMaps(original map[string]interface{}, copy map[string]interface{}) {
	if original == nil {
		return
	}

	for key, value := range original {
		copy[key] = value
	}
}

const (
	space = " "
	empty = ""
)
<commit_msg>better way of duping maps<commit_after>package scrolls

import (
	"fmt"
	"io"
	"os"
	"strings"
)

type Logger struct {
	Stream  io.Writer
	context map[string]interface{}
}

func NewLogger(stream io.Writer) *Logger {
	return NewLoggerWithContext(stream, make(map[string]interface{}))
}

func NewLoggerWithContext(stream io.Writer, context map[string]interface{}) *Logger {
	if stream == nil {
		stream = os.Stdout
	}
	return &Logger{stream, context}
}

func (l *Logger) Log(data map[string]interface{}) {
	l.Stream.Write([]byte(l.buildLine(data)))
}

func (l *Logger) NewContext(data map[string]interface{}) *Logger {
	return NewLoggerWithContext(l.Stream, dupeMaps(l.context, data))
}

func (l *Logger) AddContext(key string, value interface{}) {
	l.context[key] = value
}

func (l *Logger) DeleteContext(key string) {
	delete(l.context, key)
}

func (l *Logger) buildLine(data map[string]interface{}) string {
	merged := dupeMaps(l.context, data)
	pieces := make([]string, len(merged))
	l.convertDataMap(merged, pieces)
	return strings.Join(pieces, space)
}

func (l *Logger) convertDataMap(data map[string]interface{}, pieces []string) {
	index := 0
	for key, value := range data {
		pieces[index] = fmt.Sprintf("%s=%s", key, value)
		index = index + 1
	}
}

func dupeMaps(maps ...map[string]interface{}) map[string]interface{} {
	merged := make(map[string]interface{})
	for _, orig := range maps {
		for key, value := range orig {
			merged[key] = value
		}
	}
	return merged
}

const (
	space = " "
	empty = ""
)
<|endoftext|>


<commit_before>package logging

import (
	"log"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
)

var (
	standardFields logrus.Fields
	logger         *logrus.Logger
)

func init() {
	logger = logrus.New()
	logger.Formatter = customFormatter{&logrus.JSONFormatter{
		TimestampFormat: time.RFC3339Nano,
	}}
}

// SetStandardFields sets up the service name, version, hostname and pid fields
func SetStandardFields(service, version string) {
	hostname, _ := os.Hostname()
	standardFields = logrus.Fields{
		"service":  service,
		"version":  version,
		"hostname": hostname,
		"pid":      os.Getpid(),
	}
}

// UsePrettyPrint tells the logger to print in human readable format
func UsePrettyPrint() {
	logger.Formatter = customFormatter{&logrus.TextFormatter{
		FullTimestamp:    true,
		TimestampFormat:  time.RFC3339Nano,
		QuoteEmptyFields: true,
	}}
}

// ErrorLogger creates a logger than can plug in to an HTTP server
func ErrorLogger() (basicLogger *log.Logger, dispose func()) {
	w := logger.WriterLevel(logrus.ErrorLevel)
	basicLogger = log.New(w, "", 0)
	dispose = func() {
		w.Close()
	}

	return
}

// Info logs an info level message with standard fields
func Info(msg string) {
	logger.Info(msg)
}

// InfoWithFields logs an info level message with standard and additional fields
func InfoWithFields(msg string, fields logrus.Fields) {
	logger.WithFields(fields).Info(msg)
}

// Warn logs a warn level message with standard fields
func Warn(msg string) {
	logger.Warn(msg)
}

// WarnWithFields logs a warn level message with standard and additional fields
func WarnWithFields(msg string, fields logrus.Fields) {
	logger.WithFields(fields).Warn(msg)
}

// Error logs an error level message with standard fields
func Error(msg string) {
	logger.Error(msg)
}

// ErrorWithFields logs an error level message with standard and additional fields
func ErrorWithFields(msg string, fields logrus.Fields) {
	logger.WithFields(fields).Error(msg)
}

// Fatal logs a fatal level message with standard fields
func Fatal(msg string) {
	logger.Fatal(msg)
}

// FatalWithFields logs an fatal level message with standard and additional fields
func FatalWithFields(msg string, fields logrus.Fields) {
	logger.WithFields(fields).Fatal(msg)
}
<commit_msg>Removed *WithFields functions and replaced with an id parameter in the logging functions<commit_after>package logging

import (
	"log"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
)

var (
	standardFields logrus.Fields
	logger         *logrus.Logger
)

func init() {
	logger = logrus.New()
	logger.Formatter = customFormatter{&logrus.JSONFormatter{
		TimestampFormat: time.RFC3339Nano,
	}}
}

// SetStandardFields sets up the service name, version, hostname and pid fields
func SetStandardFields(service, version string) {
	hostname, _ := os.Hostname()
	standardFields = logrus.Fields{
		"service":  service,
		"version":  version,
		"hostname": hostname,
		"pid":      os.Getpid(),
	}
}

// UsePrettyPrint tells the logger to print in human readable format
func UsePrettyPrint() {
	logger.Formatter = customFormatter{&logrus.TextFormatter{
		FullTimestamp:    true,
		TimestampFormat:  time.RFC3339Nano,
		QuoteEmptyFields: true,
	}}
}

// ErrorLogger creates a logger than can plug in to an HTTP server
func ErrorLogger() (basicLogger *log.Logger, dispose func()) {
	w := logger.WriterLevel(logrus.ErrorLevel)
	basicLogger = log.New(w, "", 0)
	dispose = func() {
		w.Close()
	}

	return
}

// Info logs an info level message with standard fields
func Info(msg string, id string) {
	if id != "" {
		logger.WithField("request_id", id).Info(msg)
		return
	}
	logger.Info(msg)
}

// Warn logs a warn level message with standard fields
func Warn(msg string, id string) {
	if id != "" {
		logger.WithField("request_id", id).Warn(msg)
		return
	}
	logger.Warn(msg)
}

// Error logs an error level message with standard fields
func Error(msg string, id string) {
	if id != "" {
		logger.WithField("request_id", id).Error(msg)
		return
	}
	logger.Error(msg)
}

// Fatal logs a fatal level message with standard fields
func Fatal(msg string, id string) {
	if id != "" {
		logger.WithField("request_id", id).Fatal(msg)
		return
	}
	logger.Fatal(msg)
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"log"
	"os"
	"time"
)

func ErrorLogger() HandlerFunc {
	return ErrorLoggerT(ErrorTypeAll)
}

func ErrorLoggerT(typ uint32) HandlerFunc {
	return func(c *Context) {
		c.Next()

		errs := c.Errors.ByType(typ)
		if len(errs) > 0 {
			// -1 status code = do not change current one
			c.JSON(-1, c.Errors)
		}
	}
}

var (
	green  = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white  = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red    = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	reset  = string([]byte{27, 91, 48, 109})
)

func Logger() HandlerFunc {
	stdlogger := log.New(os.Stdout, "", 0)
	//errlogger := log.New(os.Stderr, "", 0)

	return func(c *Context) {
		// Start timer
		start := time.Now()

		// Process request
		c.Next()

		// save the IP of the requester
		requester := c.Request.Header.Get("X-Real-IP")
		// if the requester-header is empty, check the forwarded-header
		if len(requester) == 0 {
			requester = c.Request.Header.Get("X-Forwarded-For")
		}
		// if the requester is still empty, use the hard-coded address from the socket
		if len(requester) == 0 {
			requester = c.Request.RemoteAddr
		}

		var color string
		code := c.Writer.Status()
		switch {
		case code >= 200 && code <= 299:
			color = green
		case code >= 300 && code <= 399:
			color = white
		case code >= 400 && code <= 499:
			color = yellow
		default:
			color = red
		}
		end := time.Now()
		latency := end.Sub(start)
		stdlogger.Printf("[GIN] %v |%s %3d %s| %12v | %s %4s %s\n%s",
			end.Format("2006/01/02 - 15:04:05"),
			color, code, reset,
			latency,
			requester,
			c.Request.Method, c.Request.URL.Path,
			c.Errors.String(),
		)
	}
}
<commit_msg>(feature)add http method log-color,like http response status code<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"log"
	"os"
	"time"
)

func ErrorLogger() HandlerFunc {
	return ErrorLoggerT(ErrorTypeAll)
}

func ErrorLoggerT(typ uint32) HandlerFunc {
	return func(c *Context) {
		c.Next()

		errs := c.Errors.ByType(typ)
		if len(errs) > 0 {
			// -1 status code = do not change current one
			c.JSON(-1, c.Errors)
		}
	}
}

var (
	green   = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white   = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow  = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red     = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue    = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan    = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset   = string([]byte{27, 91, 48, 109})
)

func Logger() HandlerFunc {
	stdlogger := log.New(os.Stdout, "", 0)
	//errlogger := log.New(os.Stderr, "", 0)

	return func(c *Context) {
		// Start timer
		start := time.Now()

		// Process request
		c.Next()

		// save the IP of the requester
		requester := c.Request.Header.Get("X-Real-IP")
		// if the requester-header is empty, check the forwarded-header
		if len(requester) == 0 {
			requester = c.Request.Header.Get("X-Forwarded-For")
		}
		// if the requester is still empty, use the hard-coded address from the socket
		if len(requester) == 0 {
			requester = c.Request.RemoteAddr
		}

		var color string
		code := c.Writer.Status()
		switch {
		case code >= 200 && code <= 299:
			color = green
		case code >= 300 && code <= 399:
			color = white
		case code >= 400 && code <= 499:
			color = yellow
		default:
			color = red
		}

		var methodColor string
		method := c.Request.Method
		switch {
		case method == "GET":
			methodColor = blue
		case method == "POST":
			methodColor = cyan
		case method == "PUT":
			methodColor = yellow
		case method == "DELETE":
			methodColor = red
		case method == "PATCH":
			methodColor = green
		case method == "HEAD":
			methodColor = magenta
		case method == "OPTIONS":
			methodColor = white
		}
		end := time.Now()
		latency := end.Sub(start)
		stdlogger.Printf("[GIN] %v |%s %3d %s| %12v | %s |%s %4s %s| %s\n%s",
			end.Format("2006/01/02 - 15:04:05"),
			color, code, reset,
			latency,
			requester,
			methodColor, method, reset,
			c.Request.URL.Path,
			c.Errors.String(),
		)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Martini Authors
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package macaron

import (
	"fmt"
	"log"
	"net/http"
	"runtime"
	"time"
)

var isWindows bool

func init() {
	isWindows = runtime.GOOS == "windows"
}

// Logger returns a middleware handler that logs the request as it goes in and the response as it goes out.
func Logger() Handler {
	return func(ctx *Context, log *log.Logger) {
		start := time.Now()

		log.Printf("Started %s %s for %s", ctx.Req.Method,ctx.Req.RequestURI, ctx.RemoteAddr())

		rw := ctx.Resp.(ResponseWriter)
		ctx.Next()

		content := fmt.Sprintf("Completed %s %v %s in %v",  ctx.Req.RequestURI, rw.Status(), http.StatusText(rw.Status()), time.Since(start))
		if !isWindows {
			switch rw.Status() {
			case 200, 201, 202:
				content = fmt.Sprintf("\033[1;32m%s\033[0m", content)
			case 301, 302:
				content = fmt.Sprintf("\033[1;37m%s\033[0m", content)
			case 304:
				content = fmt.Sprintf("\033[1;33m%s\033[0m", content)
			case 401, 403:
				content = fmt.Sprintf("\033[4;31m%s\033[0m", content)
			case 404:
				content = fmt.Sprintf("\033[1;31m%s\033[0m", content)
			case 500:
				content = fmt.Sprintf("\033[1;36m%s\033[0m", content)
			}
		}
		log.Println(content)
	}
}
<commit_msg>able to disable colorlog<commit_after>// Copyright 2013 Martini Authors
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package macaron

import (
	"fmt"
	"log"
	"net/http"
	"runtime"
	"time"
)

var ColorLog = true

func init() {
	ColorLog = runtime.GOOS != "windows"
}

// Logger returns a middleware handler that logs the request as it goes in and the response as it goes out.
func Logger() Handler {
	return func(ctx *Context, log *log.Logger) {
		start := time.Now()

		log.Printf("Started %s %s for %s", ctx.Req.Method, ctx.Req.RequestURI, ctx.RemoteAddr())

		rw := ctx.Resp.(ResponseWriter)
		ctx.Next()

		content := fmt.Sprintf("Completed %s %v %s in %v", ctx.Req.RequestURI, rw.Status(), http.StatusText(rw.Status()), time.Since(start))
		if ColorLog {
			switch rw.Status() {
			case 200, 201, 202:
				content = fmt.Sprintf("\033[1;32m%s\033[0m", content)
			case 301, 302:
				content = fmt.Sprintf("\033[1;37m%s\033[0m", content)
			case 304:
				content = fmt.Sprintf("\033[1;33m%s\033[0m", content)
			case 401, 403:
				content = fmt.Sprintf("\033[4;31m%s\033[0m", content)
			case 404:
				content = fmt.Sprintf("\033[1;31m%s\033[0m", content)
			case 500:
				content = fmt.Sprintf("\033[1;36m%s\033[0m", content)
			}
		}
		log.Println(content)
	}
}
<|endoftext|>


<commit_before>package boondoggle

import (
	"fmt"
	"log"
	"net/http"
	"strings"
)

type RequestLogger interface {
	Log(*http.Request)
}

type DefaultLogger struct{}

// By default, a timestamp will be written by the logger
func (l *DefaultLogger) Log(r *http.Request) {
	log.Printf(
		`%q %s %q`,
		fmt.Sprintf(`%s %s`, r.Method, r.URL),
		strings.SplitN(r.RemoteAddr, ":", 2)[0],
		r.Header.Get("User-Agent"),
	)
}

var defaultLogger = &DefaultLogger{}
<commit_msg>Log will now prefer the X-Real-IP header to the RemoteAddr<commit_after>package boondoggle

import (
	"fmt"
	"log"
	"net/http"
	"strings"
)

type RequestLogger interface {
	Log(*http.Request)
}

type DefaultLogger struct{}

// By default, a timestamp will be written by the logger
func (l *DefaultLogger) Log(r *http.Request) {
	ip := strings.SplitN(r.Header.Get("X-Real-IP"), ":", 2)[0]
	if ip == "" {
		ip = strings.SplitN(r.RemoteAddr, ":", 2)[0]
	}
	log.Printf(
		`%q %s %q`,
		fmt.Sprintf(`%s %s`, r.Method, r.URL),
		ip,
		r.Header.Get("User-Agent"),
	)
}

var defaultLogger = &DefaultLogger{}
<|endoftext|>


<commit_before>package fibonacci

import (
	"math"
	"strconv"
	"testing"
)

func TestFibonacciNegative(t *testing.T) {
	var test_values = []int{-1, -2, -3, -10, -1000, -1000000, math.MinInt64}
	for _, i := range test_values {
		if _, err := NewGenerator(i); err == nil {
			t.Errorf("Expected NewGenerator to return error when asked for %d iterations", i)

		}
	}
}

func TestFibonacciNumberOfValuesGenerated(t *testing.T) {
	var test_values = []int{1, 2, 3, 10, 20, 100, 1000, 100000}

	for _, i := range test_values {
		fg, err := NewGenerator(i)
		if err != nil {
			t.Errorf("Expected NewGenerator to return error when asked for %d iterations", i)
		}

		result_chan := make(chan FibNum)
		go fg.Produce(result_chan)

		cnt := 0
		for dont_care := range result_chan {
			_ = dont_care
			cnt = cnt + 1
		}

		if cnt != i {
			t.Errorf("Generated %d numbers when asked to generate %d, maxIterations is %d", cnt, i, fg.maxIterations)
		}
	}
}

func TestFibonacciVerifyCorrectOutputAgainstInt(t *testing.T) {
	//At 'iteration' 93 int rolls and cannot be used to validate the implemented algorithm
	var test_values = []int{1, 2, 3, 10, 20, 93}

	for _, i := range test_values {
		fg, err := NewGenerator(i)
		if err != nil {
			t.Errorf("Expected NewGenerator to return error when asked for %d iterations", i)
		}

		result_chan := make(chan FibNum)
		go fg.Produce(result_chan)

		cnt := 0
		x, y := 0, 1
		for val := range result_chan {
			cnt = cnt + 1
			if x < 0 {
				t.Fatalf("Test problem on %d iteration: expected value went negative %d", cnt, x)
			}
			if strconv.Itoa(x) != val.String() {
				t.Fatalf("Incorrect value on iteration %d of test %d: expected %d got %s",
					cnt, i, x, val.String())
			}
			x, y = y, x+y
		}
	}
}
<commit_msg>Added large value unit test<commit_after>package fibonacci

import (
	"math"
	"strconv"
	"testing"
)

func TestFibonacciNegative(t *testing.T) {
	var test_values = []int{-1, -2, -3, -10, -1000, -1000000, math.MinInt64}
	for _, i := range test_values {
		if _, err := NewGenerator(i); err == nil {
			t.Errorf("Expected NewGenerator to return error when asked for %d iterations", i)

		}
	}
}

func TestFibonacciNumberOfValuesGenerated(t *testing.T) {
	var test_values = []int{1, 2, 3, 10, 20, 100, 1000, 100000}

	for _, i := range test_values {
		fg, err := NewGenerator(i)
		if err != nil {
			t.Errorf("Expected NewGenerator to return error when asked for %d iterations", i)
		}

		result_chan := make(chan FibNum)
		go fg.Produce(result_chan)

		cnt := 0
		for dont_care := range result_chan {
			_ = dont_care
			cnt = cnt + 1
		}

		if cnt != i {
			t.Errorf("Generated %d numbers when asked to generate %d, maxIterations is %d", cnt, i, fg.maxIterations)
		}
	}
}

func TestFibonacciVerifyCorrectOutputAgainstInt(t *testing.T) {
	//At 'iteration' 93 int rolls and cannot be used to validate the implemented algorithm
	var test_values = []int{1, 2, 3, 10, 20, 93}

	for _, i := range test_values {
		fg, err := NewGenerator(i)
		if err != nil {
			t.Errorf("Expected NewGenerator to return successful when asked for %d iterations", i)
		}

		result_chan := make(chan FibNum)
		go fg.Produce(result_chan)

		cnt := 0
		x, y := 0, 1
		for val := range result_chan {
			cnt = cnt + 1
			if x < 0 {
				t.Fatalf("Test problem on %d iteration: expected value went negative %d", cnt, x)
			}
			if strconv.Itoa(x) != val.String() {
				t.Fatalf("Incorrect value on iteration %d of test %d: expected %d got %s",
					cnt, i, x, val.String())
			}
			x, y = y, x+y
		}
	}
}

func TestFibonacciLargeValue(t *testing.T) {
	iterations := 20001
	fg, err := NewGenerator(iterations)
	if err != nil {
		t.Errorf("Expected NewGenerator to return success when asked for %d iterations", iterations)
	}

	result_chan := make(chan FibNum)
	go fg.Produce(result_chan)

	last := newFibNum(0)
	for val := range result_chan {
		last = val
	}

	// expected value is 20000th calculated from http://php.bubble.ro/fibonacci/
	// that calculator starts with 1,1 instead of 0,1
	expected := "2531162323732361242240155003520607291766356485802485278951929841991312781760541315230153423463758831637443488219211037689033673531462742885329724071555187618026931630449193158922771331642302030331971098689235780843478258502779200293635651897483309686042860996364443514558772156043691404155819572984971754278513112487985892718229593329483578531419148805380281624260900362993556916638613939977074685016188258584312329139526393558096840812970422952418558991855772306882442574855589237165219912238201311184749075137322987656049866305366913734924425822681338966507463855180236283582409861199212323835947891143765414913345008456022009455704210891637791911265475167769704477334859109822590053774932978465651023851447920601310106288957894301592502061560528131203072778677491443420921822590709910448617329156135355464620891788459566081572824889514296350670950824208245170667601726417091127999999941149913010424532046881958285409468463211897582215075436515584016297874572183907949257286261608612401379639484713101138120404671732190451327881433201025184027541696124114463488665359385870910331476156665889459832092710304159637019707297988417848767011085425271875588008671422491434005115288334343837778792282383576736341414410248994081564830202363820504190074504566612515965134665683289356188727549463732830075811851574961558669278847363279870595320099844676879457196432535973357128305390290471349480258751812890314779723508104229525161740643984423978659638233074463100366500571977234508464710078102581304823235436518145074482824812996511614161933313389889630935320139507075992100561077534028207257574257706278201308302642634678112591091843082665721697117838726431766741158743554298864560993255547608496686850185804659790217122426535133253371422250684486113457341827911625517128815447325958547912113242367201990672230681308819195941016156001961954700241576553750737681552256845421159386858399433450045903975167084252876848848085910156941603293424067793097271128806817514906531652407763118308162377033463203514657531210413149191213595455280387631030665594589183601575340027172997222489081631144728873621805528648768511368948639522975539046995395707688938978847084621586473529546678958226255042389998718141303055036060772003887773038422366913820397748550793178167220193346017430024134496141145991896227741842515718997898627269918236920453493946658273870473264523119133765447653295022886429174942653014656521909469613184983671431465934965489425515981067546087342348350724207583544436107294087637975025147846254526938442435644928231027868701394819091132912397475713787593612758364812687556725146456646878912169274219209708166678668152184941578590201953144030519381922273252666652671717526318606676754556170379350956342095455612780202199922615392785572481747913435560866995432578680971243966868110016581395696310922519803685837460795358384618017215468122880442252343684547233668502313239328352671318130604247460452134121833305284398726438573787798499612760939462427922917659263046333084007208056631996856315539698234022953452211505675629153637867252695056925345220084020071611220575700841268302638995272842160994219632684575364180160991884885091858259996299627148614456696661412745040519981575543804847463997422326563897043803732970397488471644906183310144691243649149542394691524972023935190633672827306116525712882959108434211652465621144702015336657459532134026915214509960877430595844287585350290234547564574848753110281101545931547225811763441710217452979668178025286460158324658852904105792472468108996135476637212057508192176910900422826969523438985332067597093454021924077101784215936539638808624420121459718286059401823614213214326004270471752802725625810953787713898846144256909835116371235019527013180204030167601567064268573820697948868982630904164685161783088076506964317303709708574052747204405282785965604677674192569851918643651835755242670293612851920696732320545562286110332140065912751551110134916256237884844001366366654055079721985816714803952429301558096968202261698837096090377863017797020488044826628817462866854321356787305635653577619877987998113667928954840972022833505708587561902023411398915823487627297968947621416912816367516125096563705174220460639857683971213093125"

	if len(last.String()) != len(expected) {
		t.Errorf("%dth value was not the correct length got:%d expected:%d",
			iterations, len(last.String()), len(expected))
	}
	if last.String() != expected {
		t.Errorf("%dth value did not match the expected\ngot:%s#\nexpected:%s#\n", iterations, last, expected)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 David Lavieri.  All rights reserved.
// Use of this source code is governed by a MIT License
// License that can be found in the LICENSE file.

package goradix

// ----------------------- Look Up ------------------------ //

// LookUp will return the node matching
func (r *Radix) LookUp(s string) (interface{}, error) {
	return r.LookUpBytes([]byte(s))
}

// LookUpBytes will return the node matching
func (r *Radix) LookUpBytes(bs []byte) (interface{}, error) {
	node, key, err := r.sLookUp(bs)

	if err != nil {
		return nil, err
	}

	if !key {
		return nil, ErrNoMatchFound
	}

	return node.get(), err
}

func (r *Radix) sLookUp(bs []byte) (*Radix, bool, error) {
	var traverseNode = r

	traverseNode.mu.RLock()
	defer traverseNode.mu.RUnlock()

	lbs, matches, _ := traverseNode.match(bs)

	// && ((!r.master && matches > 0) || r.master)
	if matches == len(traverseNode.Path) {
		if matches < len(bs) {
			for _, n := range traverseNode.nodes {
				if tn, nkey, err := n.sLookUp(lbs); tn != nil {
					return tn, nkey, err
				}
			}

			// Do not jump back to parent node
			return nil, false, ErrNoMatchFound
		}

		return traverseNode, traverseNode.key, nil
	}

	return nil, false, ErrNoMatchFound
}
<commit_msg>Update lookup locks for better performance<commit_after>// Copyright 2016 David Lavieri.  All rights reserved.
// Use of this source code is governed by a MIT License
// License that can be found in the LICENSE file.

package goradix

// ----------------------- Look Up ------------------------ //

// LookUp will return the node matching
func (r *Radix) LookUp(s string) (interface{}, error) {
	return r.LookUpBytes([]byte(s))
}

// LookUpBytes will return the node matching
func (r *Radix) LookUpBytes(bs []byte) (interface{}, error) {
	node, key, err := r.sLookUp(bs)

	if err != nil {
		return nil, err
	}

	if !key {
		return nil, ErrNoMatchFound
	}

	return node.get(), err
}

func (r *Radix) sLookUp(bs []byte) (*Radix, bool, error) {
	var traverseNode = r

	traverseNode.mu.RLock()

	lbs, matches, _ := traverseNode.match(bs)

	// && ((!r.master && matches > 0) || r.master)
	if matches == len(traverseNode.Path) {
		if matches < len(bs) {
			for _, n := range traverseNode.nodes {
				if tn, nkey, err := n.sLookUp(lbs); tn != nil {
					traverseNode.mu.RUnlock()

					return tn, nkey, err
				}
			}

			traverseNode.mu.RUnlock()

			// Do not jump back to parent node
			return nil, false, ErrNoMatchFound
		}

		traverseNode.mu.RUnlock()

		return traverseNode, traverseNode.key, nil
	}

	traverseNode.mu.RUnlock()

	return nil, false, ErrNoMatchFound
}
<|endoftext|>


<commit_before>package state

import (
	"reflect"

	"github.com/hashicorp/terraform/terraform"
)

// CacheState is an implementation of the state interfaces that uses
// a StateReadWriter for a local cache.
type CacheState struct {
	Cache   CacheStateCache
	Durable CacheStateDurable

	refreshResult CacheRefreshResult
	state         *terraform.State
}

// StateReader impl.
func (s *CacheState) State() *terraform.State {
	return s.state
}

// WriteState will write and persist the state to the cache.
//
// StateWriter impl.
func (s *CacheState) WriteState(state *terraform.State) error {
	if err := s.Cache.WriteState(state); err != nil {
		return err
	}

	s.state = state
	return s.Cache.PersistState()
}

// RefreshState will refresh both the cache and the durable states. It
// can return a myriad of errors (defined at the top of this file) depending
// on potential conflicts that can occur while doing this.
//
// If the durable state is newer than the local cache, then the local cache
// will be replaced with the durable.
//
// StateRefresher impl.
func (s *CacheState) RefreshState() error {
	// Refresh the durable state
	if err := s.Durable.RefreshState(); err != nil {
		return err
	}

	// Refresh the cached state
	if err := s.Cache.RefreshState(); err != nil {
		return err
	}

	// Handle the matrix of cases that can happen when comparing these
	// two states.
	cached := s.Cache.State()
	durable := s.Durable.State()
	switch {
	case cached == nil && durable == nil:
		// Initialized
		s.refreshResult = CacheRefreshInit
	case cached != nil && durable == nil:
		// Cache is newer than remote. Not a big deal, user can just
		// persist to get correct state.
		s.refreshResult = CacheRefreshLocalNewer
	case cached == nil && durable != nil:
		// Cache should be updated since the remote is set but cache isn't
		s.refreshResult = CacheRefreshUpdateLocal
	case durable.Serial < cached.Serial:
		// Cache is newer than remote. Not a big deal, user can just
		// persist to get correct state.
		s.refreshResult = CacheRefreshLocalNewer
	case durable.Serial > cached.Serial:
		// Cache should be updated since the remote is newer
		s.refreshResult = CacheRefreshUpdateLocal
	case durable.Serial == cached.Serial:
		// They're supposedly equal, verify.
		if reflect.DeepEqual(cached, durable) {
			// Hashes are the same, everything is great
			s.refreshResult = CacheRefreshNoop
			break
		}

		// This is very bad. This means we have two state files that
		// have the same serial but have a different hash. We can't
		// reconcile this. The most likely cause is parallel apply
		// operations.
		s.refreshResult = CacheRefreshConflict

		// Return early so we don't updtae the state
		return nil
	default:
		panic("unhandled cache refresh state")
	}

	if s.refreshResult == CacheRefreshUpdateLocal {
		if err := s.Cache.WriteState(durable); err != nil {
			s.refreshResult = CacheRefreshNoop
			return err
		}
		if err := s.Cache.PersistState(); err != nil {
			s.refreshResult = CacheRefreshNoop
			return err
		}
	}

	s.state = cached

	return nil
}

// RefreshResult returns the result of the last refresh.
func (s *CacheState) RefreshResult() CacheRefreshResult {
	return s.refreshResult
}

// PersistState takes the local cache, assuming it is newer than the remote
// state, and persists it to the durable storage. If you want to challenge the
// assumption that the local state is the latest, call a RefreshState prior
// to this.
//
// StatePersister impl.
func (s *CacheState) PersistState() error {
	if err := s.Durable.WriteState(s.state); err != nil {
		return err
	}

	return s.Durable.PersistState()
}

// CacheStateCache is the meta-interface that must be implemented for
// the cache for the CacheState.
type CacheStateCache interface {
	StateReader
	StateWriter
	StatePersister
	StateRefresher
}

// CacheStateDurable is the meta-interface that must be implemented for
// the durable storage for CacheState.
type CacheStateDurable interface {
	StateReader
	StateWriter
	StatePersister
	StateRefresher
}

// CacheRefreshResult is used to explain the result of the previous
// RefreshState for a CacheState.
type CacheRefreshResult int

const (
	// CacheRefreshNoop indicates nothing has happened,
	// but that does not indicate an error. Everything is
	// just up to date. (Push/Pull)
	CacheRefreshNoop CacheRefreshResult = iota

	// CacheRefreshInit indicates that there is no local or
	// remote state, and that the state was initialized
	CacheRefreshInit

	// CacheRefreshUpdateLocal indicates the local state
	// was updated. (Pull)
	CacheRefreshUpdateLocal

	// CacheRefreshUpdateRemote indicates the remote state
	// was updated. (Push)
	CacheRefreshUpdateRemote

	// CacheRefreshLocalNewer means the pull was a no-op
	// because the local state is newer than that of the
	// server. This means a Push should take place. (Pull)
	CacheRefreshLocalNewer

	// CacheRefreshRemoteNewer means the push was a no-op
	// because the remote state is newer than that of the
	// local state. This means a Pull should take place.
	// (Push)
	CacheRefreshRemoteNewer

	// CacheRefreshConflict means that the push or pull
	// was a no-op because there is a conflict. This means
	// there are multiple state definitions at the same
	// serial number with different contents. This requires
	// an operator to intervene and resolve the conflict.
	// Shame on the user for doing concurrent apply.
	// (Push/Pull)
	CacheRefreshConflict
)
<commit_msg>state: add strings for cache refresh result<commit_after>package state

import (
	"fmt"
	"reflect"

	"github.com/hashicorp/terraform/terraform"
)

// CacheState is an implementation of the state interfaces that uses
// a StateReadWriter for a local cache.
type CacheState struct {
	Cache   CacheStateCache
	Durable CacheStateDurable

	refreshResult CacheRefreshResult
	state         *terraform.State
}

// StateReader impl.
func (s *CacheState) State() *terraform.State {
	return s.state
}

// WriteState will write and persist the state to the cache.
//
// StateWriter impl.
func (s *CacheState) WriteState(state *terraform.State) error {
	if err := s.Cache.WriteState(state); err != nil {
		return err
	}

	s.state = state
	return s.Cache.PersistState()
}

// RefreshState will refresh both the cache and the durable states. It
// can return a myriad of errors (defined at the top of this file) depending
// on potential conflicts that can occur while doing this.
//
// If the durable state is newer than the local cache, then the local cache
// will be replaced with the durable.
//
// StateRefresher impl.
func (s *CacheState) RefreshState() error {
	// Refresh the durable state
	if err := s.Durable.RefreshState(); err != nil {
		return err
	}

	// Refresh the cached state
	if err := s.Cache.RefreshState(); err != nil {
		return err
	}

	// Handle the matrix of cases that can happen when comparing these
	// two states.
	cached := s.Cache.State()
	durable := s.Durable.State()
	switch {
	case cached == nil && durable == nil:
		// Initialized
		s.refreshResult = CacheRefreshInit
	case cached != nil && durable == nil:
		// Cache is newer than remote. Not a big deal, user can just
		// persist to get correct state.
		s.refreshResult = CacheRefreshLocalNewer
	case cached == nil && durable != nil:
		// Cache should be updated since the remote is set but cache isn't
		s.refreshResult = CacheRefreshUpdateLocal
	case durable.Serial < cached.Serial:
		// Cache is newer than remote. Not a big deal, user can just
		// persist to get correct state.
		s.refreshResult = CacheRefreshLocalNewer
	case durable.Serial > cached.Serial:
		// Cache should be updated since the remote is newer
		s.refreshResult = CacheRefreshUpdateLocal
	case durable.Serial == cached.Serial:
		// They're supposedly equal, verify.
		if reflect.DeepEqual(cached, durable) {
			// Hashes are the same, everything is great
			s.refreshResult = CacheRefreshNoop
			break
		}

		// This is very bad. This means we have two state files that
		// have the same serial but have a different hash. We can't
		// reconcile this. The most likely cause is parallel apply
		// operations.
		s.refreshResult = CacheRefreshConflict

		// Return early so we don't updtae the state
		return nil
	default:
		panic("unhandled cache refresh state")
	}

	if s.refreshResult == CacheRefreshUpdateLocal {
		if err := s.Cache.WriteState(durable); err != nil {
			s.refreshResult = CacheRefreshNoop
			return err
		}
		if err := s.Cache.PersistState(); err != nil {
			s.refreshResult = CacheRefreshNoop
			return err
		}
	}

	s.state = cached

	return nil
}

// RefreshResult returns the result of the last refresh.
func (s *CacheState) RefreshResult() CacheRefreshResult {
	return s.refreshResult
}

// PersistState takes the local cache, assuming it is newer than the remote
// state, and persists it to the durable storage. If you want to challenge the
// assumption that the local state is the latest, call a RefreshState prior
// to this.
//
// StatePersister impl.
func (s *CacheState) PersistState() error {
	if err := s.Durable.WriteState(s.state); err != nil {
		return err
	}

	return s.Durable.PersistState()
}

// CacheStateCache is the meta-interface that must be implemented for
// the cache for the CacheState.
type CacheStateCache interface {
	StateReader
	StateWriter
	StatePersister
	StateRefresher
}

// CacheStateDurable is the meta-interface that must be implemented for
// the durable storage for CacheState.
type CacheStateDurable interface {
	StateReader
	StateWriter
	StatePersister
	StateRefresher
}

// CacheRefreshResult is used to explain the result of the previous
// RefreshState for a CacheState.
type CacheRefreshResult int

const (
	// CacheRefreshNoop indicates nothing has happened,
	// but that does not indicate an error. Everything is
	// just up to date. (Push/Pull)
	CacheRefreshNoop CacheRefreshResult = iota

	// CacheRefreshInit indicates that there is no local or
	// remote state, and that the state was initialized
	CacheRefreshInit

	// CacheRefreshUpdateLocal indicates the local state
	// was updated. (Pull)
	CacheRefreshUpdateLocal

	// CacheRefreshUpdateRemote indicates the remote state
	// was updated. (Push)
	CacheRefreshUpdateRemote

	// CacheRefreshLocalNewer means the pull was a no-op
	// because the local state is newer than that of the
	// server. This means a Push should take place. (Pull)
	CacheRefreshLocalNewer

	// CacheRefreshRemoteNewer means the push was a no-op
	// because the remote state is newer than that of the
	// local state. This means a Pull should take place.
	// (Push)
	CacheRefreshRemoteNewer

	// CacheRefreshConflict means that the push or pull
	// was a no-op because there is a conflict. This means
	// there are multiple state definitions at the same
	// serial number with different contents. This requires
	// an operator to intervene and resolve the conflict.
	// Shame on the user for doing concurrent apply.
	// (Push/Pull)
	CacheRefreshConflict
)

func (sc CacheRefreshResult) String() string {
	switch sc {
	case CacheRefreshNoop:
		return "Local and remote state in sync"
	case CacheRefreshInit:
		return "Local state initialized"
	case CacheRefreshUpdateLocal:
		return "Local state updated"
	case CacheRefreshUpdateRemote:
		return "Remote state updated"
	case CacheRefreshLocalNewer:
		return "Local state is newer than remote state, push required"
	case CacheRefreshRemoteNewer:
		return "Remote state is newer than local state, pull required"
	case CacheRefreshConflict:
		return "Local and remote state conflict, manual resolution required"
	default:
		return fmt.Sprintf("Unknown state change type: %d", sc)
	}
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mapper

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"

	"github.com/ernestio/aws-definition-mapper/definition"
)

func TestMapELBs(t *testing.T) {

	Convey("Given a valid input definition", t, func() {
		d := definition.Definition{
			Name:       "service",
			Datacenter: "datacenter",
		}

		d.Instances = append(d.Instances, definition.Instance{
			Name:    "web",
			Network: "web-nw",
			Count:   2,
		})

		e := definition.ELB{
			Name:    "test",
			Private: true,
			Subnets: []string{
				"web-nw",
			},
			Instances: []string{
				"web",
			},
			SecurityGroups: []string{
				"web-sg",
			},
		}

		e.Listeners = append(e.Listeners, definition.ELBListener{
			FromPort: 1,
			ToPort:   2,
			Protocol: "http",
			SSLCert:  "cert",
		})

		d.ELBs = append(d.ELBs, e)

		Convey("When i try to map elbs", func() {

			e := MapELBs(d)
			Convey("Then it should map salt and input elb rules", func() {
				So(len(e), ShouldEqual, 1)
				So(e[0].Name, ShouldEqual, "datacenter-service-test")
				So(len(e[0].NetworkAWSIDs), ShouldEqual, 1)
				So(e[0].NetworkAWSIDs[0], ShouldEqual, `$(networks.items.#[name="datacenter-service-web-nw"].network_aws_id)`)
				So(len(e[0].InstanceAWSIDs), ShouldEqual, 2)
				So(e[0].InstanceAWSIDs[0], ShouldEqual, `$(instances.items.#[name="datacenter-service-web-1"].instance_aws_id)`)
				So(e[0].InstanceAWSIDs[1], ShouldEqual, `$(instances.items.#[name="datacenter-service-web-2"].instance_aws_id)`)
				So(len(e[0].SecurityGroupAWSIDs), ShouldEqual, 1)
				So(e[0].SecurityGroupAWSIDs[0], ShouldEqual, `$(firewalls.items.#[name="datacenter-service-web-sg"].security_group_aws_id)`)
				So(len(e[0].Listeners), ShouldEqual, 1)
				So(e[0].Listeners[0].FromPort, ShouldEqual, 1)
				So(e[0].Listeners[0].ToPort, ShouldEqual, 2)
				So(e[0].Listeners[0].Protocol, ShouldEqual, "HTTP")
				So(e[0].Listeners[0].SSLCert, ShouldEqual, "cert")
				So(e[0].Tags["Name"], ShouldEqual, "test")
				So(e[0].Tags["ernest.service"], ShouldEqual, "service")
			})

		})
	})

}
<commit_msg>fixed elb test<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mapper

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"

	"github.com/ernestio/aws-definition-mapper/definition"
)

func TestMapELBs(t *testing.T) {

	Convey("Given a valid input definition", t, func() {
		d := definition.Definition{
			Name:       "service",
			Datacenter: "datacenter",
		}

		d.Instances = append(d.Instances, definition.Instance{
			Name:    "web",
			Network: "web-nw",
			Count:   2,
		})

		e := definition.ELB{
			Name:    "test",
			Private: true,
			Subnets: []string{
				"web-nw",
			},
			Instances: []string{
				"web",
			},
			SecurityGroups: []string{
				"web-sg",
			},
		}

		e.Listeners = append(e.Listeners, definition.ELBListener{
			FromPort: 1,
			ToPort:   2,
			Protocol: "http",
			SSLCert:  "cert",
		})

		d.ELBs = append(d.ELBs, e)

		Convey("When i try to map elbs", func() {

			e := MapELBs(d)
			Convey("Then it should map salt and input elb rules", func() {
				So(len(e), ShouldEqual, 1)
				So(e[0].Name, ShouldEqual, "datacenter-service-test")
				So(len(e[0].NetworkAWSIDs), ShouldEqual, 1)
				So(e[0].NetworkAWSIDs[0], ShouldEqual, `$(networks.items.#[name="datacenter-service-web-nw"].network_aws_id)`)
				So(len(e[0].InstanceAWSIDs), ShouldEqual, 2)
				So(e[0].InstanceAWSIDs[0], ShouldEqual, `$(instances.items.#[name="datacenter-service-web-1"].instance_aws_id)`)
				So(e[0].InstanceAWSIDs[1], ShouldEqual, `$(instances.items.#[name="datacenter-service-web-2"].instance_aws_id)`)
				So(len(e[0].SecurityGroupAWSIDs), ShouldEqual, 1)
				So(e[0].SecurityGroupAWSIDs[0], ShouldEqual, `$(firewalls.items.#[name="datacenter-service-web-sg"].security_group_aws_id)`)
				So(len(e[0].Listeners), ShouldEqual, 1)
				So(e[0].Listeners[0].FromPort, ShouldEqual, 1)
				So(e[0].Listeners[0].ToPort, ShouldEqual, 2)
				So(e[0].Listeners[0].Protocol, ShouldEqual, "HTTP")
				So(e[0].Listeners[0].SSLCert, ShouldEqual, "cert")
				So(e[0].Tags["ernest.service"], ShouldEqual, "service")
			})

		})
	})

}
<|endoftext|>


<commit_before>// vim: ts=8 sw=8 noet ai

package perigee

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

// The UnexpectedResponseCodeError structure represents a mismatch in understanding between server and client in terms of response codes.
// Most often, this is due to an actual error condition (e.g., getting a 404 for a resource when you expect a 200).
// However, it needn't always be the case (e.g., getting a 204 (No Content) response back when a 200 is expected).
type UnexpectedResponseCodeError struct {
	Url      string
	Expected []int
	Actual   int
	Body     []byte
}

func (err *UnexpectedResponseCodeError) Error() string {
	return fmt.Sprintf("Expected HTTP response code %d when accessing URL(%s); got %d instead with the following body:\n%s", err.Expected, err.Url, err.Actual, string(err.Body))
}

// Request issues an HTTP request, marshaling parameters, and unmarshaling results, as configured in the provided Options parameter.
// The Response structure returned, if any, will include accumulated results recovered from the HTTP server.
// See the Response structure for more details.
func Request(method string, url string, opts Options) (*Response, error) {
	var body io.Reader
	var response Response

	client := opts.CustomClient
	if client == nil {
		client = new(http.Client)
	}

	contentType := opts.ContentType

	accept := opts.Accept
	if accept == "" {
		accept = "application/json"
	}

	body = nil
	if opts.ReqBody != nil {
		if contentType == "" {
			contentType = "application/json"
		}

		if contentType == "application/json" {
			bodyText, err := json.Marshal(opts.ReqBody)
			if err != nil {
				return nil, err
			}
			body = strings.NewReader(string(bodyText))
			if opts.DumpReqJson {
				log.Printf("Making request:\n%#v\n", string(bodyText))
			}
		} else {
			// assume opts.ReqBody implements the correct interface
			body = opts.ReqBody.(io.Reader)
		}
	}

	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	if contentType != "" {
		req.Header.Add("Content-Type", contentType)
	}

	req.Header.Add("Accept", accept)

	if opts.ContentLength > 0 {
		req.ContentLength = opts.ContentLength
		req.Header.Add("Content-Length", string(opts.ContentLength))
	}

	if opts.MoreHeaders != nil {
		for k, v := range opts.MoreHeaders {
			req.Header.Add(k, v)
		}
	}

	if opts.SetHeaders != nil {
		err = opts.SetHeaders(req)
		if err != nil {
			return &response, err
		}
	}

	httpResponse, err := client.Do(req)
	if httpResponse != nil {
		response.HttpResponse = *httpResponse
		response.StatusCode = httpResponse.StatusCode
	}

	if err != nil {
		return &response, err
	}
	// This if-statement is legacy code, preserved for backward compatibility.
	if opts.StatusCode != nil {
		*opts.StatusCode = httpResponse.StatusCode
	}

	acceptableResponseCodes := opts.OkCodes
	if len(acceptableResponseCodes) != 0 {
		if not_in(httpResponse.StatusCode, acceptableResponseCodes) {
			b, _ := ioutil.ReadAll(httpResponse.Body)
			httpResponse.Body.Close()
			return &response, &UnexpectedResponseCodeError{
				Url:      url,
				Expected: acceptableResponseCodes,
				Actual:   httpResponse.StatusCode,
				Body:     b,
			}
		}
	}
	if opts.Results != nil {
		defer httpResponse.Body.Close()
		jsonResult, err := ioutil.ReadAll(httpResponse.Body)
		response.JsonResult = jsonResult
		if err != nil {
			return &response, err
		}

		err = json.Unmarshal(jsonResult, opts.Results)
		// This if-statement is legacy code, preserved for backward compatibility.
		if opts.ResponseJson != nil {
			*opts.ResponseJson = jsonResult
		}
	}
	return &response, err
}

// not_in returns false if, and only if, the provided needle is _not_
// in the given set of integers.
func not_in(needle int, haystack []int) bool {
	for _, straw := range haystack {
		if needle == straw {
			return false
		}
	}
	return true
}

// Post makes a POST request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
// DEPRECATED.  Use Request() instead.
func Post(url string, opts Options) error {
	r, err := Request("POST", url, opts)
	if opts.Response != nil {
		*opts.Response = r
	}
	return err
}

// Get makes a GET request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
// DEPRECATED.  Use Request() instead.
func Get(url string, opts Options) error {
	r, err := Request("GET", url, opts)
	if opts.Response != nil {
		*opts.Response = r
	}
	return err
}

// Delete makes a DELETE request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
// DEPRECATED.  Use Request() instead.
func Delete(url string, opts Options) error {
	r, err := Request("DELETE", url, opts)
	if opts.Response != nil {
		*opts.Response = r
	}
	return err
}

// Put makes a PUT request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
// DEPRECATED.  Use Request() instead.
func Put(url string, opts Options) error {
	r, err := Request("PUT", url, opts)
	if opts.Response != nil {
		*opts.Response = r
	}
	return err
}

// Options describes a set of optional parameters to the various request calls.
//
// The custom client can be used for a variety of purposes beyond selecting encrypted versus unencrypted channels.
// Transports can be defined to provide augmented logging, header manipulation, et. al.
//
// If the ReqBody field is provided, it will be embedded as a JSON object.
// Otherwise, provide nil.
//
// If JSON output is to be expected from the response,
// provide either a pointer to the container structure in Results,
// or a pointer to a nil-initialized pointer variable.
// The latter method will cause the unmarshaller to allocate the container type for you.
// If no response is expected, provide a nil Results value.
//
// The MoreHeaders map, if non-nil or empty, provides a set of headers to add to those
// already present in the request.  At present, only Accepted and Content-Type are set
// by default.
//
// OkCodes provides a set of acceptable, positive responses.
//
// If provided, StatusCode specifies a pointer to an integer, which will receive the
// returned HTTP status code, successful or not.  DEPRECATED; use the Response.StatusCode field instead for new software.
//
// ResponseJson, if specified, provides a means for returning the raw JSON.  This is
// most useful for diagnostics.  DEPRECATED; use the Response.JsonResult field instead for new software.
//
// DumpReqJson, if set to true, will cause the request to appear to stdout for debugging purposes.
// This attribute may be removed at any time in the future; DO NOT use this attribute in production software.
//
// Response, if set, provides a way to communicate the complete set of HTTP response, raw JSON, status code, and
// other useful attributes back to the caller.  Note that the Request() method returns a Response structure as part
// of its public interface; you don't need to set the Response field here to use this structure.  The Response field
// exists primarily for legacy or deprecated functions.
//
// SetHeaders allows the caller to provide code to set any custom headers programmatically.  Typically, this
// facility can invoke, e.g., SetBasicAuth() on the request to easily set up authentication.
// Any error generated will terminate the request and will propegate back to the caller.
type Options struct {
	CustomClient  *http.Client
	ReqBody       interface{}
	Results       interface{}
	MoreHeaders   map[string]string
	OkCodes       []int
	StatusCode    *int    `DEPRECATED`
	DumpReqJson   bool    `UNSUPPORTED`
	ResponseJson  *[]byte `DEPRECATED`
	Response      **Response
	ContentType   string `json:"Content-Type,omitempty"`
	ContentLength int64  `json:"Content-Length,omitempty"`
	Accept        string `json:"Accept,omitempty"`
	SetHeaders    func(r *http.Request) error
}

// Response contains return values from the various request calls.
//
// HttpResponse will return the http response from the request call.
// Note: HttpResponse.Body is always closed and will not be available from this return value.
//
// StatusCode specifies the returned HTTP status code, successful or not.
//
// If Results is specified in the Options:
// - JsonResult will contain the raw return from the request call
//   This is most useful for diagnostics.
// - Result will contain the unmarshalled json either in the Result passed in
//   or the unmarshaller will allocate the container type for you.

type Response struct {
	HttpResponse http.Response
	JsonResult   []byte
	Results      interface{}
	StatusCode   int
}
<commit_msg>use Accept header if it exists<commit_after>// vim: ts=8 sw=8 noet ai

package perigee

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

// The UnexpectedResponseCodeError structure represents a mismatch in understanding between server and client in terms of response codes.
// Most often, this is due to an actual error condition (e.g., getting a 404 for a resource when you expect a 200).
// However, it needn't always be the case (e.g., getting a 204 (No Content) response back when a 200 is expected).
type UnexpectedResponseCodeError struct {
	Url      string
	Expected []int
	Actual   int
	Body     []byte
}

func (err *UnexpectedResponseCodeError) Error() string {
	return fmt.Sprintf("Expected HTTP response code %d when accessing URL(%s); got %d instead with the following body:\n%s", err.Expected, err.Url, err.Actual, string(err.Body))
}

// Request issues an HTTP request, marshaling parameters, and unmarshaling results, as configured in the provided Options parameter.
// The Response structure returned, if any, will include accumulated results recovered from the HTTP server.
// See the Response structure for more details.
func Request(method string, url string, opts Options) (*Response, error) {
	var body io.Reader
	var response Response

	client := opts.CustomClient
	if client == nil {
		client = new(http.Client)
	}

	contentType := opts.ContentType

	body = nil
	if opts.ReqBody != nil {
		if contentType == "" {
			contentType = "application/json"
		}

		if contentType == "application/json" {
			bodyText, err := json.Marshal(opts.ReqBody)
			if err != nil {
				return nil, err
			}
			body = strings.NewReader(string(bodyText))
			if opts.DumpReqJson {
				log.Printf("Making request:\n%#v\n", string(bodyText))
			}
		} else {
			// assume opts.ReqBody implements the correct interface
			body = opts.ReqBody.(io.Reader)
		}
	}

	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	if contentType != "" {
		req.Header.Add("Content-Type", contentType)
	}

	if opts.ContentLength > 0 {
		req.ContentLength = opts.ContentLength
		req.Header.Add("Content-Length", string(opts.ContentLength))
	}

	if opts.MoreHeaders != nil {
		for k, v := range opts.MoreHeaders {
			req.Header.Add(k, v)
		}
	}

	if accept := req.Header.Get("Accept"); accept == "" {
		accept = opts.Accept
		if accept == "" {
			accept = "application/json"
		}
		req.Header.Add("Accept", accept)
	}

	if opts.SetHeaders != nil {
		err = opts.SetHeaders(req)
		if err != nil {
			return &response, err
		}
	}

	httpResponse, err := client.Do(req)
	if httpResponse != nil {
		response.HttpResponse = *httpResponse
		response.StatusCode = httpResponse.StatusCode
	}

	if err != nil {
		return &response, err
	}
	// This if-statement is legacy code, preserved for backward compatibility.
	if opts.StatusCode != nil {
		*opts.StatusCode = httpResponse.StatusCode
	}

	acceptableResponseCodes := opts.OkCodes
	if len(acceptableResponseCodes) != 0 {
		if not_in(httpResponse.StatusCode, acceptableResponseCodes) {
			b, _ := ioutil.ReadAll(httpResponse.Body)
			httpResponse.Body.Close()
			return &response, &UnexpectedResponseCodeError{
				Url:      url,
				Expected: acceptableResponseCodes,
				Actual:   httpResponse.StatusCode,
				Body:     b,
			}
		}
	}
	if opts.Results != nil {
		defer httpResponse.Body.Close()
		jsonResult, err := ioutil.ReadAll(httpResponse.Body)
		response.JsonResult = jsonResult
		if err != nil {
			return &response, err
		}

		err = json.Unmarshal(jsonResult, opts.Results)
		// This if-statement is legacy code, preserved for backward compatibility.
		if opts.ResponseJson != nil {
			*opts.ResponseJson = jsonResult
		}
	}
	return &response, err
}

// not_in returns false if, and only if, the provided needle is _not_
// in the given set of integers.
func not_in(needle int, haystack []int) bool {
	for _, straw := range haystack {
		if needle == straw {
			return false
		}
	}
	return true
}

// Post makes a POST request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
// DEPRECATED.  Use Request() instead.
func Post(url string, opts Options) error {
	r, err := Request("POST", url, opts)
	if opts.Response != nil {
		*opts.Response = r
	}
	return err
}

// Get makes a GET request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
// DEPRECATED.  Use Request() instead.
func Get(url string, opts Options) error {
	r, err := Request("GET", url, opts)
	if opts.Response != nil {
		*opts.Response = r
	}
	return err
}

// Delete makes a DELETE request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
// DEPRECATED.  Use Request() instead.
func Delete(url string, opts Options) error {
	r, err := Request("DELETE", url, opts)
	if opts.Response != nil {
		*opts.Response = r
	}
	return err
}

// Put makes a PUT request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
// DEPRECATED.  Use Request() instead.
func Put(url string, opts Options) error {
	r, err := Request("PUT", url, opts)
	if opts.Response != nil {
		*opts.Response = r
	}
	return err
}

// Options describes a set of optional parameters to the various request calls.
//
// The custom client can be used for a variety of purposes beyond selecting encrypted versus unencrypted channels.
// Transports can be defined to provide augmented logging, header manipulation, et. al.
//
// If the ReqBody field is provided, it will be embedded as a JSON object.
// Otherwise, provide nil.
//
// If JSON output is to be expected from the response,
// provide either a pointer to the container structure in Results,
// or a pointer to a nil-initialized pointer variable.
// The latter method will cause the unmarshaller to allocate the container type for you.
// If no response is expected, provide a nil Results value.
//
// The MoreHeaders map, if non-nil or empty, provides a set of headers to add to those
// already present in the request.  At present, only Accepted and Content-Type are set
// by default.
//
// OkCodes provides a set of acceptable, positive responses.
//
// If provided, StatusCode specifies a pointer to an integer, which will receive the
// returned HTTP status code, successful or not.  DEPRECATED; use the Response.StatusCode field instead for new software.
//
// ResponseJson, if specified, provides a means for returning the raw JSON.  This is
// most useful for diagnostics.  DEPRECATED; use the Response.JsonResult field instead for new software.
//
// DumpReqJson, if set to true, will cause the request to appear to stdout for debugging purposes.
// This attribute may be removed at any time in the future; DO NOT use this attribute in production software.
//
// Response, if set, provides a way to communicate the complete set of HTTP response, raw JSON, status code, and
// other useful attributes back to the caller.  Note that the Request() method returns a Response structure as part
// of its public interface; you don't need to set the Response field here to use this structure.  The Response field
// exists primarily for legacy or deprecated functions.
//
// SetHeaders allows the caller to provide code to set any custom headers programmatically.  Typically, this
// facility can invoke, e.g., SetBasicAuth() on the request to easily set up authentication.
// Any error generated will terminate the request and will propegate back to the caller.
type Options struct {
	CustomClient  *http.Client
	ReqBody       interface{}
	Results       interface{}
	MoreHeaders   map[string]string
	OkCodes       []int
	StatusCode    *int    `DEPRECATED`
	DumpReqJson   bool    `UNSUPPORTED`
	ResponseJson  *[]byte `DEPRECATED`
	Response      **Response
	ContentType   string `json:"Content-Type,omitempty"`
	ContentLength int64  `json:"Content-Length,omitempty"`
	Accept        string `json:"Accept,omitempty"`
	SetHeaders    func(r *http.Request) error
}

// Response contains return values from the various request calls.
//
// HttpResponse will return the http response from the request call.
// Note: HttpResponse.Body is always closed and will not be available from this return value.
//
// StatusCode specifies the returned HTTP status code, successful or not.
//
// If Results is specified in the Options:
// - JsonResult will contain the raw return from the request call
//   This is most useful for diagnostics.
// - Result will contain the unmarshalled json either in the Result passed in
//   or the unmarshaller will allocate the container type for you.

type Response struct {
	HttpResponse http.Response
	JsonResult   []byte
	Results      interface{}
	StatusCode   int
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/slack-go/slack"
)

var (
	api     *slack.Client
	msgOpts = &slack.PostMessageParameters{AsUser: true}
)

// InitAPI for Slack
func InitAPI(token string) {
	api = slack.New(token)
	res, err := api.AuthTest()
	failOnError(err, "Slack API Error")
	output(fmt.Sprintf("connected to %s as %s", res.Team, res.User))
}

// Return list of all channels by name
func listChannels() (names []string) {
	list, err := getConversations("public_channel", "private_channel")
	failOnError(err)
	for _, c := range list {
		names = append(names, c.Name)
	}
	return names
}

// Return list of all groups by name
func listGroups() (names []string) {
	list, err := getConversations("mpim")
	failOnError(err)
	for _, c := range list {
		names = append(names, c.Name)
	}
	return names
}

// Return list of all ims by name
func listIms() (names []string) {
	users, err := api.GetUsers()
	failOnError(err)

	list, err := getConversations("im")
	failOnError(err)
	for _, c := range list {
		for _, u := range users {
			if u.ID == c.User {
				names = append(names, u.Name)
				continue
			}
		}
	}
	return names
}

// Lookup Slack id for channel, group, or im by name
func lookupSlackID(name string) string {
	list, err := getConversations("public_channel", "private_channel", "mpim")
	if err == nil {
		for _, c := range list {
			if c.Name == name {
				return c.ID
			}
		}
	}
	users, err := api.GetUsers()
	if err == nil {
		list, err := getConversations("im")
		if err == nil {
			for _, c := range list {
				for _, u := range users {
					if u.Name == name && u.ID == c.User {
						return c.ID
					}
				}
			}
		}
	}
	exitErr(fmt.Errorf("No such channel, group, or im"))
	return ""
}

func getConversations(types ...string) (list []slack.Channel, err error) {
	cursor := ""
	for {
		param := &slack.GetConversationsParameters{
			Cursor:          cursor,
			ExcludeArchived: "true",
			Types:           types,
		}
		channels, cur, err := api.GetConversations(param)
		if err != nil {
			return list, err
		}
		list = append(list, channels...)
		if cur == "" {
			break
		}
		cursor = cur
	}
	return list, nil
}
<commit_msg>get conversation: change limit parameter and retry for rate limit<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/slack-go/slack"
)

var (
	api     *slack.Client
	msgOpts = &slack.PostMessageParameters{AsUser: true}
)

// InitAPI for Slack
func InitAPI(token string) {
	api = slack.New(token)
	res, err := api.AuthTest()
	failOnError(err, "Slack API Error")
	output(fmt.Sprintf("connected to %s as %s", res.Team, res.User))
}

// Return list of all channels by name
func listChannels() (names []string) {
	list, err := getConversations("public_channel", "private_channel")
	failOnError(err)
	for _, c := range list {
		names = append(names, c.Name)
	}
	return names
}

// Return list of all groups by name
func listGroups() (names []string) {
	list, err := getConversations("mpim")
	failOnError(err)
	for _, c := range list {
		names = append(names, c.Name)
	}
	return names
}

// Return list of all ims by name
func listIms() (names []string) {
	users, err := api.GetUsers()
	failOnError(err)

	list, err := getConversations("im")
	failOnError(err)
	for _, c := range list {
		for _, u := range users {
			if u.ID == c.User {
				names = append(names, u.Name)
				continue
			}
		}
	}
	return names
}

// Lookup Slack id for channel, group, or im by name
func lookupSlackID(name string) string {
	list, err := getConversations("public_channel", "private_channel", "mpim")
	if err == nil {
		for _, c := range list {
			if c.Name == name {
				return c.ID
			}
		}
	}
	users, err := api.GetUsers()
	if err == nil {
		list, err := getConversations("im")
		if err == nil {
			for _, c := range list {
				for _, u := range users {
					if u.Name == name && u.ID == c.User {
						return c.ID
					}
				}
			}
		}
	}
	exitErr(fmt.Errorf("No such channel, group, or im"))
	return ""
}

func getConversations(types ...string) (list []slack.Channel, err error) {
	cursor := ""
	for {
		param := &slack.GetConversationsParameters{
			Cursor:          cursor,
			ExcludeArchived: "true",
			Types:           types,
			Limit:           1000,
		}
		channels, cur, err := api.GetConversations(param)
		if err != nil {
			if rateLimitedError, ok := err.(*slack.RateLimitedError); ok {
				output(fmt.Sprintf("%v", rateLimitedError))
				time.Sleep(rateLimitedError.RetryAfter)
				continue
			}
			return list, err
		}
		list = append(list, channels...)
		if cur == "" {
			break
		}
		cursor = cur
	}
	return list, nil
}
<|endoftext|>