<commit_before>package pubsub_test

import (
	. "github.com/puffinframework/local.pubsub"
	. "github.com/puffinframework/pubsub/impltests"
	"testing"
)

func TestRemote(t *testing.T) {
	pb := NewPubSub()
	TestSubscribe(t, pb)
	pb.Close()

	pb = NewPubSub()
	TestUnsubscribe(t, pb)
	pb.Close()

	pb = NewPubSub()
	TestSubscribeSync(t, pb)
	pb.Close()
}
<commit_msg>update<commit_after>package pubsub_test

import (
	. "github.com/puffinframework/local.pubsub"
	. "github.com/puffinframework/pubsub/impltests"
	"testing"
)

func Test(t *testing.T) {
	pb := NewPubSub()
	TestSubscribe(t, pb)
	pb.Close()

	pb = NewPubSub()
	TestUnsubscribe(t, pb)
	pb.Close()

	pb = NewPubSub()
	TestSubscribeSync(t, pb)
	pb.Close()
}
<|endoftext|>


<commit_before>package restful

import (
	"log"
	"net/http"
	"sort"
	"strings"
)

// CurlyRouter expects Routes with paths that contain zero or more parameters in curly brackets.
type CurlyRouter struct{}

// SelectRoute finds a Route given the input HTTP Request and report if found (ok).
// The HTTP writer is be used to directly communicate non-200 HTTP stati.
func (c CurlyRouter) SelectRoute(
	webServices []*WebService,
	httpWriter http.ResponseWriter,
	httpRequest *http.Request) (selectedService *WebService, selected *Route, ok bool) {

	requestTokens := tokenizePath(httpRequest.URL.Path)

	detectedService := c.detectWebService(requestTokens, webServices)
	if detectedService == nil {
		return nil, nil, false
	}
	candidateRoutes := c.selectRoutes(detectedService, httpWriter, requestTokens)
	if len(candidateRoutes) == 0 {
		return detectedService, nil, false
	}
	selectedRoute := c.detectRoute(candidateRoutes, httpWriter, httpRequest)
	if selectedRoute == nil {
		return detectedService, nil, false
	}
	return detectedService, selectedRoute, true
}

func (c CurlyRouter) selectRoutes(ws *WebService, httpWriter http.ResponseWriter, requestTokens []string) []Route {
	candidates := &sortableCurlyRoutes{[]*curlyRoute{}}
	for _, each := range ws.routes {
		matches, paramCount, staticCount := c.matchesRouteByPathTokens(each.pathParts, requestTokens)
		if matches {
			candidates.add(&curlyRoute{each, paramCount, staticCount}) // TODO make sure Routes() return pointers?
		}
	}
	sort.Sort(sort.Reverse(candidates))
	return candidates.routes()
}

func (c CurlyRouter) matchesRouteByPathTokens(routeTokens, requestTokens []string) (matches bool, paramCount int, staticCount int) {
	if len(routeTokens) != len(requestTokens) {
		return false, 0, 0
	}
	for i, routeToken := range routeTokens {
		requestToken := requestTokens[i]
		if !strings.HasPrefix(routeToken, "{") {
			if requestToken != routeToken {
				return false, 0, 0
			}
			staticCount++
		} else {
			paramCount++
		}
	}
	return true, paramCount, staticCount
}

func (c CurlyRouter) detectRoute(candidateRoutes []Route, httpWriter http.ResponseWriter, httpRequest *http.Request) *Route {
	route, found := RouterJSR311{}.detectRoute(candidateRoutes, httpWriter, httpRequest) // TODO change signature
	if found {
		return route
	} else {
		return nil
	}
}

func (c CurlyRouter) detectWebService(requestTokens []string, webServices []*WebService) *WebService {
	if len(webServices) == 0 {
		return nil
	}
	var best *WebService
	score := -1
	for _, each := range webServices {
		matches, eachScore := c.computeWebserviceScore(requestTokens, each.pathExpr.tokens)
		if matches && (eachScore > score) {
			best = each
			score = eachScore
		}
	}
	return best
}

func (c CurlyRouter) computeWebserviceScore(requestTokens []string, tokens []string) (bool, int) {
	// return whether tokens match and the weighted score of the longest matching consecutive tokens from the beginning

	if len(tokens) > len(requestTokens) {
		return false, 0
	}
	score := 0
	for i := 0; i < len(tokens); i++ {
		each := requestTokens[i]
		other := tokens[i]
		if len(each) == 0 && len(other) == 0 {
			score++
			continue
		}
		if len(other) > 0 && strings.HasPrefix(other, "{") {
			// no empty match
			if len(each) == 0 {
				return false, score
			}
			score += 1
		} else {
			// not a parameter
			if each != other {
				return false, score
			}
			score += (len(tokens) - i) * 10 //fuzzy
		}
	}
	return true, score
}
<commit_msg>remove unused import<commit_after>package restful

import (
	//	"log"
	"net/http"
	"sort"
	"strings"
)

// CurlyRouter expects Routes with paths that contain zero or more parameters in curly brackets.
type CurlyRouter struct{}

// SelectRoute finds a Route given the input HTTP Request and report if found (ok).
// The HTTP writer is be used to directly communicate non-200 HTTP stati.
func (c CurlyRouter) SelectRoute(
	webServices []*WebService,
	httpWriter http.ResponseWriter,
	httpRequest *http.Request) (selectedService *WebService, selected *Route, ok bool) {

	requestTokens := tokenizePath(httpRequest.URL.Path)

	detectedService := c.detectWebService(requestTokens, webServices)
	if detectedService == nil {
		return nil, nil, false
	}
	candidateRoutes := c.selectRoutes(detectedService, httpWriter, requestTokens)
	if len(candidateRoutes) == 0 {
		return detectedService, nil, false
	}
	selectedRoute := c.detectRoute(candidateRoutes, httpWriter, httpRequest)
	if selectedRoute == nil {
		return detectedService, nil, false
	}
	return detectedService, selectedRoute, true
}

func (c CurlyRouter) selectRoutes(ws *WebService, httpWriter http.ResponseWriter, requestTokens []string) []Route {
	candidates := &sortableCurlyRoutes{[]*curlyRoute{}}
	for _, each := range ws.routes {
		matches, paramCount, staticCount := c.matchesRouteByPathTokens(each.pathParts, requestTokens)
		if matches {
			candidates.add(&curlyRoute{each, paramCount, staticCount}) // TODO make sure Routes() return pointers?
		}
	}
	sort.Sort(sort.Reverse(candidates))
	return candidates.routes()
}

func (c CurlyRouter) matchesRouteByPathTokens(routeTokens, requestTokens []string) (matches bool, paramCount int, staticCount int) {
	if len(routeTokens) != len(requestTokens) {
		return false, 0, 0
	}
	for i, routeToken := range routeTokens {
		requestToken := requestTokens[i]
		if !strings.HasPrefix(routeToken, "{") {
			if requestToken != routeToken {
				return false, 0, 0
			}
			staticCount++
		} else {
			paramCount++
		}
	}
	return true, paramCount, staticCount
}

func (c CurlyRouter) detectRoute(candidateRoutes []Route, httpWriter http.ResponseWriter, httpRequest *http.Request) *Route {
	route, found := RouterJSR311{}.detectRoute(candidateRoutes, httpWriter, httpRequest) // TODO change signature
	if found {
		return route
	} else {
		return nil
	}
}

func (c CurlyRouter) detectWebService(requestTokens []string, webServices []*WebService) *WebService {
	if len(webServices) == 0 {
		return nil
	}
	var best *WebService
	score := -1
	for _, each := range webServices {
		matches, eachScore := c.computeWebserviceScore(requestTokens, each.pathExpr.tokens)
		if matches && (eachScore > score) {
			best = each
			score = eachScore
		}
	}
	return best
}

func (c CurlyRouter) computeWebserviceScore(requestTokens []string, tokens []string) (bool, int) {
	// return whether tokens match and the weighted score of the longest matching consecutive tokens from the beginning

	if len(tokens) > len(requestTokens) {
		return false, 0
	}
	score := 0
	for i := 0; i < len(tokens); i++ {
		each := requestTokens[i]
		other := tokens[i]
		if len(each) == 0 && len(other) == 0 {
			score++
			continue
		}
		if len(other) > 0 && strings.HasPrefix(other, "{") {
			// no empty match
			if len(each) == 0 {
				return false, score
			}
			score += 1
		} else {
			// not a parameter
			if each != other {
				return false, score
			}
			score += (len(tokens) - i) * 10 //fuzzy
		}
	}
	return true, score
}
<|endoftext|>


<commit_before>/*
动态加载配置文件, 需要继承DLoad来实现自己的热文件加载类
2016-03-15
@aosen
*/

package goutils

import "sync"

const (
    BUF0 = 0
    BUF1 = 1
)

type DLoad struct {
    buf0   interface{}
    buf1   interface{}
    state  int32 //0            使用0  1 使用1
    locker *sync.Mutex
}

func NewDLoad() *DLoad {
        return &DLoad{
            //默认使用one
            state:  0,
            locker: new(sync.Mutex),
        }
    }

func (self *DLoad) getState() int32 {
    return self.state
}

    //如果state为0 则设置为1 如果为1则设置为0
func (self *DLoad) setState(s int32) {
    self.state = s
}

//动态加载配置文件æ¹法
func (self *DLoad) Load(filename string, fn func(string) (interface{}, error)) error {
    self.locker.Lock()
    defer self.locker.Unlock()
    tmp, err := fn(errfilename)
    if err != nil {
        return err
    }
    if self.getState() == BUF0 {
        self.buf1 = tmp
        self.setState(BUF1)
    } else if self.getState() == BUF1 {
        self.buf0 = tmp
        self.setState(BUF0)
    }
    return nil
}

//获取配置文件中的map
func (self *DLoad) Get() interface{} {
    if selfelf.getState() == BUF0 {
        return self.buf0
    } else if self.getState() == BUF1 {
        return self.buf1
    } else {
        return self.buf0
    }
}
<commit_msg>修复bug<commit_after>/*
动态加载配置文件, 需要继承DLoad来实现自己的热文件加载类
2016-03-15
@aosen
*/

package goutils

import "sync"

const (
    BUF0 = 0
    BUF1 = 1
)

type DLoad struct {
    buf0   interface{}
    buf1   interface{}
    state  int32 //0            使用0  1 使用1
    locker *sync.Mutex
}

func NewDLoad() *DLoad {
        return &DLoad{
            //默认使用one
            state:  0,
            locker: new(sync.Mutex),
        }
    }

func (self *DLoad) getState() int32 {
    return self.state
}

    //如果state为0 则设置为1 如果为1则设置为0
func (self *DLoad) setState(s int32) {
    self.state = s
}

//动态加载配置文件æ¹法
func (self *DLoad) Load(filename string, fn func(string) (interface{}, error)) error {
    self.locker.Lock()
    defer self.locker.Unlock()
    tmp, err := fn(filename)
    if err != nil {
        return err
    }
    if self.getState() == BUF0 {
        self.buf1 = tmp
        self.setState(BUF1)
    } else if self.getState() == BUF1 {
        self.buf0 = tmp
        self.setState(BUF0)
    }
    return nil
}

//获取配置文件中的map
func (self *DLoad) Get() interface{} {
    if self.getState() == BUF0 {
        return self.buf0
    } else if self.getState() == BUF1 {
        return self.buf1
    } else {
        return self.buf0
    }
}
<|endoftext|>


<commit_before>package runner

import (
	"context"
	"io"
	"os"
	"os/exec"
	"runtime"
	"time"

	"go.uber.org/zap"
)

func RunServer(ctx context.Context, r io.Reader, w io.Writer, once bool) {
	zap.L().Info("starting blocking process")

	// a signaller that uses logs to understand when the gamemode is restarting
	// with gmx that means this program can kill the process and restart it.
	restartKiller := make(chan struct{})
	outputWriter := LogParser(restartKiller)

	for {
		err := runBlocking(ctx, restartKiller, r, outputWriter)
		if err != nil {
			if err == context.Canceled {
				break
			}
			zap.L().Info("process exited with error, restarting", zap.Error(err))
		} else {
			zap.L().Info("process exited cleanly, restarting")
		}

		cleanup()

		if once {
			break
		}
	}
}

func runBlocking(parentctx context.Context, restartKiller chan struct{}, in io.Reader, out io.Writer) (err error) {
	ctx, cancel := context.WithCancel(parentctx)
	defer cancel()

	var binary string
	switch runtime.GOOS {
	case "windows":
		binary = "./samp-server.exe"
	case "linux":
		binary = "./samp03svr"
	default:
		panic("unknown OS")
	}

	cmd := exec.CommandContext(ctx, binary)
	cmd.Stdin = in
	r := cmdReader(cmd)
	if err := cmd.Start(); err != nil {
		return err
	}

	go func() {
		// TODO: Handle IO copy errors.
		// nolint:errcheck
		io.Copy(out, r)
	}()

	go func() {
		<-restartKiller
		time.Sleep(time.Second * 5)
		zap.L().Info("internally triggered process restart")
		cancel()
	}()

	return cmd.Wait()
}

func cmdReader(cmd *exec.Cmd) io.Reader {
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil
	}
	return io.MultiReader(stdout, stderr)
}

func cleanup() {
	os.Remove("server_log.txt") //nolint:errcheck
}
<commit_msg>move the restart sleep to the for loop so it happens for all kinds of restarts, not just server-triggered<commit_after>package runner

import (
	"context"
	"io"
	"os"
	"os/exec"
	"runtime"
	"time"

	"go.uber.org/zap"
)

func RunServer(ctx context.Context, r io.Reader, w io.Writer, once bool) {
	zap.L().Info("starting blocking process")

	// a signaller that uses logs to understand when the gamemode is restarting
	// with gmx that means this program can kill the process and restart it.
	restartKiller := make(chan struct{})
	outputWriter := LogParser(restartKiller)

	for {
		err := runBlocking(ctx, restartKiller, r, outputWriter)
		if err != nil {
			if err == context.Canceled {
				break
			}
			zap.L().Info("process exited with error, restarting", zap.Error(err))
		} else {
			zap.L().Info("process exited cleanly, restarting")
		}

		cleanup()

		if once {
			break
		}

		time.Sleep(time.Second * 5)
	}
}

func runBlocking(parentctx context.Context, restartKiller chan struct{}, in io.Reader, out io.Writer) (err error) {
	ctx, cancel := context.WithCancel(parentctx)
	defer cancel()

	var binary string
	switch runtime.GOOS {
	case "windows":
		binary = "./samp-server.exe"
	case "linux":
		binary = "./samp03svr"
	default:
		panic("unknown OS")
	}

	cmd := exec.CommandContext(ctx, binary)
	cmd.Stdin = in
	r := cmdReader(cmd)
	if err := cmd.Start(); err != nil {
		return err
	}

	go func() {
		// TODO: Handle IO copy errors.
		// nolint:errcheck
		io.Copy(out, r)
	}()

	go func() {
		<-restartKiller
		zap.L().Info("internally triggered process restart")
		cancel()
	}()

	return cmd.Wait()
}

func cmdReader(cmd *exec.Cmd) io.Reader {
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil
	}
	return io.MultiReader(stdout, stderr)
}

func cleanup() {
	os.Remove("server_log.txt") //nolint:errcheck
}
<|endoftext|>


<commit_before>// +build binary_log

package zerolog

import (
	"bytes"
	"errors"
	"fmt"
	//	"io/ioutil"
	stdlog "log"
	"time"
)

func ExampleBinaryNew() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Info().Msg("hello world")
	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"info","message":"hello world"}
}

func ExampleLogger_With() {
	dst := bytes.Buffer{}
	log := New(&dst).
		With().
		Str("foo", "bar").
		Logger()

	log.Info().Msg("hello world")
	fmt.Println(decodeIfBinaryToString(dst.Bytes()))

	// Output: {"level":"info","foo":"bar","message":"hello world"}
}

func ExampleLogger_Level() {
	dst := bytes.Buffer{}
	log := New(&dst).Level(WarnLevel)

	log.Info().Msg("filtered out message")
	log.Error().Msg("kept message")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"error","message":"kept message"}
}

func ExampleLogger_Sample() {
	dst := bytes.Buffer{}
	log := New(&dst).Sample(&BasicSampler{N: 2})

	log.Info().Msg("message 1")
	log.Info().Msg("message 2")
	log.Info().Msg("message 3")
	log.Info().Msg("message 4")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"info","message":"message 2"}
	// {"level":"info","message":"message 4"}
}

type LevelNameHook1 struct{}

func (h LevelNameHook1) Run(e *Event, l Level, msg string) {
	if l != NoLevel {
		e.Str("level_name", l.String())
	} else {
		e.Str("level_name", "NoLevel")
	}
}

type MessageHook string

func (h MessageHook) Run(e *Event, l Level, msg string) {
	e.Str("the_message", msg)
}

func ExampleLogger_Hook() {
	var levelNameHook LevelNameHook1
	var messageHook MessageHook = "The message"

	dst := bytes.Buffer{}
	log := New(&dst).Hook(levelNameHook).Hook(messageHook)

	log.Info().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"info","level_name":"info","the_message":"hello world","message":"hello world"}
}

func ExampleLogger_Print() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Print("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"debug","message":"hello world"}
}

func ExampleLogger_Printf() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Printf("hello %s", "world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"debug","message":"hello world"}
}

func ExampleLogger_Debug() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Debug().
		Str("foo", "bar").
		Int("n", 123).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"debug","foo":"bar","n":123,"message":"hello world"}
}

func ExampleLogger_Info() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Info().
		Str("foo", "bar").
		Int("n", 123).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"info","foo":"bar","n":123,"message":"hello world"}
}

func ExampleLogger_Warn() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Warn().
		Str("foo", "bar").
		Msg("a warning message")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"warn","foo":"bar","message":"a warning message"}
}

func ExampleLogger_Error() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Error().
		Err(errors.New("some error")).
		Msg("error doing something")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"error","error":"some error","message":"error doing something"}
}

func ExampleLogger_WithLevel() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.WithLevel(InfoLevel).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"info","message":"hello world"}
}

func ExampleLogger_Write() {
	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Logger()

	stdlog.SetFlags(0)
	stdlog.SetOutput(log)

	stdlog.Print("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","message":"hello world"}
}

func ExampleLogger_Log() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		Str("bar", "baz").
		Msg("")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","bar":"baz"}
}

func ExampleEvent_Dict() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		Dict("dict", Dict().
			Str("bar", "baz").
			Int("n", 1),
		).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","dict":{"bar":"baz","n":1},"message":"hello world"}
}

type User struct {
	Name    string
	Age     int
	Created time.Time
}

func (u User) MarshalZerologObject(e *Event) {
	e.Str("name", u.Name).
		Int("age", u.Age).
		Time("created", u.Created)
}

type Users []User

func (uu Users) MarshalZerologArray(a *Array) {
	for _, u := range uu {
		a.Object(u)
	}
}

func ExampleEvent_Array() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		Array("array", Arr().
			Str("baz").
			Int(1),
		).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","array":["baz",1],"message":"hello world"}
}

func ExampleEvent_Array_object() {
	dst := bytes.Buffer{}
	log := New(&dst)

	// Users implements LogArrayMarshaler
	u := Users{
		User{"John", 35, time.Time{}},
		User{"Bob", 55, time.Time{}},
	}

	log.Log().
		Str("foo", "bar").
		Array("users", u).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","users":[{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},{"name":"Bob","age":55,"created":"0001-01-01T00:00:00Z"}],"message":"hello world"}
}

func ExampleEvent_Object() {
	dst := bytes.Buffer{}
	log := New(&dst)

	// User implements LogObjectMarshaler
	u := User{"John", 35, time.Time{}}

	log.Log().
		Str("foo", "bar").
		Object("user", u).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","user":{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},"message":"hello world"}
}

func ExampleEvent_EmbedObject() {
	price := Price{val: 6449, prec: 2, unit: "$"}

	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		EmbedObject(price).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","price":"$64.49","message":"hello world"}
}

func ExampleEvent_Interface() {
	dst := bytes.Buffer{}
	log := New(&dst)

	obj := struct {
		Name string `json:"name"`
	}{
		Name: "john",
	}

	log.Log().
		Str("foo", "bar").
		Interface("obj", obj).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","obj":{"name":"john"},"message":"hello world"}
}

func ExampleEvent_Dur() {
	d := time.Duration(10 * time.Second)

	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		Dur("dur", d).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","dur":10000,"message":"hello world"}
}

func ExampleEvent_Durs() {
	d := []time.Duration{
		time.Duration(10 * time.Second),
		time.Duration(20 * time.Second),
	}

	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		Durs("durs", d).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"}
}

func ExampleContext_Dict() {
	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Dict("dict", Dict().
			Str("bar", "baz").
			Int("n", 1),
		).Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","dict":{"bar":"baz","n":1},"message":"hello world"}
}

func ExampleContext_Array() {
	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Array("array", Arr().
			Str("baz").
			Int(1),
		).Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","array":["baz",1],"message":"hello world"}
}

func ExampleContext_Array_object() {
	// Users implements LogArrayMarshaler
	u := Users{
		User{"John", 35, time.Time{}},
		User{"Bob", 55, time.Time{}},
	}

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Array("users", u).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","users":[{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},{"name":"Bob","age":55,"created":"0001-01-01T00:00:00Z"}],"message":"hello world"}
}

type Price struct {
	val  uint64
	prec int
	unit string
}

func (p Price) MarshalZerologObject(e *Event) {
	denom := uint64(1)
	for i := 0; i < p.prec; i++ {
		denom *= 10
	}
	result := []byte(p.unit)
	result = append(result, fmt.Sprintf("%d.%d", p.val/denom, p.val%denom)...)
	e.Str("price", string(result))
}

func ExampleContext_EmbedObject() {
	price := Price{val: 6449, prec: 2, unit: "$"}

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		EmbedObject(price).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","price":"$64.49","message":"hello world"}
}
func ExampleContext_Object() {
	// User implements LogObjectMarshaler
	u := User{"John", 35, time.Time{}}

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Object("user", u).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","user":{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},"message":"hello world"}
}

func ExampleContext_Interface() {
	obj := struct {
		Name string `json:"name"`
	}{
		Name: "john",
	}

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Interface("obj", obj).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","obj":{"name":"john"},"message":"hello world"}
}

func ExampleContext_Dur() {
	d := time.Duration(10 * time.Second)

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Dur("dur", d).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","dur":10000,"message":"hello world"}
}

func ExampleContext_Durs() {
	d := []time.Duration{
		time.Duration(10 * time.Second),
		time.Duration(20 * time.Second),
	}

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Durs("durs", d).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"}
}
<commit_msg>Fix binary test too<commit_after>// +build binary_log

package zerolog

import (
	"bytes"
	"errors"
	"fmt"

	//	"io/ioutil"
	stdlog "log"
	"time"
)

func ExampleBinaryNew() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Info().Msg("hello world")
	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"info","message":"hello world"}
}

func ExampleLogger_With() {
	dst := bytes.Buffer{}
	log := New(&dst).
		With().
		Str("foo", "bar").
		Logger()

	log.Info().Msg("hello world")
	fmt.Println(decodeIfBinaryToString(dst.Bytes()))

	// Output: {"level":"info","foo":"bar","message":"hello world"}
}

func ExampleLogger_Level() {
	dst := bytes.Buffer{}
	log := New(&dst).Level(WarnLevel)

	log.Info().Msg("filtered out message")
	log.Error().Msg("kept message")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"error","message":"kept message"}
}

func ExampleLogger_Sample() {
	dst := bytes.Buffer{}
	log := New(&dst).Sample(&BasicSampler{N: 2})

	log.Info().Msg("message 1")
	log.Info().Msg("message 2")
	log.Info().Msg("message 3")
	log.Info().Msg("message 4")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"info","message":"message 1"}
	// {"level":"info","message":"message 3"}
}

type LevelNameHook1 struct{}

func (h LevelNameHook1) Run(e *Event, l Level, msg string) {
	if l != NoLevel {
		e.Str("level_name", l.String())
	} else {
		e.Str("level_name", "NoLevel")
	}
}

type MessageHook string

func (h MessageHook) Run(e *Event, l Level, msg string) {
	e.Str("the_message", msg)
}

func ExampleLogger_Hook() {
	var levelNameHook LevelNameHook1
	var messageHook MessageHook = "The message"

	dst := bytes.Buffer{}
	log := New(&dst).Hook(levelNameHook).Hook(messageHook)

	log.Info().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"info","level_name":"info","the_message":"hello world","message":"hello world"}
}

func ExampleLogger_Print() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Print("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"debug","message":"hello world"}
}

func ExampleLogger_Printf() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Printf("hello %s", "world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"debug","message":"hello world"}
}

func ExampleLogger_Debug() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Debug().
		Str("foo", "bar").
		Int("n", 123).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"debug","foo":"bar","n":123,"message":"hello world"}
}

func ExampleLogger_Info() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Info().
		Str("foo", "bar").
		Int("n", 123).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"info","foo":"bar","n":123,"message":"hello world"}
}

func ExampleLogger_Warn() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Warn().
		Str("foo", "bar").
		Msg("a warning message")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"warn","foo":"bar","message":"a warning message"}
}

func ExampleLogger_Error() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Error().
		Err(errors.New("some error")).
		Msg("error doing something")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"error","error":"some error","message":"error doing something"}
}

func ExampleLogger_WithLevel() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.WithLevel(InfoLevel).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"level":"info","message":"hello world"}
}

func ExampleLogger_Write() {
	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Logger()

	stdlog.SetFlags(0)
	stdlog.SetOutput(log)

	stdlog.Print("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","message":"hello world"}
}

func ExampleLogger_Log() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		Str("bar", "baz").
		Msg("")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","bar":"baz"}
}

func ExampleEvent_Dict() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		Dict("dict", Dict().
			Str("bar", "baz").
			Int("n", 1),
		).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","dict":{"bar":"baz","n":1},"message":"hello world"}
}

type User struct {
	Name    string
	Age     int
	Created time.Time
}

func (u User) MarshalZerologObject(e *Event) {
	e.Str("name", u.Name).
		Int("age", u.Age).
		Time("created", u.Created)
}

type Users []User

func (uu Users) MarshalZerologArray(a *Array) {
	for _, u := range uu {
		a.Object(u)
	}
}

func ExampleEvent_Array() {
	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		Array("array", Arr().
			Str("baz").
			Int(1),
		).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","array":["baz",1],"message":"hello world"}
}

func ExampleEvent_Array_object() {
	dst := bytes.Buffer{}
	log := New(&dst)

	// Users implements LogArrayMarshaler
	u := Users{
		User{"John", 35, time.Time{}},
		User{"Bob", 55, time.Time{}},
	}

	log.Log().
		Str("foo", "bar").
		Array("users", u).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","users":[{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},{"name":"Bob","age":55,"created":"0001-01-01T00:00:00Z"}],"message":"hello world"}
}

func ExampleEvent_Object() {
	dst := bytes.Buffer{}
	log := New(&dst)

	// User implements LogObjectMarshaler
	u := User{"John", 35, time.Time{}}

	log.Log().
		Str("foo", "bar").
		Object("user", u).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","user":{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},"message":"hello world"}
}

func ExampleEvent_EmbedObject() {
	price := Price{val: 6449, prec: 2, unit: "$"}

	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		EmbedObject(price).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","price":"$64.49","message":"hello world"}
}

func ExampleEvent_Interface() {
	dst := bytes.Buffer{}
	log := New(&dst)

	obj := struct {
		Name string `json:"name"`
	}{
		Name: "john",
	}

	log.Log().
		Str("foo", "bar").
		Interface("obj", obj).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","obj":{"name":"john"},"message":"hello world"}
}

func ExampleEvent_Dur() {
	d := time.Duration(10 * time.Second)

	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		Dur("dur", d).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","dur":10000,"message":"hello world"}
}

func ExampleEvent_Durs() {
	d := []time.Duration{
		time.Duration(10 * time.Second),
		time.Duration(20 * time.Second),
	}

	dst := bytes.Buffer{}
	log := New(&dst)

	log.Log().
		Str("foo", "bar").
		Durs("durs", d).
		Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"}
}

func ExampleContext_Dict() {
	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Dict("dict", Dict().
			Str("bar", "baz").
			Int("n", 1),
		).Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","dict":{"bar":"baz","n":1},"message":"hello world"}
}

func ExampleContext_Array() {
	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Array("array", Arr().
			Str("baz").
			Int(1),
		).Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","array":["baz",1],"message":"hello world"}
}

func ExampleContext_Array_object() {
	// Users implements LogArrayMarshaler
	u := Users{
		User{"John", 35, time.Time{}},
		User{"Bob", 55, time.Time{}},
	}

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Array("users", u).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","users":[{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},{"name":"Bob","age":55,"created":"0001-01-01T00:00:00Z"}],"message":"hello world"}
}

type Price struct {
	val  uint64
	prec int
	unit string
}

func (p Price) MarshalZerologObject(e *Event) {
	denom := uint64(1)
	for i := 0; i < p.prec; i++ {
		denom *= 10
	}
	result := []byte(p.unit)
	result = append(result, fmt.Sprintf("%d.%d", p.val/denom, p.val%denom)...)
	e.Str("price", string(result))
}

func ExampleContext_EmbedObject() {
	price := Price{val: 6449, prec: 2, unit: "$"}

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		EmbedObject(price).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","price":"$64.49","message":"hello world"}
}
func ExampleContext_Object() {
	// User implements LogObjectMarshaler
	u := User{"John", 35, time.Time{}}

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Object("user", u).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","user":{"name":"John","age":35,"created":"0001-01-01T00:00:00Z"},"message":"hello world"}
}

func ExampleContext_Interface() {
	obj := struct {
		Name string `json:"name"`
	}{
		Name: "john",
	}

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Interface("obj", obj).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","obj":{"name":"john"},"message":"hello world"}
}

func ExampleContext_Dur() {
	d := time.Duration(10 * time.Second)

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Dur("dur", d).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","dur":10000,"message":"hello world"}
}

func ExampleContext_Durs() {
	d := []time.Duration{
		time.Duration(10 * time.Second),
		time.Duration(20 * time.Second),
	}

	dst := bytes.Buffer{}
	log := New(&dst).With().
		Str("foo", "bar").
		Durs("durs", d).
		Logger()

	log.Log().Msg("hello world")

	fmt.Println(decodeIfBinaryToString(dst.Bytes()))
	// Output: {"foo":"bar","durs":[10000,20000],"message":"hello world"}
}
<|endoftext|>


<commit_before>/*
This package implements a Furan RPC client that can be directly imported by other Go programs.
It uses Consul service discovery to pick a random node.
*/

package rpcclient

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"google.golang.org/grpc/codes"

	"google.golang.org/grpc"

	"golang.org/x/net/context"

	consul "github.com/hashicorp/consul/api"
)

const (
	connTimeoutSecs = 30
)

// ErrCanceled is an error returned by Build() when the build has been canceled
var ErrCanceled = errors.New("build canceled")

//go:generate stringer -type=NodeSelectionStrategy

//NodeSelectionStrategy enumerates the ways that rpcclient will use to pick a node
type NodeSelectionStrategy int

const (
	RandomNodeSelection NodeSelectionStrategy = iota // Choose a random node
	NetworkProximity                                 // Pick the closest node as determined by Consul
)

// ImageBuildPusher describes an object capable of building and pushing container images
type ImageBuildPusher interface {
	Build(context.Context, chan *BuildEvent, *BuildRequest) (string, error)
}

// FuranClient is an object which issues remote RPC calls to a Furan server
type FuranClient struct {
	n      furanNode
	logger *log.Logger
}

// DiscoveryOptions describes the options for determining the Furan node to use
// for the client.
type DiscoveryOptions struct {
	UseConsul         bool                  // Whether to use Consul service discovery
	ConsulAddr        string                // Consul address to use (defaults to '127.0.0.1:8500')
	SelectionStrategy NodeSelectionStrategy // If UseConsul is true, this specifies the strategy for node selection
	ServiceName       string                // Required if UseConsul is true
	NodeList          []string              // Required if UseConsul is false. Nodes in the format "{host}:{port}". A random host will be used if len(NodeList) > 1
}

type furanNode struct {
	addr string
	port int
}

// NewFuranClient takes a Consul service name and returns a client which connects
// to a randomly chosen Furan host and uses the optional logger
func NewFuranClient(opts *DiscoveryOptions, logger *log.Logger) (*FuranClient, error) {
	fc := &FuranClient{}
	if logger == nil {
		fc.logger = log.New(os.Stderr, "", log.LstdFlags)
	} else {
		fc.logger = logger
	}
	if opts.UseConsul {
		if opts.ServiceName == "" {
			return nil, fmt.Errorf("ConsulService is required if UseConsul is true")
		}
	} else {
		if len(opts.NodeList) == 0 {
			return nil, fmt.Errorf("non-empty NodeList is required if UseConsul is false")
		}
		opts.SelectionStrategy = RandomNodeSelection
	}
	err := fc.init(opts)
	return fc, err
}

func (fc *FuranClient) init(opts *DiscoveryOptions) error {
	nodes := []furanNode{}
	if opts.UseConsul {
		cc := consul.DefaultConfig()
		if opts.ConsulAddr != "" {
			cc.Address = opts.ConsulAddr
		}
		c, err := consul.NewClient(cc)
		if err != nil {
			return err
		}
		qopts := &consul.QueryOptions{}
		if opts.SelectionStrategy == NetworkProximity {
			qopts.Near = "_agent"
		}
		fc.logger.Printf("connecting to Consul on %v", cc.Address)
		se, _, err := c.Health().Service(opts.ServiceName, "", true, qopts)
		if err != nil {
			return err
		}
		if len(se) == 0 {
			return fmt.Errorf("no Furan hosts found via Consul")
		}
		fc.logger.Printf("found %v Furan hosts", len(se))
		for _, s := range se {
			n := furanNode{
				addr: s.Node.Address,
				port: s.Service.Port,
			}
			nodes = append(nodes, n)
		}
	} else {
		for _, s := range opts.NodeList {
			ns := strings.Split(s, ":")
			if len(ns) != 2 {
				return fmt.Errorf("malformed node: %v", s)
			}
			p, err := strconv.Atoi(ns[1])
			if err != nil {
				return fmt.Errorf("malformed port: %v", s)
			}
			n := furanNode{
				addr: ns[0],
				port: p,
			}
			nodes = append(nodes, n)
		}
	}
	if opts.SelectionStrategy == RandomNodeSelection && len(nodes) > 1 {
		i, err := randomRange(len(nodes)) // Random node
		if err != nil {
			return err
		}
		fc.n = nodes[i]
	} else {
		fc.n = nodes[0]
	}
	fc.logger.Printf("using node %v", fc.n.addr)
	return nil
}

func (fc FuranClient) validateBuildRequest(req *BuildRequest) error {
	if req.Build.GithubRepo == "" {
		return fmt.Errorf("Build.GithubRepo is required")
	}
	if req.Build.Ref == "" {
		return fmt.Errorf("Build.Ref is required")
	}
	if len(req.Build.Tags) == 0 {
		return fmt.Errorf("at least one tag is required") // no tags causes datalayer failure
	}
	if req.Push.Registry.Repo == "" &&
		req.Push.S3.Region == "" &&
		req.Push.S3.Bucket == "" &&
		req.Push.S3.KeyPrefix == "" {
		return fmt.Errorf("you must specify either a Docker registry or S3 region/bucket/key-prefix as a push target")
	}
	return nil
}

func (fc FuranClient) rpcerr(err error, msg string, params ...interface{}) error {
	code := grpc.Code(err)
	msg = fmt.Sprintf(msg, params...)
	return fmt.Errorf("rpc error: %v: %v: %v", msg, code.String(), err)
}

// Build starts and monitors a build synchronously, sending BuildEvents to out and returning the build ID when completed, or error.
// Returns an error if there was an RPC error or if the build/push fails
// You must read from out (or provide a sufficiently buffered channel) to prevent Build from blocking forever
func (fc FuranClient) Build(ctx context.Context, out chan *BuildEvent, req *BuildRequest) (string, error) {
	err := fc.validateBuildRequest(req)
	if err != nil {
		return "", err
	}

	remoteHost := fmt.Sprintf("%v:%v", fc.n.addr, fc.n.port)

	fc.logger.Printf("connecting to %v", remoteHost)

	conn, err := grpc.Dial(remoteHost, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithTimeout(connTimeoutSecs*time.Second))
	if err != nil {
		return "", fmt.Errorf("error connecting to remote host: %v", err)
	}
	defer conn.Close()

	c := NewFuranExecutorClient(conn)

	if ctx.Err() == context.Canceled {
		return "", ErrCanceled
	}

	fc.logger.Printf("triggering build")
	// use a new context so StartBuild won't get cancelled if
	// ctx is cancelled
	resp, err := c.StartBuild(context.Background(), req)
	if err != nil {
		return "", fc.rpcerr(err, "StartBuild")
	}

	mreq := BuildStatusRequest{
		BuildId: resp.BuildId,
	}

	fc.logger.Printf("monitoring build: %v", resp.BuildId)
	stream, err := c.MonitorBuild(ctx, &mreq)
	if err != nil {

		if grpc.Code(err) == codes.Canceled || err == context.Canceled {
			creq := BuildCancelRequest{
				BuildId: resp.BuildId,
			}

			c.CancelBuild(context.Background(), &creq) // best effort but doesn't matter if it fails
			return resp.BuildId, ErrCanceled
		}

		return resp.BuildId, fc.rpcerr(err, "MonitorBuild")
	}

	for {
		event, err := stream.Recv()
		if err != nil {
			if err == io.EOF {
				break
			}

			if grpc.Code(err) == codes.Canceled || err == context.Canceled {
				creq := BuildCancelRequest{
					BuildId: resp.BuildId,
				}

				c.CancelBuild(context.Background(), &creq) // best effort but doesn't matter if it fails
				return resp.BuildId, ErrCanceled
			}

			return resp.BuildId, fc.rpcerr(err, "stream.Recv")
		}
		out <- event
		if event.EventError.IsError {
			return resp.BuildId, fmt.Errorf("build error: %v", event.Message)
		}
	}

	return resp.BuildId, nil
}
<commit_msg>datadog apm: update client to use unary interceptor<commit_after>/*
This package implements a Furan RPC client that can be directly imported by other Go programs.
It uses Consul service discovery to pick a random node.
*/

package rpcclient

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"

	"google.golang.org/grpc/codes"

	consul "github.com/hashicorp/consul/api"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	grpctrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/google.golang.org/grpc"
)

const (
	connTimeoutSecs = 30
)

// ErrCanceled is an error returned by Build() when the build has been canceled
var ErrCanceled = errors.New("build canceled")

//go:generate stringer -type=NodeSelectionStrategy

//NodeSelectionStrategy enumerates the ways that rpcclient will use to pick a node
type NodeSelectionStrategy int

const (
	RandomNodeSelection NodeSelectionStrategy = iota // Choose a random node
	NetworkProximity                                 // Pick the closest node as determined by Consul
)

// ImageBuildPusher describes an object capable of building and pushing container images
type ImageBuildPusher interface {
	Build(context.Context, chan *BuildEvent, *BuildRequest) (string, error)
}

// FuranClient is an object which issues remote RPC calls to a Furan server
type FuranClient struct {
	n      furanNode
	logger *log.Logger
	ddName string
}

// DiscoveryOptions describes the options for determining the Furan node to use
// for the client.
type DiscoveryOptions struct {
	UseConsul         bool                  // Whether to use Consul service discovery
	ConsulAddr        string                // Consul address to use (defaults to '127.0.0.1:8500')
	SelectionStrategy NodeSelectionStrategy // If UseConsul is true, this specifies the strategy for node selection
	ServiceName       string                // Required if UseConsul is true
	NodeList          []string              // Required if UseConsul is false. Nodes in the format "{host}:{port}". A random host will be used if len(NodeList) > 1
}

type furanNode struct {
	addr string
	port int
}

// NewFuranClient takes a Consul service name and returns a client which connects
// to a randomly chosen Furan host and uses the optional logger
func NewFuranClient(opts *DiscoveryOptions, logger *log.Logger, datadogServiceName string) (*FuranClient, error) {
	fc := &FuranClient{}
	fc.ddName = datadogServiceName
	if logger == nil {
		fc.logger = log.New(os.Stderr, "", log.LstdFlags)
	} else {
		fc.logger = logger
	}
	if opts.UseConsul {
		if opts.ServiceName == "" {
			return nil, fmt.Errorf("ConsulService is required if UseConsul is true")
		}
	} else {
		if len(opts.NodeList) == 0 {
			return nil, fmt.Errorf("non-empty NodeList is required if UseConsul is false")
		}
		opts.SelectionStrategy = RandomNodeSelection
	}
	err := fc.init(opts)
	return fc, err
}

func (fc *FuranClient) init(opts *DiscoveryOptions) error {
	nodes := []furanNode{}
	if opts.UseConsul {
		cc := consul.DefaultConfig()
		if opts.ConsulAddr != "" {
			cc.Address = opts.ConsulAddr
		}
		c, err := consul.NewClient(cc)
		if err != nil {
			return err
		}
		qopts := &consul.QueryOptions{}
		if opts.SelectionStrategy == NetworkProximity {
			qopts.Near = "_agent"
		}
		fc.logger.Printf("connecting to Consul on %v", cc.Address)
		se, _, err := c.Health().Service(opts.ServiceName, "", true, qopts)
		if err != nil {
			return err
		}
		if len(se) == 0 {
			return fmt.Errorf("no Furan hosts found via Consul")
		}
		fc.logger.Printf("found %v Furan hosts", len(se))
		for _, s := range se {
			n := furanNode{
				addr: s.Node.Address,
				port: s.Service.Port,
			}
			nodes = append(nodes, n)
		}
	} else {
		for _, s := range opts.NodeList {
			ns := strings.Split(s, ":")
			if len(ns) != 2 {
				return fmt.Errorf("malformed node: %v", s)
			}
			p, err := strconv.Atoi(ns[1])
			if err != nil {
				return fmt.Errorf("malformed port: %v", s)
			}
			n := furanNode{
				addr: ns[0],
				port: p,
			}
			nodes = append(nodes, n)
		}
	}
	if opts.SelectionStrategy == RandomNodeSelection && len(nodes) > 1 {
		i, err := randomRange(len(nodes)) // Random node
		if err != nil {
			return err
		}
		fc.n = nodes[i]
	} else {
		fc.n = nodes[0]
	}
	fc.logger.Printf("using node %v", fc.n.addr)
	return nil
}

func (fc FuranClient) validateBuildRequest(req *BuildRequest) error {
	if req.Build.GithubRepo == "" {
		return fmt.Errorf("Build.GithubRepo is required")
	}
	if req.Build.Ref == "" {
		return fmt.Errorf("Build.Ref is required")
	}
	if len(req.Build.Tags) == 0 {
		return fmt.Errorf("at least one tag is required") // no tags causes datalayer failure
	}
	if req.Push.Registry.Repo == "" &&
		req.Push.S3.Region == "" &&
		req.Push.S3.Bucket == "" &&
		req.Push.S3.KeyPrefix == "" {
		return fmt.Errorf("you must specify either a Docker registry or S3 region/bucket/key-prefix as a push target")
	}
	return nil
}

func (fc FuranClient) rpcerr(err error, msg string, params ...interface{}) error {
	code := grpc.Code(err)
	msg = fmt.Sprintf(msg, params...)
	return fmt.Errorf("rpc error: %v: %v: %v", msg, code.String(), err)
}

// Build starts and monitors a build synchronously, sending BuildEvents to out and returning the build ID when completed, or error.
// Returns an error if there was an RPC error or if the build/push fails
// You must read from out (or provide a sufficiently buffered channel) to prevent Build from blocking forever
func (fc FuranClient) Build(ctx context.Context, out chan *BuildEvent, req *BuildRequest) (_ string, err error) {
	parentSpan, _ := tracer.StartSpanFromContext(ctx, "furan_client.build")
	defer func() {
		parentSpan.Finish(tracer.WithError(err))
	}()
	err = fc.validateBuildRequest(req)
	if err != nil {
		return "", err
	}

	remoteHost := fmt.Sprintf("%v:%v", fc.n.addr, fc.n.port)

	fc.logger.Printf("connecting to %v", remoteHost)
	i := grpctrace.UnaryClientInterceptor(grpctrace.WithServiceName(fc.ddName))

	conn, err := grpc.Dial(remoteHost, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithTimeout(connTimeoutSecs*time.Second), grpc.WithUnaryInterceptor(i))
	if err != nil {
		return "", fmt.Errorf("error connecting to remote host: %v", err)
	}
	defer conn.Close()

	c := NewFuranExecutorClient(conn)

	if ctx.Err() == context.Canceled {
		return "", ErrCanceled
	}

	fc.logger.Printf("triggering build")
	// use a new context so StartBuild won't get cancelled if
	// ctx is cancelled
	buildContext := tracer.ContextWithSpan(context.Background(), parentSpan)
	resp, err := c.StartBuild(buildContext, req)
	if err != nil {
		return "", fc.rpcerr(err, "StartBuild")
	}

	mreq := BuildStatusRequest{
		BuildId: resp.BuildId,
	}

	fc.logger.Printf("monitoring build: %v", resp.BuildId)
	stream, err := c.MonitorBuild(ctx, &mreq)
	if err != nil {

		if grpc.Code(err) == codes.Canceled || err == context.Canceled {
			creq := BuildCancelRequest{
				BuildId: resp.BuildId,
			}

			c.CancelBuild(context.Background(), &creq) // best effort but doesn't matter if it fails
			return resp.BuildId, ErrCanceled
		}

		return resp.BuildId, fc.rpcerr(err, "MonitorBuild")
	}

	for {
		event, err := stream.Recv()
		if err != nil {
			if err == io.EOF {
				break
			}

			if grpc.Code(err) == codes.Canceled || err == context.Canceled {
				creq := BuildCancelRequest{
					BuildId: resp.BuildId,
				}

				c.CancelBuild(context.Background(), &creq) // best effort but doesn't matter if it fails
				return resp.BuildId, ErrCanceled
			}

			return resp.BuildId, fc.rpcerr(err, "stream.Recv")
		}
		out <- event
		if event.EventError.IsError {
			return resp.BuildId, fmt.Errorf("build error: %v", event.Message)
		}
	}

	return resp.BuildId, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package rpctest

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcrpcclient"
	"github.com/btcsuite/btcutil"
)

const (
	// These constants define the minimum and maximum p2p and rpc port
	// numbers used by a test harness.  The min port is inclusive while the
	// max port is exclusive.
	minPeerPort = 10000
	maxPeerPort = 35000
	minRPCPort  = maxPeerPort
	maxRPCPort  = 60000
)

var (
	// current number of active test nodes.
	numTestInstances = 0

	// processID is the process ID of the current running process.  It is
	// used to calculate ports based upon it when launching an rpc
	// harnesses.  The intent is to allow multiple process to run in
	// parallel without port collisions.
	//
	// It should be noted however that there is still some small probability
	// that there will be port collisions either due to other processes
	// running or simply due to the stars aligning on the process IDs.
	processID = os.Getppid()

	// testInstances is a private package-level slice used to keep track of
	// all active test harnesses. This global can be used to perform
	// various "joins", shutdown several active harnesses after a test,
	// etc.
	testInstances = make(map[string]*Harness)

	// Used to protest concurrent access to above declared variables.
	harnessStateMtx sync.RWMutex
)

// HarnessTestCase represents a test-case which utilizes an instance of the
// Harness to exercise functionality.
type HarnessTestCase func(r *Harness, t *testing.T)

// Harness fully encapsulates an active btcd process to provide a unified
// platform for creating rpc driven integration tests involving btcd. The
// active btcd node will typically be run in simnet mode in order to allow for
// easy generation of test blockchains.  The active btcd process is fully
// managed by Harness, which handles the necessary initialization, and teardown
// of the process along with any temporary directories created as a result.
// Multiple Harness instances may be run concurrently, in order to allow for
// testing complex scenarios involving multiple nodes. The harness also
// includes an in-memory wallet to streamline various classes of tests.
type Harness struct {
	// ActiveNet is the parameters of the blockchain the Harness belongs
	// to.
	ActiveNet *chaincfg.Params

	Node     *btcrpcclient.Client
	node     *node
	handlers *btcrpcclient.NotificationHandlers

	wallet *memWallet

	testNodeDir    string
	maxConnRetries int
	nodeNum        int

	sync.Mutex
}

// New creates and initializes new instance of the rpc test harness.
// Optionally, websocket handlers and a specified configuration may be passed.
// In the case that a nil config is passed, a default configuration will be
// used.
//
// NOTE: This function is safe for concurrent access.
func New(activeNet *chaincfg.Params, handlers *btcrpcclient.NotificationHandlers,
	extraArgs []string) (*Harness, error) {

	harnessStateMtx.Lock()
	defer harnessStateMtx.Unlock()

	harnessID := strconv.Itoa(int(numTestInstances))
	nodeTestData, err := ioutil.TempDir("", "rpctest-"+harnessID)
	if err != nil {
		return nil, err
	}

	certFile := filepath.Join(nodeTestData, "rpc.cert")
	keyFile := filepath.Join(nodeTestData, "rpc.key")
	if err := genCertPair(certFile, keyFile); err != nil {
		return nil, err
	}

	wallet, err := newMemWallet(activeNet, uint32(numTestInstances))
	if err != nil {
		return nil, err
	}

	miningAddr := fmt.Sprintf("--miningaddr=%s", wallet.coinbaseAddr)
	extraArgs = append(extraArgs, miningAddr)

	config, err := newConfig("rpctest", certFile, keyFile, extraArgs)
	if err != nil {
		return nil, err
	}

	// Generate p2p+rpc listening addresses.
	config.listen, config.rpcListen = generateListeningAddresses()

	// Create the testing node bounded to the simnet.
	node, err := newNode(config, nodeTestData)
	if err != nil {
		return nil, err
	}

	nodeNum := numTestInstances
	numTestInstances++

	if handlers == nil {
		handlers = &btcrpcclient.NotificationHandlers{}
	}

	// If a handler for the OnBlockConnected/OnBlockDisconnected callback
	// has already been set, then we create a wrapper callback which
	// executes both the currently registered callback, and the mem
	// wallet's callback.
	if handlers.OnBlockConnected != nil {
		obc := handlers.OnBlockConnected
		handlers.OnBlockConnected = func(hash *chainhash.Hash, height int32, t time.Time) {
			wallet.IngestBlock(hash, height, t)
			obc(hash, height, t)
		}
	} else {
		// Otherwise, we can claim the callback ourselves.
		handlers.OnBlockConnected = wallet.IngestBlock
	}
	if handlers.OnBlockDisconnected != nil {
		obd := handlers.OnBlockConnected
		handlers.OnBlockDisconnected = func(hash *chainhash.Hash, height int32, t time.Time) {
			wallet.UnwindBlock(hash, height, t)
			obd(hash, height, t)
		}
	} else {
		handlers.OnBlockDisconnected = wallet.UnwindBlock
	}

	h := &Harness{
		handlers:       handlers,
		node:           node,
		maxConnRetries: 20,
		testNodeDir:    nodeTestData,
		ActiveNet:      activeNet,
		nodeNum:        nodeNum,
		wallet:         wallet,
	}

	// Track this newly created test instance within the package level
	// global map of all active test instances.
	testInstances[h.testNodeDir] = h

	return h, nil
}

// SetUp initializes the rpc test state. Initialization includes: starting up a
// simnet node, creating a websockets client and connecting to the started
// node, and finally: optionally generating and submitting a testchain with a
// configurable number of mature coinbase outputs coinbase outputs.
//
// NOTE: This method and TearDown should always be called from the same
// goroutine as they are not concurrent safe.
func (h *Harness) SetUp(createTestChain bool, numMatureOutputs uint32) error {
	// Start the btcd node itself. This spawns a new process which will be
	// managed
	if err := h.node.start(); err != nil {
		return err
	}
	if err := h.connectRPCClient(); err != nil {
		return err
	}

	h.wallet.Start()

	// Ensure the btcd properly dispatches our registered call-back for
	// each new block. Otherwise, the memWallet won't function properly.
	if err := h.Node.NotifyBlocks(); err != nil {
		return err
	}

	// Create a test chain with the desired number of mature coinbase
	// outputs.
	if createTestChain && numMatureOutputs != 0 {
		numToGenerate := (uint32(h.ActiveNet.CoinbaseMaturity) +
			numMatureOutputs)
		_, err := h.Node.Generate(numToGenerate)
		if err != nil {
			return err
		}
	}

	// Block until the wallet has fully synced up to the tip of the main
	// chain.
	_, height, err := h.Node.GetBestBlock()
	if err != nil {
		return err
	}
	ticker := time.NewTicker(time.Millisecond * 100)
out:
	for {
		select {
		case <-ticker.C:
			walletHeight := h.wallet.SyncedHeight()
			if walletHeight == height {
				break out
			}
		}
	}

	return nil
}

// TearDown stops the running rpc test instance. All created processes are
// killed, and temporary directories removed.
//
// NOTE: This method and SetUp should always be called from the same goroutine
// as they are not concurrent safe.
func (h *Harness) TearDown() error {
	if h.Node != nil {
		h.Node.Shutdown()
	}

	if err := h.node.shutdown(); err != nil {
		return err
	}

	if err := os.RemoveAll(h.testNodeDir); err != nil {
		return err
	}

	delete(testInstances, h.testNodeDir)

	return nil
}

// connectRPCClient attempts to establish an RPC connection to the created btcd
// process belonging to this Harness instance. If the initial connection
// attempt fails, this function will retry h.maxConnRetries times, backing off
// the time between subsequent attempts. If after h.maxConnRetries attempts,
// we're not able to establish a connection, this function returns with an
// error.
func (h *Harness) connectRPCClient() error {
	var client *btcrpcclient.Client
	var err error

	rpcConf := h.node.config.rpcConnConfig()
	for i := 0; i < h.maxConnRetries; i++ {
		if client, err = btcrpcclient.New(&rpcConf, h.handlers); err != nil {
			time.Sleep(time.Duration(i) * 50 * time.Millisecond)
			continue
		}
		break
	}

	if client == nil {
		return fmt.Errorf("connection timeout")
	}

	h.Node = client
	h.wallet.SetRPCClient(client)
	return nil
}

// NewAddress returns a fresh address spendable by the Harness' internal
// wallet.
//
// This function is safe for concurrent access.
func (h *Harness) NewAddress() (btcutil.Address, error) {
	return h.wallet.NewAddress()
}

// ConfirmedBalance returns the confirmed balance of the Harness' internal
// wallet.
//
// This function is safe for concurrent access.
func (h *Harness) ConfirmedBalance() btcutil.Amount {
	return h.wallet.ConfirmedBalance()
}

// SendOutputs creates, signs, and finally broadcasts a transaction spending
// the harness' available mature coinbase outputs creating new outputs
// according to targetOutputs.
//
// This function is safe for concurrent access.
func (h *Harness) SendOutputs(targetOutputs []*wire.TxOut,
	feeRate btcutil.Amount) (*chainhash.Hash, error) {

	return h.wallet.SendOutputs(targetOutputs, feeRate)
}

// CreateTransaction returns a fully signed transaction paying to the specified
// outputs while observing the desired fee rate. The passed fee rate should be
// expressed in satoshis-per-byte. Any unspent outputs selected as inputs for
// the crafted transaction are marked as unspendable in order to avoid
// potential double-spends by future calls to this method. If the created
// transaction is cancelled for any reason then the selected inputs MUST be
// freed via a call to UnlockOutputs. Otherwise, the locked inputs won't be
// returned to the pool of spendable outputs.
//
// This function is safe for concurrent access.
func (h *Harness) CreateTransaction(targetOutputs []*wire.TxOut,
	feeRate btcutil.Amount) (*wire.MsgTx, error) {

	return h.wallet.CreateTransaction(targetOutputs, feeRate)
}

// UnlockOutputs unlocks any outputs which were previously marked as
// unspendabe due to being selected to fund a transaction via the
// CreateTransaction method.
//
// This function is safe for concurrent access.
func (h *Harness) UnlockOutputs(inputs []*wire.TxIn) {
	h.wallet.UnlockOutputs(inputs)
}

// RPCConfig returns the harnesses current rpc configuration. This allows other
// potential RPC clients created within tests to connect to a given test
// harness instance.
func (h *Harness) RPCConfig() btcrpcclient.ConnConfig {
	return h.node.config.rpcConnConfig()
}

// GenerateAndSubmitBlock creates a block whose contents include the passed
// transactions and submits it to the running simnet node. For generating
// blocks with only a coinbase tx, callers can simply pass nil instead of
// transactions to be mined. Additionally, a custom block version can be set by
// the caller. A blockVersion of -1 indicates that the current default block
// version should be used. An uninitialized time.Time should be used for the
// blockTime parameter if one doesn't wish to set a custom time.
//
// This function is safe for concurrent access.
func (h *Harness) GenerateAndSubmitBlock(txns []*btcutil.Tx, blockVersion int32,
	blockTime time.Time) (*btcutil.Block, error) {

	h.Lock()
	defer h.Unlock()

	if blockVersion == -1 {
		blockVersion = wire.BlockVersion
	}

	prevBlockHash, prevBlockHeight, err := h.Node.GetBestBlock()
	if err != nil {
		return nil, err
	}
	mBlock, err := h.Node.GetBlock(prevBlockHash)
	if err != nil {
		return nil, err
	}
	prevBlock := btcutil.NewBlock(mBlock)
	prevBlock.SetHeight(prevBlockHeight)

	// Create a new block including the specified transactions
	newBlock, err := createBlock(prevBlock, txns, blockVersion,
		blockTime, h.wallet.coinbaseAddr, h.ActiveNet)
	if err != nil {
		return nil, err
	}

	// Submit the block to the simnet node.
	if err := h.Node.SubmitBlock(newBlock, nil); err != nil {
		return nil, err
	}

	return newBlock, nil
}

// generateListeningAddresses returns two strings representing listening
// addresses designated for the current rpc test. If there haven't been any
// test instances created, the default ports are used. Otherwise, in order to
// support multiple test nodes running at once, the p2p and rpc port are
// incremented after each initialization.
func generateListeningAddresses() (string, string) {
	localhost := "127.0.0.1"

	portString := func(minPort, maxPort int) string {
		port := minPort + numTestInstances + ((20 * processID) %
			(maxPort - minPort))
		return strconv.Itoa(port)
	}

	p2p := net.JoinHostPort(localhost, portString(minPeerPort, maxPeerPort))
	rpc := net.JoinHostPort(localhost, portString(minRPCPort, maxRPCPort))
	return p2p, rpc
}
<commit_msg>rpctest: Fix typo<commit_after>// Copyright (c) 2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package rpctest

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcrpcclient"
	"github.com/btcsuite/btcutil"
)

const (
	// These constants define the minimum and maximum p2p and rpc port
	// numbers used by a test harness.  The min port is inclusive while the
	// max port is exclusive.
	minPeerPort = 10000
	maxPeerPort = 35000
	minRPCPort  = maxPeerPort
	maxRPCPort  = 60000
)

var (
	// current number of active test nodes.
	numTestInstances = 0

	// processID is the process ID of the current running process.  It is
	// used to calculate ports based upon it when launching an rpc
	// harnesses.  The intent is to allow multiple process to run in
	// parallel without port collisions.
	//
	// It should be noted however that there is still some small probability
	// that there will be port collisions either due to other processes
	// running or simply due to the stars aligning on the process IDs.
	processID = os.Getpid()

	// testInstances is a private package-level slice used to keep track of
	// all active test harnesses. This global can be used to perform
	// various "joins", shutdown several active harnesses after a test,
	// etc.
	testInstances = make(map[string]*Harness)

	// Used to protest concurrent access to above declared variables.
	harnessStateMtx sync.RWMutex
)

// HarnessTestCase represents a test-case which utilizes an instance of the
// Harness to exercise functionality.
type HarnessTestCase func(r *Harness, t *testing.T)

// Harness fully encapsulates an active btcd process to provide a unified
// platform for creating rpc driven integration tests involving btcd. The
// active btcd node will typically be run in simnet mode in order to allow for
// easy generation of test blockchains.  The active btcd process is fully
// managed by Harness, which handles the necessary initialization, and teardown
// of the process along with any temporary directories created as a result.
// Multiple Harness instances may be run concurrently, in order to allow for
// testing complex scenarios involving multiple nodes. The harness also
// includes an in-memory wallet to streamline various classes of tests.
type Harness struct {
	// ActiveNet is the parameters of the blockchain the Harness belongs
	// to.
	ActiveNet *chaincfg.Params

	Node     *btcrpcclient.Client
	node     *node
	handlers *btcrpcclient.NotificationHandlers

	wallet *memWallet

	testNodeDir    string
	maxConnRetries int
	nodeNum        int

	sync.Mutex
}

// New creates and initializes new instance of the rpc test harness.
// Optionally, websocket handlers and a specified configuration may be passed.
// In the case that a nil config is passed, a default configuration will be
// used.
//
// NOTE: This function is safe for concurrent access.
func New(activeNet *chaincfg.Params, handlers *btcrpcclient.NotificationHandlers,
	extraArgs []string) (*Harness, error) {

	harnessStateMtx.Lock()
	defer harnessStateMtx.Unlock()

	harnessID := strconv.Itoa(int(numTestInstances))
	nodeTestData, err := ioutil.TempDir("", "rpctest-"+harnessID)
	if err != nil {
		return nil, err
	}

	certFile := filepath.Join(nodeTestData, "rpc.cert")
	keyFile := filepath.Join(nodeTestData, "rpc.key")
	if err := genCertPair(certFile, keyFile); err != nil {
		return nil, err
	}

	wallet, err := newMemWallet(activeNet, uint32(numTestInstances))
	if err != nil {
		return nil, err
	}

	miningAddr := fmt.Sprintf("--miningaddr=%s", wallet.coinbaseAddr)
	extraArgs = append(extraArgs, miningAddr)

	config, err := newConfig("rpctest", certFile, keyFile, extraArgs)
	if err != nil {
		return nil, err
	}

	// Generate p2p+rpc listening addresses.
	config.listen, config.rpcListen = generateListeningAddresses()

	// Create the testing node bounded to the simnet.
	node, err := newNode(config, nodeTestData)
	if err != nil {
		return nil, err
	}

	nodeNum := numTestInstances
	numTestInstances++

	if handlers == nil {
		handlers = &btcrpcclient.NotificationHandlers{}
	}

	// If a handler for the OnBlockConnected/OnBlockDisconnected callback
	// has already been set, then we create a wrapper callback which
	// executes both the currently registered callback, and the mem
	// wallet's callback.
	if handlers.OnBlockConnected != nil {
		obc := handlers.OnBlockConnected
		handlers.OnBlockConnected = func(hash *chainhash.Hash, height int32, t time.Time) {
			wallet.IngestBlock(hash, height, t)
			obc(hash, height, t)
		}
	} else {
		// Otherwise, we can claim the callback ourselves.
		handlers.OnBlockConnected = wallet.IngestBlock
	}
	if handlers.OnBlockDisconnected != nil {
		obd := handlers.OnBlockConnected
		handlers.OnBlockDisconnected = func(hash *chainhash.Hash, height int32, t time.Time) {
			wallet.UnwindBlock(hash, height, t)
			obd(hash, height, t)
		}
	} else {
		handlers.OnBlockDisconnected = wallet.UnwindBlock
	}

	h := &Harness{
		handlers:       handlers,
		node:           node,
		maxConnRetries: 20,
		testNodeDir:    nodeTestData,
		ActiveNet:      activeNet,
		nodeNum:        nodeNum,
		wallet:         wallet,
	}

	// Track this newly created test instance within the package level
	// global map of all active test instances.
	testInstances[h.testNodeDir] = h

	return h, nil
}

// SetUp initializes the rpc test state. Initialization includes: starting up a
// simnet node, creating a websockets client and connecting to the started
// node, and finally: optionally generating and submitting a testchain with a
// configurable number of mature coinbase outputs coinbase outputs.
//
// NOTE: This method and TearDown should always be called from the same
// goroutine as they are not concurrent safe.
func (h *Harness) SetUp(createTestChain bool, numMatureOutputs uint32) error {
	// Start the btcd node itself. This spawns a new process which will be
	// managed
	if err := h.node.start(); err != nil {
		return err
	}
	if err := h.connectRPCClient(); err != nil {
		return err
	}

	h.wallet.Start()

	// Ensure the btcd properly dispatches our registered call-back for
	// each new block. Otherwise, the memWallet won't function properly.
	if err := h.Node.NotifyBlocks(); err != nil {
		return err
	}

	// Create a test chain with the desired number of mature coinbase
	// outputs.
	if createTestChain && numMatureOutputs != 0 {
		numToGenerate := (uint32(h.ActiveNet.CoinbaseMaturity) +
			numMatureOutputs)
		_, err := h.Node.Generate(numToGenerate)
		if err != nil {
			return err
		}
	}

	// Block until the wallet has fully synced up to the tip of the main
	// chain.
	_, height, err := h.Node.GetBestBlock()
	if err != nil {
		return err
	}
	ticker := time.NewTicker(time.Millisecond * 100)
out:
	for {
		select {
		case <-ticker.C:
			walletHeight := h.wallet.SyncedHeight()
			if walletHeight == height {
				break out
			}
		}
	}

	return nil
}

// TearDown stops the running rpc test instance. All created processes are
// killed, and temporary directories removed.
//
// NOTE: This method and SetUp should always be called from the same goroutine
// as they are not concurrent safe.
func (h *Harness) TearDown() error {
	if h.Node != nil {
		h.Node.Shutdown()
	}

	if err := h.node.shutdown(); err != nil {
		return err
	}

	if err := os.RemoveAll(h.testNodeDir); err != nil {
		return err
	}

	delete(testInstances, h.testNodeDir)

	return nil
}

// connectRPCClient attempts to establish an RPC connection to the created btcd
// process belonging to this Harness instance. If the initial connection
// attempt fails, this function will retry h.maxConnRetries times, backing off
// the time between subsequent attempts. If after h.maxConnRetries attempts,
// we're not able to establish a connection, this function returns with an
// error.
func (h *Harness) connectRPCClient() error {
	var client *btcrpcclient.Client
	var err error

	rpcConf := h.node.config.rpcConnConfig()
	for i := 0; i < h.maxConnRetries; i++ {
		if client, err = btcrpcclient.New(&rpcConf, h.handlers); err != nil {
			time.Sleep(time.Duration(i) * 50 * time.Millisecond)
			continue
		}
		break
	}

	if client == nil {
		return fmt.Errorf("connection timeout")
	}

	h.Node = client
	h.wallet.SetRPCClient(client)
	return nil
}

// NewAddress returns a fresh address spendable by the Harness' internal
// wallet.
//
// This function is safe for concurrent access.
func (h *Harness) NewAddress() (btcutil.Address, error) {
	return h.wallet.NewAddress()
}

// ConfirmedBalance returns the confirmed balance of the Harness' internal
// wallet.
//
// This function is safe for concurrent access.
func (h *Harness) ConfirmedBalance() btcutil.Amount {
	return h.wallet.ConfirmedBalance()
}

// SendOutputs creates, signs, and finally broadcasts a transaction spending
// the harness' available mature coinbase outputs creating new outputs
// according to targetOutputs.
//
// This function is safe for concurrent access.
func (h *Harness) SendOutputs(targetOutputs []*wire.TxOut,
	feeRate btcutil.Amount) (*chainhash.Hash, error) {

	return h.wallet.SendOutputs(targetOutputs, feeRate)
}

// CreateTransaction returns a fully signed transaction paying to the specified
// outputs while observing the desired fee rate. The passed fee rate should be
// expressed in satoshis-per-byte. Any unspent outputs selected as inputs for
// the crafted transaction are marked as unspendable in order to avoid
// potential double-spends by future calls to this method. If the created
// transaction is cancelled for any reason then the selected inputs MUST be
// freed via a call to UnlockOutputs. Otherwise, the locked inputs won't be
// returned to the pool of spendable outputs.
//
// This function is safe for concurrent access.
func (h *Harness) CreateTransaction(targetOutputs []*wire.TxOut,
	feeRate btcutil.Amount) (*wire.MsgTx, error) {

	return h.wallet.CreateTransaction(targetOutputs, feeRate)
}

// UnlockOutputs unlocks any outputs which were previously marked as
// unspendabe due to being selected to fund a transaction via the
// CreateTransaction method.
//
// This function is safe for concurrent access.
func (h *Harness) UnlockOutputs(inputs []*wire.TxIn) {
	h.wallet.UnlockOutputs(inputs)
}

// RPCConfig returns the harnesses current rpc configuration. This allows other
// potential RPC clients created within tests to connect to a given test
// harness instance.
func (h *Harness) RPCConfig() btcrpcclient.ConnConfig {
	return h.node.config.rpcConnConfig()
}

// GenerateAndSubmitBlock creates a block whose contents include the passed
// transactions and submits it to the running simnet node. For generating
// blocks with only a coinbase tx, callers can simply pass nil instead of
// transactions to be mined. Additionally, a custom block version can be set by
// the caller. A blockVersion of -1 indicates that the current default block
// version should be used. An uninitialized time.Time should be used for the
// blockTime parameter if one doesn't wish to set a custom time.
//
// This function is safe for concurrent access.
func (h *Harness) GenerateAndSubmitBlock(txns []*btcutil.Tx, blockVersion int32,
	blockTime time.Time) (*btcutil.Block, error) {

	h.Lock()
	defer h.Unlock()

	if blockVersion == -1 {
		blockVersion = wire.BlockVersion
	}

	prevBlockHash, prevBlockHeight, err := h.Node.GetBestBlock()
	if err != nil {
		return nil, err
	}
	mBlock, err := h.Node.GetBlock(prevBlockHash)
	if err != nil {
		return nil, err
	}
	prevBlock := btcutil.NewBlock(mBlock)
	prevBlock.SetHeight(prevBlockHeight)

	// Create a new block including the specified transactions
	newBlock, err := createBlock(prevBlock, txns, blockVersion,
		blockTime, h.wallet.coinbaseAddr, h.ActiveNet)
	if err != nil {
		return nil, err
	}

	// Submit the block to the simnet node.
	if err := h.Node.SubmitBlock(newBlock, nil); err != nil {
		return nil, err
	}

	return newBlock, nil
}

// generateListeningAddresses returns two strings representing listening
// addresses designated for the current rpc test. If there haven't been any
// test instances created, the default ports are used. Otherwise, in order to
// support multiple test nodes running at once, the p2p and rpc port are
// incremented after each initialization.
func generateListeningAddresses() (string, string) {
	localhost := "127.0.0.1"

	portString := func(minPort, maxPort int) string {
		port := minPort + numTestInstances + ((20 * processID) %
			(maxPort - minPort))
		return strconv.Itoa(port)
	}

	p2p := net.JoinHostPort(localhost, portString(minPeerPort, maxPeerPort))
	rpc := net.JoinHostPort(localhost, portString(minRPCPort, maxRPCPort))
	return p2p, rpc
}
<|endoftext|>


<commit_before>package bootstrap

import (
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/viant/asc"
	_ "github.com/viant/bgc"
	_ "github.com/viant/endly/static" //load external resource like .csv .json files to mem storage
	_ "github.com/viant/toolbox/storage/aws"
	_ "github.com/viant/toolbox/storage/gs"

	"encoding/json"
	"flag"
	"fmt"
	"github.com/viant/endly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/url"
	"gopkg.in/yaml.v2"
	"log"
	"os"
	"path"
	"strings"
	"time"
)

func init() {

	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	flag.String("r", "run.json", "<path/url to workflow run request in JSON format> ")
	flag.String("w", "manager", "<workflow name>  if both -r and -w valid options are specified, -w is ignored")
	flag.String("t", "*", "<task/s to run>")
	flag.String("l", "logs", "<log directory>")
	flag.Bool("d", false, "enable logging")
	flag.Bool("p", false, "print neatly workflow as JSON")
	flag.String("f", "json", "<workflow or request format>, json or yaml")

	flag.Bool("h", false, "print help")
	flag.Bool("v", false, "print version")

	flag.String("s", "", "<serviceID> print service details, -s='*' prints all service IDs")
	flag.String("a", "", "<action> prints action request representation")

}

func Bootstrap() {
	flag.Usage = printHelp
	flag.Parse()
	flagset := make(map[string]string)
	flag.Visit(func(f *flag.Flag) {
		flagset[f.Name] = f.Value.String()
	})

	_, shouldQuit := flagset["v"]
	flagset["v"] = flag.Lookup("v").Value.String()

	if toolbox.AsBoolean(flagset["v"]) {
		printVersion()
		if shouldQuit {
			return
		}
	}

	if _, ok := flagset["h"]; ok {
		printHelp()
		return
	}

	if _, ok := flagset["a"]; ok {
		printServiceActionRequest()
		return
	}
	if _, ok := flagset["s"]; ok {
		printServiceActions()
		return
	}

	request, option, err := getRunRequestWithOptons(flagset)
	if request == nil {
		flagset["r"] = flag.Lookup("r").Value.String()
		flagset["w"] = flag.Lookup("w").Value.String()
		request, option, err = getRunRequestWithOptons(flagset)
		if err != nil && strings.Contains(err.Error(), "failed to locate workflow: manager") {
			printHelp()
			return
		}
	}

	if err != nil {
		log.Fatal(err)
	}
	if value, ok := flagset["p"]; ok && toolbox.AsBoolean(value) {
		printWorkflow(request.WorkflowURL)
		return
	}

	runner := endly.NewCliRunner()
	err = runner.Run(request, option)
	if err != nil {
		log.Fatal(err)
	}
	time.Sleep(time.Second)
}

func printServiceActionRequest() {
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())

	var serviceID = flag.Lookup("s").Value.String()
	service, err := context.Service(serviceID)
	if err != nil {
		log.Fatal(err)
	}
	var action = flag.Lookup("a").Value.String()
	request, err := service.NewRequest(action)
	if err != nil {
		log.Fatal(err)
	}
	toolbox.InitStruct(request)
	fmt.Printf("Request: %T\n", request)
	printInFormat(request, fmt.Sprintf("failed to print %v.%v request (%T)", serviceID, action, request)+", %v")
	response, _ := service.NewResponse(action)
	fmt.Printf("\nResponse: %T\n", response)
	toolbox.InitStruct(response)
	printInFormat(response, fmt.Sprintf("failed to print %v.%v response (%T)", serviceID, action, request)+", %v")

}

func printServiceActions() {
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())

	var serviceID = flag.Lookup("s").Value.String()

	if serviceID == "*" {
		fmt.Printf("endly services:\n")
		for k, v := range endly.Services(manager) {
			fmt.Printf("%v %T\n", k, v)
		}
		return;
	}

	service, err := context.Service(serviceID)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("'%v' service actions: \n", serviceID)
	for _, action := range service.Actions() {
		fmt.Printf("\t%v\n", action)
	}
}

func printWorkflow(URL string) {
	dao := endly.NewWorkflowDao()
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())
	if workflow, _ := dao.Load(context, url.NewResource(URL)); workflow != nil {
		printInFormat(workflow, "failed to print workflow: "+URL+", %v")
	}
}

func printInFormat(source interface{}, errorTemplate string) {
	format := flag.Lookup("f").Value.String()
	var buf []byte
	var err error
	switch format {
	case "yaml":
		buf, err = yaml.Marshal(source)
	default:
		buf, err = json.MarshalIndent(source, "", "\t")
	}
	if err != nil {
		log.Fatalf(errorTemplate, err)
	}
	fmt.Printf("%s\n", buf)
}

func printHelp() {
	_, name := path.Split(os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", name)
	fmt.Fprintf(os.Stderr, "endly [options] [params...]\n")
	fmt.Fprintf(os.Stderr, "\tparams should be key value pair to be supplied as actual workflow parameters\n")
	fmt.Fprintf(os.Stderr, "\tif -r options is used, original request params may be overriden\n\n")

	fmt.Fprintf(os.Stderr, "where options include:\n")
	flag.PrintDefaults()
}

func printVersion() {
	fmt.Fprintf(os.Stdout, "%v %v\n", endly.AppName, endly.GetVersion())
}

func getWorkflowURL(candidate string) (string, string, error) {
	var _, name = path.Split(candidate)
	if path.Ext(candidate) == "" {
		candidate = candidate + ".csv"
	} else {
		name = string(name[:len(name)-4]) //remove extension
	}
	resource := url.NewResource(candidate)
	if _, err := resource.Download(); err != nil {
		resource = url.NewResource(fmt.Sprintf("mem://%v/workflow/%v", endly.EndlyNamespace, candidate))
		if _, memError := resource.Download(); memError != nil {
			return "", "", err
		}
	}
	return name, resource.URL, nil
}

func getRunRequestURL(candidate string) (*url.Resource, error) {
	if path.Ext(candidate) == "" {
		candidate = candidate + ".json"
	}
	resource := url.NewResource(candidate)
	if _, err := resource.Download(); err != nil {
		resource = url.NewResource(fmt.Sprintf("mem://%v/req/%v", endly.EndlyNamespace, candidate))
		if _, memError := resource.Download(); memError != nil {
			return nil, err
		}
	}
	return resource, nil

}

func getRunRequestWithOptons(flagset map[string]string) (*endly.WorkflowRunRequest, *endly.RunnerReportingOptions, error) {
	var request *endly.WorkflowRunRequest
	var options = &endly.RunnerReportingOptions{}
	if value, ok := flagset["w"]; ok {
		name, URL, err := getWorkflowURL(value)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to locate workflow: %v %v", value, err)
		}
		request = &endly.WorkflowRunRequest{
			WorkflowURL: URL,
			Name:        name,
		}
		options = endly.DefaultRunnerReportingOption()
	}
	if value, ok := flagset["r"]; ok {
		resource, err := getRunRequestURL(value)
		if err == nil {
			request = &endly.WorkflowRunRequest{}
			err = resource.JSONDecode(request)
		}
		if request.WorkflowURL == "" {
			parent, _ := toolbox.URLSplit(resource.URL)
			parent = strings.Replace(parent, "req", "workflow", 1)
			request.WorkflowURL = toolbox.URLPathJoin(parent, request.Name+".csv")
		}
		if err != nil {
			return nil, nil, fmt.Errorf("failed to locate workflow run request: %v %v", value, err)
		}
		resource.JSONDecode(options)
		if options.Filter == nil {
			options.Filter = endly.DefaultRunnerReportingOption().Filter
		}
	}

	var params = endly.Pairs(getArguments()...)

	if request != nil {
		if len(request.Params) == 0 {
			request.Params = params
		}
		for k, v := range params {
			request.Params[k] = v
		}
		if value, ok := flagset["d"]; ok {
			request.EnableLogging = toolbox.AsBoolean(value)
			request.LoggingDirectory = flag.Lookup("l").Value.String()
		}
		if value, ok := flagset["t"]; ok {
			request.Tasks = value
		}
	}
	return request, options, nil
}

func normalizeArgument(value string) interface{} {
	value = strings.Trim(value, " \"'")
	if strings.HasPrefix(value, "#") {
		resource := url.NewResource(string(value[1:]))
		text, err := resource.DownloadText();
		if err == nil {
			value = text
		}
	}
	_, structure := endly.AsExtractable(value)
	if len(structure) > 0 {
		if strings.HasPrefix(value, "[") {
			return structure[endly.SliceKey]
		}
		return structure
	}
	return value
}

func getArguments() []interface{} {
	var arguments = make([]interface{}, 0)
	if len(os.Args) > 1 {
		for i := 1; i < len(os.Args); i++ {
			if strings.HasPrefix(os.Args[i], "-") {
				if !strings.Contains(os.Args[i], "=") {
					i++
				}
				continue
			}
			arguments = append(arguments, normalizeArgument(os.Args[i]))
		}
	}
	return arguments
}
<commit_msg>updated bootstrap to load external file with either @ or #<commit_after>package bootstrap

import (
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	_ "github.com/viant/asc"
	_ "github.com/viant/bgc"
	_ "github.com/viant/endly/static" //load external resource like .csv .json files to mem storage
	_ "github.com/viant/toolbox/storage/aws"
	_ "github.com/viant/toolbox/storage/gs"

	"encoding/json"
	"flag"
	"fmt"
	"github.com/viant/endly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/url"
	"gopkg.in/yaml.v2"
	"log"
	"os"
	"path"
	"strings"
	"time"
)

func init() {

	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	flag.String("r", "run.json", "<path/url to workflow run request in JSON format> ")
	flag.String("w", "manager", "<workflow name>  if both -r and -w valid options are specified, -w is ignored")
	flag.String("t", "*", "<task/s to run>")
	flag.String("l", "logs", "<log directory>")
	flag.Bool("d", false, "enable logging")
	flag.Bool("p", false, "print neatly workflow as JSON")
	flag.String("f", "json", "<workflow or request format>, json or yaml")

	flag.Bool("h", false, "print help")
	flag.Bool("v", false, "print version")

	flag.String("s", "", "<serviceID> print service details, -s='*' prints all service IDs")
	flag.String("a", "", "<action> prints action request representation")

}

func Bootstrap() {
	flag.Usage = printHelp
	flag.Parse()
	flagset := make(map[string]string)
	flag.Visit(func(f *flag.Flag) {
		flagset[f.Name] = f.Value.String()
	})

	_, shouldQuit := flagset["v"]
	flagset["v"] = flag.Lookup("v").Value.String()

	if toolbox.AsBoolean(flagset["v"]) {
		printVersion()
		if shouldQuit {
			return
		}
	}

	if _, ok := flagset["h"]; ok {
		printHelp()
		return
	}

	if _, ok := flagset["a"]; ok {
		printServiceActionRequest()
		return
	}
	if _, ok := flagset["s"]; ok {
		printServiceActions()
		return
	}

	request, option, err := getRunRequestWithOptons(flagset)
	if request == nil {
		flagset["r"] = flag.Lookup("r").Value.String()
		flagset["w"] = flag.Lookup("w").Value.String()
		request, option, err = getRunRequestWithOptons(flagset)
		if err != nil && strings.Contains(err.Error(), "failed to locate workflow: manager") {
			printHelp()
			return
		}
	}

	if err != nil {
		log.Fatal(err)
	}
	if value, ok := flagset["p"]; ok && toolbox.AsBoolean(value) {
		printWorkflow(request.WorkflowURL)
		return
	}

	runner := endly.NewCliRunner()
	err = runner.Run(request, option)
	if err != nil {
		log.Fatal(err)
	}
	time.Sleep(time.Second)
}

func printServiceActionRequest() {
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())

	var serviceID = flag.Lookup("s").Value.String()
	service, err := context.Service(serviceID)
	if err != nil {
		log.Fatal(err)
	}
	var action = flag.Lookup("a").Value.String()
	request, err := service.NewRequest(action)
	if err != nil {
		log.Fatal(err)
	}
	toolbox.InitStruct(request)
	fmt.Printf("Request: %T\n", request)
	printInFormat(request, fmt.Sprintf("failed to print %v.%v request (%T)", serviceID, action, request)+", %v")
	response, _ := service.NewResponse(action)
	fmt.Printf("\nResponse: %T\n", response)
	toolbox.InitStruct(response)
	printInFormat(response, fmt.Sprintf("failed to print %v.%v response (%T)", serviceID, action, request)+", %v")

}

func printServiceActions() {
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())

	var serviceID = flag.Lookup("s").Value.String()

	if serviceID == "*" {
		fmt.Printf("endly services:\n")
		for k, v := range endly.Services(manager) {
			fmt.Printf("%v %T\n", k, v)
		}
		return;
	}

	service, err := context.Service(serviceID)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("'%v' service actions: \n", serviceID)
	for _, action := range service.Actions() {
		fmt.Printf("\t%v\n", action)
	}
}

func printWorkflow(URL string) {
	dao := endly.NewWorkflowDao()
	manager := endly.NewManager()
	context := manager.NewContext(toolbox.NewContext())
	if workflow, _ := dao.Load(context, url.NewResource(URL)); workflow != nil {
		printInFormat(workflow, "failed to print workflow: "+URL+", %v")
	}
}

func printInFormat(source interface{}, errorTemplate string) {
	format := flag.Lookup("f").Value.String()
	var buf []byte
	var err error
	switch format {
	case "yaml":
		buf, err = yaml.Marshal(source)
	default:
		buf, err = json.MarshalIndent(source, "", "\t")
	}
	if err != nil {
		log.Fatalf(errorTemplate, err)
	}
	fmt.Printf("%s\n", buf)
}

func printHelp() {
	_, name := path.Split(os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", name)
	fmt.Fprintf(os.Stderr, "endly [options] [params...]\n")
	fmt.Fprintf(os.Stderr, "\tparams should be key value pair to be supplied as actual workflow parameters\n")
	fmt.Fprintf(os.Stderr, "\tif -r options is used, original request params may be overriden\n\n")

	fmt.Fprintf(os.Stderr, "where options include:\n")
	flag.PrintDefaults()
}

func printVersion() {
	fmt.Fprintf(os.Stdout, "%v %v\n", endly.AppName, endly.GetVersion())
}

func getWorkflowURL(candidate string) (string, string, error) {
	var _, name = path.Split(candidate)
	if path.Ext(candidate) == "" {
		candidate = candidate + ".csv"
	} else {
		name = string(name[:len(name)-4]) //remove extension
	}
	resource := url.NewResource(candidate)
	if _, err := resource.Download(); err != nil {
		resource = url.NewResource(fmt.Sprintf("mem://%v/workflow/%v", endly.EndlyNamespace, candidate))
		if _, memError := resource.Download(); memError != nil {
			return "", "", err
		}
	}
	return name, resource.URL, nil
}

func getRunRequestURL(candidate string) (*url.Resource, error) {
	if path.Ext(candidate) == "" {
		candidate = candidate + ".json"
	}
	resource := url.NewResource(candidate)
	if _, err := resource.Download(); err != nil {
		resource = url.NewResource(fmt.Sprintf("mem://%v/req/%v", endly.EndlyNamespace, candidate))
		if _, memError := resource.Download(); memError != nil {
			return nil, err
		}
	}
	return resource, nil

}

func getRunRequestWithOptons(flagset map[string]string) (*endly.WorkflowRunRequest, *endly.RunnerReportingOptions, error) {
	var request *endly.WorkflowRunRequest
	var options = &endly.RunnerReportingOptions{}
	if value, ok := flagset["w"]; ok {
		name, URL, err := getWorkflowURL(value)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to locate workflow: %v %v", value, err)
		}
		request = &endly.WorkflowRunRequest{
			WorkflowURL: URL,
			Name:        name,
		}
		options = endly.DefaultRunnerReportingOption()
	}
	if value, ok := flagset["r"]; ok {
		resource, err := getRunRequestURL(value)
		if err == nil {
			request = &endly.WorkflowRunRequest{}
			err = resource.JSONDecode(request)
		}
		if request.WorkflowURL == "" {
			parent, _ := toolbox.URLSplit(resource.URL)
			parent = strings.Replace(parent, "req", "workflow", 1)
			request.WorkflowURL = toolbox.URLPathJoin(parent, request.Name+".csv")
		}
		if err != nil {
			return nil, nil, fmt.Errorf("failed to locate workflow run request: %v %v", value, err)
		}
		resource.JSONDecode(options)
		if options.Filter == nil {
			options.Filter = endly.DefaultRunnerReportingOption().Filter
		}
	}

	var params = endly.Pairs(getArguments()...)

	if request != nil {
		if len(request.Params) == 0 {
			request.Params = params
		}
		for k, v := range params {
			request.Params[k] = v
		}
		if value, ok := flagset["d"]; ok {
			request.EnableLogging = toolbox.AsBoolean(value)
			request.LoggingDirectory = flag.Lookup("l").Value.String()
		}
		if value, ok := flagset["t"]; ok {
			request.Tasks = value
		}
	}
	return request, options, nil
}

func normalizeArgument(value string) interface{} {
	value = strings.Trim(value, " \"'")
	if strings.HasPrefix(value, "#") || strings.HasPrefix(value, "@") {
		resource := url.NewResource(string(value[1:]))
		text, err := resource.DownloadText();
		if err == nil {
			value = text
		}
	}
	_, structure := endly.AsExtractable(value)
	if len(structure) > 0 {
		if strings.HasPrefix(value, "[") {
			return structure[endly.SliceKey]
		}
		return structure
	}
	return value
}

func getArguments() []interface{} {
	var arguments = make([]interface{}, 0)
	if len(os.Args) > 1 {
		for i := 1; i < len(os.Args); i++ {
			if strings.HasPrefix(os.Args[i], "-") {
				if !strings.Contains(os.Args[i], "=") {
					i++
				}
				continue
			}
			arguments = append(arguments, normalizeArgument(os.Args[i]))
		}
	}
	return arguments
}
<|endoftext|>