<commit_before>package pymlstate

import (
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/ugorji/go/codec"
	"io"
	"pfi/sensorbee/py/pystate"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/data"
	"sync"
)

var (
	lossPath = data.MustCompilePath("loss")
	accPath  = data.MustCompilePath("accuracy")
)

// State is python instance specialized to multiple layer classification.
// The python instance and this struct must not be coppied directly by assignment
// statement because it doesn't increase reference count of instance.
type State struct {
	base   *pystate.Base
	params MLParams
	bucket []data.Value
	rwm    sync.RWMutex
}

// MLParams is parameters pymlstate defines in addition to those pystate does.
// These parameters come from a WITH clause of a CREATE STATE statement.
type MLParams struct {
	// BatchSize is number of tuples in a single batch training. Write method,
	// which is usually called by an INSERT INTOT statement via uds Sink, stores
	// tuples without training until it has tuples as many as batch_train_size.
	// This is an optional parameter and its default value is 10.
	BatchSize int `codec:"batch_train_size"`
}

// New creates `core.SharedState` for multiple layer classification.
func New(baseParams *pystate.BaseParams, mlParams *MLParams, params data.Map) (*State, error) {
	b, err := pystate.NewBase(baseParams, params)
	if err != nil {
		return nil, err
	}

	s := &State{
		base:   b,
		params: *mlParams,
		bucket: make([]data.Value, 0, mlParams.BatchSize),
	}
	return s, nil
}

// Terminate terminates this state.
func (s *State) Terminate(ctx *core.Context) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.Terminate(ctx); err != nil {
		return err
	}
	// Don't set s.base = nil because it's used for the termination detection.
	s.bucket = nil
	return nil
}

// Write stores a tuple to its bucket and calls "fit" function every
// "batch_train_size" times.
func (s *State) Write(ctx *core.Context, t *core.Tuple) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}

	s.bucket = append(s.bucket, t.Data)
	if len(s.bucket) < s.params.BatchSize {
		return nil
	}

	m, err := s.fit(ctx, s.bucket)
	prevBucketSize := len(s.bucket)
	s.bucket = s.bucket[:0] // clear slice but keep capacity
	if err != nil {
		ctx.ErrLog(err).WithField("bucket_size", prevBucketSize).
			Error("pymlstate's training via Write (INSERT INTO) failed")
		return err
	}

	// TODO: add option to toggle the following logging

	ret, err := data.AsArray(m)
	if err != nil || len(ret) != 2 {
		// The following log is optional. So, it isn't a error even if the
		// result doesn't have accuracy and loss fields.
		// TODO: write a warning log after the logging option is added.
		return nil
	}

	loss, err := data.ToFloat(ret[0])
	if err != nil {
		// TODO: add warning
		return nil
	}

	acc, err := data.ToFloat(ret[1])
	if err != nil {
		return nil
	}
	ctx.Log().Infof("loss=%.3f acc=%.3f", loss/float64(s.params.BatchSize),
		acc/float64(s.params.BatchSize))
	return nil
}

// Fit receives `data.Array` type but it assumes `[]data.Map` type
// for passing arguments to `fit` method.
func (s *State) Fit(ctx *core.Context, bucket []data.Value) (data.Value, error) {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	return s.fit(ctx, bucket)
}

// fit is the internal implementation of Fit. fit doesn't acquire the lock nor
// check s.ins == nil. RLock is sufficient when calling this method because
// this method itself doesn't change any field of State. Although the model
// will be updated by the data, the model is protected by Python's GIL. So,
// this method doesn't require a write lock.
func (s *State) fit(ctx *core.Context, bucket []data.Value) (data.Value, error) {
	return s.base.Call("fit", data.Array(bucket))
}

// Predict applies the model to the data. It returns a result returned from
// Python script.
func (s *State) Predict(ctx *core.Context, dt data.Value) (data.Value, error) {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	return s.base.Call("predict", dt)
}

// Save saves the model of the state. pystate calls `save` method and
// use its return value as dumped model.
func (s *State) Save(ctx *core.Context, w io.Writer, params data.Map) error {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}

	if err := s.saveState(w); err != nil {
		return err
	}
	return s.base.Save(ctx, w, params)
}

const (
	pyMLStateFormatVersion uint8 = 1
)

func (s *State) saveState(w io.Writer) error {
	if _, err := w.Write([]byte{pyMLStateFormatVersion}); err != nil {
		return err
	}

	// Save parameter of State before save python's model
	msgpackHandle := &codec.MsgpackHandle{}
	var out []byte
	enc := codec.NewEncoderBytes(&out, msgpackHandle)
	if err := enc.Encode(&s.params); err != nil {
		return err
	}

	// Write size of MLParams
	dataSize := uint32(len(out))
	err := binary.Write(w, binary.LittleEndian, dataSize)
	if err != nil {
		return err
	}

	// Write MLParams in msgpack
	n, err := w.Write(out)
	if err != nil {
		return err
	}

	if n < len(out) {
		return errors.New("cannot save the MLParams data")
	}

	return nil
}

// Load loads the model of the state. pystate calls `load` method and
// pass to the model data by using method parameter.
func (s *State) Load(ctx *core.Context, r io.Reader, params data.Map) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}
	return s.load(ctx, r, params)
}

func (s *State) load(ctx *core.Context, r io.Reader, params data.Map) error {
	var formatVersion uint8
	if err := binary.Read(r, binary.LittleEndian, &formatVersion); err != nil {
		return err
	}

	// TODO: remove MLParams specific parameters from params

	switch formatVersion {
	case 1:
		return s.loadMLParamsAndDataV1(ctx, r, params)
	default:
		return fmt.Errorf("unsupported format version of State container: %v", formatVersion)
	}
}

func (s *State) loadMLParamsAndDataV1(ctx *core.Context, r io.Reader, params data.Map) error {
	var dataSize uint32
	if err := binary.Read(r, binary.LittleEndian, &dataSize); err != nil {
		return err
	}
	if dataSize == 0 {
		return errors.New("size of MLParams must be greater than 0")
	}

	// Read MLParams from reader
	buf := make([]byte, dataSize)
	n, err := r.Read(buf)
	if err != nil {
		return err
	}
	if n != int(dataSize) {
		return errors.New("read size is different from the size of MLParams")
	}

	// Desirialize MLParams
	var saved MLParams
	msgpackHandle := &codec.MsgpackHandle{}
	dec := codec.NewDecoderBytes(buf, msgpackHandle)
	if err := dec.Decode(&saved); err != nil {
		return err
	}

	if s.base == nil { // loading for the first time
		s.base, err = pystate.LoadBase(ctx, r, params)
		if err != nil {
			return err
		}

	} else {
		if err := s.base.Load(ctx, r, params); err != nil {
			return err
		}
	}
	s.params = saved
	return nil
}

// Fit trains the model. It applies tuples that bucket has in a batch manner.
// The return value of this function depends on the implementation of Python
// UDS.
func Fit(ctx *core.Context, stateName string, bucket []data.Value) (data.Value, error) {
	s, err := lookupState(ctx, stateName)
	if err != nil {
		return nil, err
	}

	return s.Fit(ctx, bucket)
}

// Predict applies the model to the given data and returns estimated values.
// The format of the return value depends on each Python UDS.
func Predict(ctx *core.Context, stateName string, dt data.Value) (data.Value, error) {
	s, err := lookupState(ctx, stateName)
	if err != nil {
		return nil, err
	}

	return s.Predict(ctx, dt)
}

func lookupState(ctx *core.Context, stateName string) (*State, error) {
	st, err := ctx.SharedStates.Get(stateName)
	if err != nil {
		return nil, err
	}

	if s, ok := st.(*State); ok {
		return s, nil
	}

	return nil, fmt.Errorf("state '%v' isn't a State", stateName)
}
<commit_msg>modify to apply not need to store tuples in writing<commit_after>package pymlstate

import (
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/ugorji/go/codec"
	"io"
	"pfi/sensorbee/py/pystate"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/data"
	"sync"
)

var (
	datPath  = data.MustCompilePath("data")
	lossPath = data.MustCompilePath("loss")
	accPath  = data.MustCompilePath("accuracy")
)

// State is python instance specialized to multiple layer classification.
// The python instance and this struct must not be coppied directly by assignment
// statement because it doesn't increase reference count of instance.
type State struct {
	base   *pystate.Base
	params MLParams
	bucket []data.Value
	rwm    sync.RWMutex
}

// MLParams is parameters pymlstate defines in addition to those pystate does.
// These parameters come from a WITH clause of a CREATE STATE statement.
type MLParams struct {
	// BatchSize is number of tuples in a single batch training. Write method,
	// which is usually called by an INSERT INTOT statement via uds Sink, stores
	// tuples without training until it has tuples as many as batch_train_size.
	// This is an optional parameter and its default value is 10.
	BatchSize int `codec:"batch_train_size"`
}

// New creates `core.SharedState` for multiple layer classification.
func New(baseParams *pystate.BaseParams, mlParams *MLParams, params data.Map) (*State, error) {
	b, err := pystate.NewBase(baseParams, params)
	if err != nil {
		return nil, err
	}

	s := &State{
		base:   b,
		params: *mlParams,
		bucket: make([]data.Value, 0, mlParams.BatchSize),
	}
	return s, nil
}

// Terminate terminates this state.
func (s *State) Terminate(ctx *core.Context) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.Terminate(ctx); err != nil {
		return err
	}
	// Don't set s.base = nil because it's used for the termination detection.
	s.bucket = nil
	return nil
}

// Write stores a tuple to its bucket and calls "fit" function every
// "batch_train_size" times.
func (s *State) Write(ctx *core.Context, t *core.Tuple) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}

	dataSet, err := t.Data.Get(datPath)
	if err != nil {
		return err
	}

	dataSize := s.params.BatchSize
	if s.params.BatchSize > 1 {
		s.bucket = append(s.bucket, dataSet)
		if len(s.bucket) < s.params.BatchSize {
			return nil
		}
	} else {
		if dataSet.Type() == data.TypeArray {
			arr, _ := data.AsArray(dataSet)
			s.bucket = arr
			dataSize = len(arr)
		} else {
			s.bucket = []data.Value{dataSet}
		}
	}

	m, err := s.fit(ctx, s.bucket)
	prevBucketSize := len(s.bucket)
	s.bucket = s.bucket[:0] // clear slice but keep capacity
	if err != nil {
		ctx.ErrLog(err).WithField("bucket_size", prevBucketSize).
			Error("pymlstate's training via Write (INSERT INTO) failed")
		return err
	}

	// TODO: add option to toggle the following logging

	ret, err := data.AsArray(m)
	if err != nil || len(ret) != 2 {
		// The following log is optional. So, it isn't a error even if the
		// result doesn't have accuracy and loss fields.
		// TODO: write a warning log after the logging option is added.
		return nil
	}

	loss, err := data.ToFloat(ret[0])
	if err != nil {
		// TODO: add warning
		return nil
	}

	acc, err := data.ToFloat(ret[1])
	if err != nil {
		return nil
	}
	ctx.Log().Debugf("loss=%.3f acc=%.3f", loss/float64(dataSize),
		acc/float64(dataSize))
	return nil
}

// Fit receives `data.Array` type but it assumes `[]data.Map` type
// for passing arguments to `fit` method.
func (s *State) Fit(ctx *core.Context, bucket []data.Value) (data.Value, error) {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	return s.fit(ctx, bucket)
}

// fit is the internal implementation of Fit. fit doesn't acquire the lock nor
// check s.ins == nil. RLock is sufficient when calling this method because
// this method itself doesn't change any field of State. Although the model
// will be updated by the data, the model is protected by Python's GIL. So,
// this method doesn't require a write lock.
func (s *State) fit(ctx *core.Context, bucket []data.Value) (data.Value, error) {
	return s.base.Call("fit", data.Array(bucket))
}

// Predict applies the model to the data. It returns a result returned from
// Python script.
func (s *State) Predict(ctx *core.Context, dt data.Value) (data.Value, error) {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	return s.base.Call("predict", dt)
}

// Save saves the model of the state. pystate calls `save` method and
// use its return value as dumped model.
func (s *State) Save(ctx *core.Context, w io.Writer, params data.Map) error {
	s.rwm.RLock()
	defer s.rwm.RUnlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}

	if err := s.saveState(w); err != nil {
		return err
	}
	return s.base.Save(ctx, w, params)
}

const (
	pyMLStateFormatVersion uint8 = 1
)

func (s *State) saveState(w io.Writer) error {
	if _, err := w.Write([]byte{pyMLStateFormatVersion}); err != nil {
		return err
	}

	// Save parameter of State before save python's model
	msgpackHandle := &codec.MsgpackHandle{}
	var out []byte
	enc := codec.NewEncoderBytes(&out, msgpackHandle)
	if err := enc.Encode(&s.params); err != nil {
		return err
	}

	// Write size of MLParams
	dataSize := uint32(len(out))
	err := binary.Write(w, binary.LittleEndian, dataSize)
	if err != nil {
		return err
	}

	// Write MLParams in msgpack
	n, err := w.Write(out)
	if err != nil {
		return err
	}

	if n < len(out) {
		return errors.New("cannot save the MLParams data")
	}

	return nil
}

// Load loads the model of the state. pystate calls `load` method and
// pass to the model data by using method parameter.
func (s *State) Load(ctx *core.Context, r io.Reader, params data.Map) error {
	s.rwm.Lock()
	defer s.rwm.Unlock()
	if err := s.base.CheckTermination(); err != nil {
		return err
	}
	return s.load(ctx, r, params)
}

func (s *State) load(ctx *core.Context, r io.Reader, params data.Map) error {
	var formatVersion uint8
	if err := binary.Read(r, binary.LittleEndian, &formatVersion); err != nil {
		return err
	}

	// TODO: remove MLParams specific parameters from params

	switch formatVersion {
	case 1:
		return s.loadMLParamsAndDataV1(ctx, r, params)
	default:
		return fmt.Errorf("unsupported format version of State container: %v", formatVersion)
	}
}

func (s *State) loadMLParamsAndDataV1(ctx *core.Context, r io.Reader, params data.Map) error {
	var dataSize uint32
	if err := binary.Read(r, binary.LittleEndian, &dataSize); err != nil {
		return err
	}
	if dataSize == 0 {
		return errors.New("size of MLParams must be greater than 0")
	}

	// Read MLParams from reader
	buf := make([]byte, dataSize)
	n, err := r.Read(buf)
	if err != nil {
		return err
	}
	if n != int(dataSize) {
		return errors.New("read size is different from the size of MLParams")
	}

	// Desirialize MLParams
	var saved MLParams
	msgpackHandle := &codec.MsgpackHandle{}
	dec := codec.NewDecoderBytes(buf, msgpackHandle)
	if err := dec.Decode(&saved); err != nil {
		return err
	}

	if s.base == nil { // loading for the first time
		s.base, err = pystate.LoadBase(ctx, r, params)
		if err != nil {
			return err
		}

	} else {
		if err := s.base.Load(ctx, r, params); err != nil {
			return err
		}
	}
	s.params = saved
	return nil
}

// Fit trains the model. It applies tuples that bucket has in a batch manner.
// The return value of this function depends on the implementation of Python
// UDS.
func Fit(ctx *core.Context, stateName string, bucket []data.Value) (data.Value, error) {
	s, err := lookupState(ctx, stateName)
	if err != nil {
		return nil, err
	}

	return s.Fit(ctx, bucket)
}

// Predict applies the model to the given data and returns estimated values.
// The format of the return value depends on each Python UDS.
func Predict(ctx *core.Context, stateName string, dt data.Value) (data.Value, error) {
	s, err := lookupState(ctx, stateName)
	if err != nil {
		return nil, err
	}

	return s.Predict(ctx, dt)
}

func lookupState(ctx *core.Context, stateName string) (*State, error) {
	st, err := ctx.SharedStates.Get(stateName)
	if err != nil {
		return nil, err
	}

	if s, ok := st.(*State); ok {
		return s, nil
	}

	return nil, fmt.Errorf("state '%v' isn't a State", stateName)
}
<|endoftext|>


<commit_before>package structcheck

import (
	"container/list"
	"fmt"
	"reflect"
	"strings"
)

// value plus information from a few levels up
type metaValue struct {
	reflect.Value
	Name []string
	tag  *reflect.StructTag
}

func (v metaValue) buildDeeperName(n string) []string {
	name := make([]string, len(v.Name), len(v.Name)+1)
	copy(name, v.Name)
	return append(name, n)
}

func (v metaValue) Field(i int) metaValue {
	v2 := v.Value.Field(i)
	f := v.Value.Type().Field(i)
	n := v.buildDeeperName(f.Name)
	return metaValue{Value: v2, Name: n, tag: &f.Tag}
}

// InterfaceValue returns the Value wrapped by v (assuming v is an interface)
func (v metaValue) InterfaceValue() metaValue {
	v2 := reflect.ValueOf(v.Value.Interface())
	n := v.buildDeeperName(fmt.Sprintf("(%v)", v2.Type().Name()))
	return metaValue{Value: v2, Name: n, tag: v.tag}
}

func (v metaValue) Indirect() metaValue {
	return metaValue{Value: reflect.Indirect(v.Value), Name: v.Name, tag: v.tag}
}

func (v metaValue) getChecks() checks {
	var c checks
	if v.tag != nil {
		for _, str := range strings.Split(v.tag.Get("checks"), ",") {
			switch strings.ToLower(str) {
			case "notnil":
				c.NotNil = true
			}
		}
	}
	return c
}

type checks struct {
	NotNil bool
}

// Breadth First Search queue for reflective struct exploration. Prevents infinite recursion by marking pointers and refusing to push marked pointers.
type valueQueue struct {
	queuedPointers map[uintptr]interface{}
	queue          *list.List
}

func newValueQueue() *valueQueue {
	return &valueQueue{
		queuedPointers: make(map[uintptr]interface{}),
		queue:          list.New(),
	}
}

func (q *valueQueue) Push(v metaValue) {
	kind := v.Kind()
	// take internal value of interfaces
	if kind == reflect.Interface {
		v = v.InterfaceValue()
	}
	// mark pointers
	if kind == reflect.Ptr && !v.IsNil() {
		ptr := v.Pointer()
		if _, present := q.queuedPointers[ptr]; present {
			return
		} else {
			q.queuedPointers[ptr] = nil
		}
	}
	// enqueue value
	q.queue.PushBack(v)
	return
}

func (q *valueQueue) Pop() metaValue {
	front := q.queue.Front()
	v := front.Value.(metaValue)
	q.queue.Remove(front)
	return v
}

func (q *valueQueue) Len() int {
	return q.queue.Len()
}

// recursively runs checks on all
func Validate(o interface{}) error {
	if o == nil {
		return ErrorNilValue{}
	}
	top := reflect.ValueOf(o)
	// drill down to first struct
	for k := top.Kind(); k == reflect.Ptr || k == reflect.Interface; k = top.Kind() {
		if top.IsNil() {
			return ErrorNilValue{}
		} else if k == reflect.Ptr {
			top = reflect.Indirect(top)
		} else if k == reflect.Interface {
			top = reflect.ValueOf(top.Interface())
		}
	}

	if top.Kind() != reflect.Struct {
		return ErrorInvalidKind{Kind: top.Kind()}
	}
	// Breadth first search to find nil required fields
	namedTop := metaValue{Value: top, Name: []string{top.Type().Name()}}
	field2checks := make(map[Field][]Check)
	q := newValueQueue()
	q.Push(namedTop)
	for q.Len() > 0 {
		v := q.Pop()
		// check NotNil conditions
		switch v.Kind() {
		case reflect.Ptr, reflect.Interface, reflect.Chan, reflect.Func, reflect.Map, reflect.Slice:
			if v.IsNil() && v.getChecks().NotNil {
				f := NewField(v)
				field2checks[f] = append(field2checks[f], NotNil)
			}
		}
		// push new nodes onto queue
		switch v.Kind() {
		case reflect.Ptr:
			if !v.IsNil() {
				q.Push(v.Indirect())
			}
		case reflect.Interface:
			if !v.IsNil() {
				q.Push(v.InterfaceValue())
			}
		case reflect.Struct:
			for i := 0; i < v.NumField(); i++ {
				q.Push(v.Field(i))
			}
		}
	}

	if len(field2checks) != 0 {
		return ErrorChecksFailed{Field2Checks: field2checks}
	} else {
		return nil
	}
}

type Check string

const (
	NotNil Check = "NotNil"
)

type Field struct {
	Name  string
	Value string
}

func NewField(v metaValue) Field {
	return Field{Name: strings.Join(v.Name, "."), Value: fmt.Sprintf("%#v", v.Value.Interface())}
}

// returned when the top level object does not drill down to a struct.
type ErrorInvalidKind struct {
	reflect.Kind
}

func (e ErrorInvalidKind) Error() string {
	return fmt.Sprintf("Provided object must drill down to a struct. Received: %v", e.Kind)
}

// returned when a top level nil is received
type ErrorNilValue struct{}

func (e ErrorNilValue) Error() string {
	return fmt.Sprintf("Provided object must drill down to a struct. Encountered nil.")
}

// returned when checks fail on fields
type ErrorChecksFailed struct {
	Field2Checks map[Field][]Check
}

func (e ErrorChecksFailed) Error() string {
	failLines := make([]string, 0, len(e.Field2Checks))
	for field, checks := range e.Field2Checks {
		fails := make([]string, 0, len(checks))
		for _, check := range checks {
			fails = append(fails, string(check))
		}
		failLines = append(failLines, fmt.Sprintf("%v: %v: %v", strings.Join(fails, ", "), field.Name, field.Value))
	}
	return fmt.Sprintf("The following field(s) failed checks: \n\t%v", strings.Join(failLines, "\n\t"))
}
<commit_msg>renamed NewField() -> newField()<commit_after>package structcheck

import (
	"container/list"
	"fmt"
	"reflect"
	"strings"
)

// value plus information from a few levels up
type metaValue struct {
	reflect.Value
	Name []string
	tag  *reflect.StructTag
}

func (v metaValue) buildDeeperName(n string) []string {
	name := make([]string, len(v.Name), len(v.Name)+1)
	copy(name, v.Name)
	return append(name, n)
}

func (v metaValue) Field(i int) metaValue {
	v2 := v.Value.Field(i)
	f := v.Value.Type().Field(i)
	n := v.buildDeeperName(f.Name)
	return metaValue{Value: v2, Name: n, tag: &f.Tag}
}

// InterfaceValue returns the Value wrapped by v (assuming v is an interface)
func (v metaValue) InterfaceValue() metaValue {
	v2 := reflect.ValueOf(v.Value.Interface())
	n := v.buildDeeperName(fmt.Sprintf("(%v)", v2.Type().Name()))
	return metaValue{Value: v2, Name: n, tag: v.tag}
}

func (v metaValue) Indirect() metaValue {
	return metaValue{Value: reflect.Indirect(v.Value), Name: v.Name, tag: v.tag}
}

func (v metaValue) getChecks() checks {
	var c checks
	if v.tag != nil {
		for _, str := range strings.Split(v.tag.Get("checks"), ",") {
			switch strings.ToLower(str) {
			case "notnil":
				c.NotNil = true
			}
		}
	}
	return c
}

type checks struct {
	NotNil bool
}

// Breadth First Search queue for reflective struct exploration. Prevents infinite recursion by marking pointers and refusing to push marked pointers.
type valueQueue struct {
	queuedPointers map[uintptr]interface{}
	queue          *list.List
}

func newValueQueue() *valueQueue {
	return &valueQueue{
		queuedPointers: make(map[uintptr]interface{}),
		queue:          list.New(),
	}
}

func (q *valueQueue) Push(v metaValue) {
	kind := v.Kind()
	// take internal value of interfaces
	if kind == reflect.Interface {
		v = v.InterfaceValue()
	}
	// mark pointers
	if kind == reflect.Ptr && !v.IsNil() {
		ptr := v.Pointer()
		if _, present := q.queuedPointers[ptr]; present {
			return
		} else {
			q.queuedPointers[ptr] = nil
		}
	}
	// enqueue value
	q.queue.PushBack(v)
	return
}

func (q *valueQueue) Pop() metaValue {
	front := q.queue.Front()
	v := front.Value.(metaValue)
	q.queue.Remove(front)
	return v
}

func (q *valueQueue) Len() int {
	return q.queue.Len()
}

// drills down (follows pointer and interface indirection) to a struct and recursively runs checks on all fields.
func Validate(o interface{}) error {
	if o == nil {
		return ErrorNilValue{}
	}
	top := reflect.ValueOf(o)
	// drill down to first struct
	for k := top.Kind(); k == reflect.Ptr || k == reflect.Interface; k = top.Kind() {
		if top.IsNil() {
			return ErrorNilValue{}
		} else if k == reflect.Ptr {
			top = reflect.Indirect(top)
		} else if k == reflect.Interface {
			top = reflect.ValueOf(top.Interface())
		}
	}

	if top.Kind() != reflect.Struct {
		return ErrorInvalidKind{Kind: top.Kind()}
	}
	// Breadth first search to find nil required fields
	namedTop := metaValue{Value: top, Name: []string{top.Type().Name()}}
	field2checks := make(map[Field][]Check)
	q := newValueQueue()
	q.Push(namedTop)
	for q.Len() > 0 {
		v := q.Pop()
		// check NotNil conditions
		switch v.Kind() {
		case reflect.Ptr, reflect.Interface, reflect.Chan, reflect.Func, reflect.Map, reflect.Slice:
			if v.IsNil() && v.getChecks().NotNil {
				f := newField(v)
				field2checks[f] = append(field2checks[f], NotNil)
			}
		}
		// push new nodes onto queue
		switch v.Kind() {
		case reflect.Ptr:
			if !v.IsNil() {
				q.Push(v.Indirect())
			}
		case reflect.Interface:
			if !v.IsNil() {
				q.Push(v.InterfaceValue())
			}
		case reflect.Struct:
			for i := 0; i < v.NumField(); i++ {
				q.Push(v.Field(i))
			}
		}
	}

	if len(field2checks) != 0 {
		return ErrorChecksFailed{Field2Checks: field2checks}
	} else {
		return nil
	}
}

type Check string

const (
	NotNil Check = "NotNil"
)

type Field struct {
	Name  string
	Value string
}

func newField(v metaValue) Field {
	return Field{Name: strings.Join(v.Name, "."), Value: fmt.Sprintf("%#v", v.Value.Interface())}
}

// returned when the top level object does not drill down to a struct.
type ErrorInvalidKind struct {
	reflect.Kind
}

func (e ErrorInvalidKind) Error() string {
	return fmt.Sprintf("Provided object must drill down to a struct. Received: %v", e.Kind)
}

// returned when a top level nil is received
type ErrorNilValue struct{}

func (e ErrorNilValue) Error() string {
	return fmt.Sprintf("Provided object must drill down to a struct. Encountered nil.")
}

// returned when checks fail on fields
type ErrorChecksFailed struct {
	Field2Checks map[Field][]Check
}

func (e ErrorChecksFailed) Error() string {
	failLines := make([]string, 0, len(e.Field2Checks))
	for field, checks := range e.Field2Checks {
		fails := make([]string, 0, len(checks))
		for _, check := range checks {
			fails = append(fails, string(check))
		}
		failLines = append(failLines, fmt.Sprintf("%v: %v: %v", strings.Join(fails, ", "), field.Name, field.Value))
	}
	return fmt.Sprintf("The following field(s) failed checks: \n\t%v", strings.Join(failLines, "\n\t"))
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package subcommands implements a simple way for a single command to have many
// subcommands, each of which takes arguments and so forth.
package subcommands

import (
	"fmt"
	"io"
	"os"
	"path"
	"sort"

	"flag"
	"golang.org/x/net/context"
)

// A Command represents a single command.
type Command interface {
	// Name returns the name of the command.
	Name() string

	// Synopsis returns a short string (less than one line) describing the command.
	Synopsis() string

	// Usage returns a long string explaining the command and giving usage
	// information.
	Usage() string

	// SetFlags adds the flags for this command to the specified set.
	SetFlags(*flag.FlagSet)

	// Execute executes the command and returns an ExitStatus.
	Execute(ctx context.Context, f *flag.FlagSet, args ...interface{}) ExitStatus
}

// A Commander represents a set of commands.
type Commander struct {
	commands  []*commandGroup
	topFlags  *flag.FlagSet // top-level flags
	important []string      // important top-level flags
	name      string        // normally path.Base(os.Args[0])
}

// A commandGroup represents a set of commands about a common topic.
type commandGroup struct {
	name     string
	commands []Command
}

// An ExitStatus represents a Posix exit status that a subcommand
// expects to be returned to the shell.
type ExitStatus int

const (
	ExitSuccess ExitStatus = iota
	ExitFailure
	ExitUsageError
)

// NewCommander returns a new commander with the specified top-level
// flags and command name. The Usage function for the topLevelFlags
// will be set as well.
func NewCommander(topLevelFlags *flag.FlagSet, name string) *Commander {
	cdr := &Commander{
		topFlags: topLevelFlags,
		name:     name,
	}
	topLevelFlags.Usage = func() { cdr.explain(os.Stderr) }
	return cdr
}

// Register adds a subcommand to the supported subcommands in the
// specified group. (Help output is sorted and arranged by group name.)
// The empty string is an acceptable group name; such subcommands are
// explained first before named groups.
func (cdr *Commander) Register(cmd Command, group string) {
	for _, g := range cdr.commands {
		if g.name == group {
			g.commands = append(g.commands, cmd)
			return
		}
	}
	cdr.commands = append(cdr.commands, &commandGroup{
		name:     group,
		commands: []Command{cmd},
	})
}

// ImportantFlag marks a top-level flag as important, which means it
// will be printed out as part of the output of an ordinary "help"
// subcommand.  (All flags, important or not, are printed by the
// "flags" subcommand.)
func (cdr *Commander) ImportantFlag(name string) {
	cdr.important = append(cdr.important, name)
}

// Execute should be called once the top-level-flags on a Commander
// have been initialized. It finds the correct subcommand and executes
// it, and returns an ExitStatus with the result. On a usage error, an
// appropriate message is printed to os.Stderr, and ExitUsageError is
// returned. The additional args are provided as-is to the Execute method
// of the selected Command.
func (cdr *Commander) Execute(ctx context.Context, args ...interface{}) ExitStatus {
	if cdr.topFlags.NArg() < 1 {
		cdr.topFlags.Usage()
		return ExitUsageError
	}

	name := cdr.topFlags.Arg(0)

	for _, group := range cdr.commands {
		for _, cmd := range group.commands {
			if name != cmd.Name() {
				continue
			}
			f := flag.NewFlagSet(name, flag.ContinueOnError)
			f.Usage = func() { explain(os.Stderr, cmd) }
			cmd.SetFlags(f)
			if f.Parse(cdr.topFlags.Args()[1:]) != nil {
				return ExitUsageError
			}
			return cmd.Execute(ctx, f, args...)
		}
	}

	// Cannot find this command.
	cdr.topFlags.Usage()
	return ExitUsageError
}

// Sorting of a slice of command groups.
type byGroupName []*commandGroup

func (p byGroupName) Len() int           { return len(p) }
func (p byGroupName) Less(i, j int) bool { return p[i].name < p[j].name }
func (p byGroupName) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// explain prints a brief description of all the subcommands and the
// important top-level flags.
func (cdr *Commander) explain(w io.Writer) {
	fmt.Fprintf(w, "Usage: %s <flags> <subcommand> <subcommand args>\n\n", cdr.name)
	sort.Sort(byGroupName(cdr.commands))
	for _, group := range cdr.commands {
		explainGroup(w, group)
	}
	if cdr.topFlags == nil {
		fmt.Fprintln(w, "\nNo top level flags.")
		return
	}
	if len(cdr.important) == 0 {
		fmt.Fprintf(w, "\nUse \"%s flags\" for a list of top-level flags\n", cdr.name)
		return
	}

	fmt.Fprintf(w, "\nTop-level flags (use \"%s flags\" for a full list):\n", cdr.name)
	for _, name := range cdr.important {
		f := cdr.topFlags.Lookup(name)
		if f == nil {
			panic("Important flag is not defined")
		}
		fmt.Fprintf(w, "  -%s=%s: %s\n", f.Name, f.DefValue, f.Usage)
	}
}

// Sorting of the commands within a group.
func (g commandGroup) Len() int           { return len(g.commands) }
func (g commandGroup) Less(i, j int) bool { return g.commands[i].Name() < g.commands[j].Name() }
func (g commandGroup) Swap(i, j int)      { g.commands[i], g.commands[j] = g.commands[j], g.commands[i] }

// explainGroup explains all the subcommands for a particular group.
func explainGroup(w io.Writer, group *commandGroup) {
	if len(group.commands) == 0 {
		return
	}
	if group.name == "" {
		fmt.Fprintf(w, "Subcommands:\n")
	} else {
		fmt.Fprintf(w, "Subcommands for %s:\n", group.name)
	}
	sort.Sort(group)
	for _, cmd := range group.commands {
		fmt.Fprintf(w, "\t%-15s  %s\n", cmd.Name(), cmd.Synopsis())
	}
	fmt.Fprintln(w)
}

// explainCmd prints a brief description of a single command.
func explain(w io.Writer, cmd Command) {
	fmt.Fprintf(w, "%s", cmd.Usage())
	subflags := flag.NewFlagSet(cmd.Name(), flag.PanicOnError)
	subflags.SetOutput(w)
	cmd.SetFlags(subflags)
	subflags.PrintDefaults()
}

// A helper is a Command implementing a "help" command for
// a given Commander.
type helper Commander

func (h *helper) Name() string           { return "help" }
func (h *helper) Synopsis() string       { return "describe subcommands and their syntax" }
func (h *helper) SetFlags(*flag.FlagSet) {}
func (h *helper) Usage() string {
	return `help [<subcommand>]:
	With an argument, prints detailed information on the use of
	the specified subcommand. With no argument, print a list of
	all commands and a brief description of each.
`
}
func (h *helper) Execute(_ context.Context, f *flag.FlagSet, args ...interface{}) ExitStatus {
	switch f.NArg() {
	case 0:
		(*Commander)(h).explain(os.Stdout)
		return ExitSuccess

	case 1:
		for _, group := range h.commands {
			for _, cmd := range group.commands {
				if f.Arg(0) != cmd.Name() {
					continue
				}
				explain(os.Stdout, cmd)
				return ExitSuccess
			}
		}
		fmt.Fprintf(os.Stderr, "Subcommand %s not understood\n", f.Arg(0))
	}

	f.Usage()
	return ExitUsageError
}

// HelpCommand returns a Command which implements a "help" subcommand.
func (cdr *Commander) HelpCommand() Command {
	return (*helper)(cdr)
}

// A flagger is a Command implementing a "flags" command for a given Commander.
type flagger Commander

func (flg *flagger) Name() string           { return "flags" }
func (flg *flagger) Synopsis() string       { return "describe all known top-level flags" }
func (flg *flagger) SetFlags(*flag.FlagSet) {}
func (flg *flagger) Usage() string {
	return `flags [<subcommand>]:
	With an argument, print all flags of <subcommand>. Else,
	print a description of all known top-level flags.  (The basic
	help information only discusses the most generally important
	top-level flags.)
`
}
func (flg *flagger) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) ExitStatus {
	if f.NArg() > 1 {
		f.Usage()
		return ExitUsageError
	}

	if f.NArg() == 0 {
		if flg.topFlags == nil {
			fmt.Println("No top-level flags are defined.")
		} else {
			flg.topFlags.PrintDefaults()
		}
		return ExitSuccess
	}

	for _, group := range flg.commands {
		for _, cmd := range group.commands {
			if f.Arg(0) != cmd.Name() {
				continue
			}
			subflags := flag.NewFlagSet(cmd.Name(), flag.PanicOnError)
			subflags.SetOutput(os.Stdout)
			cmd.SetFlags(subflags)
			subflags.PrintDefaults()
			return ExitSuccess
		}
	}
	fmt.Fprintf(os.Stderr, "Subcommand %s not understood\n", f.Arg(0))
	return ExitFailure
}

// FlagsCommand returns a Command which implements a "flags" subcommand.
func (cdr *Commander) FlagsCommand() Command {
	return (*flagger)(cdr)
}

// A lister is a Command implementing a "commands" command for a given Commander.
type lister Commander

func (l *lister) Name() string           { return "commands" }
func (l *lister) Synopsis() string       { return "list all command names" }
func (l *lister) SetFlags(*flag.FlagSet) {}
func (l *lister) Usage() string {
	return `commands:
	Print a list of all commands.
`
}
func (l *lister) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) ExitStatus {
	if f.NArg() != 0 {
		f.Usage()
		return ExitUsageError
	}

	for _, group := range l.commands {
		for _, cmd := range group.commands {
			fmt.Printf("%s\n", cmd.Name())
		}
	}
	return ExitSuccess
}

// CommandsCommand returns Command which implements a "commands" subcommand.
func (cdr *Commander) CommandsCommand() Command {
	return (*lister)(cdr)
}

// DefaultCommander is the default commander using flag.CommandLine for flags
// and os.Args[0] for the command name.
var DefaultCommander *Commander

func init() {
	DefaultCommander = NewCommander(flag.CommandLine, path.Base(os.Args[0]))
}

// Register adds a subcommand to the supported subcommands in the
// specified group. (Help output is sorted and arranged by group
// name.)  The empty string is an acceptable group name; such
// subcommands are explained first before named groups. It is a
// wrapper around DefaultCommander.Register.
func Register(cmd Command, group string) {
	DefaultCommander.Register(cmd, group)
}

// ImportantFlag marks a top-level flag as important, which means it
// will be printed out as part of the output of an ordinary "help"
// subcommand.  (All flags, important or not, are printed by the
// "flags" subcommand.) It is a wrapper around
// DefaultCommander.ImportantFlag.
func ImportantFlag(name string) {
	DefaultCommander.ImportantFlag(name)
}

// Execute should be called once the default flags have been
// initialized by flag.Parse. It finds the correct subcommand and
// executes it, and returns an ExitStatus with the result. On a usage
// error, an appropriate message is printed to os.Stderr, and
// ExitUsageError is returned. The additional args are provided as-is
// to the Execute method of the selected Command. It is a wrapper
// around DefaultCommander.Execute.
func Execute(ctx context.Context, args ...interface{}) ExitStatus {
	return DefaultCommander.Execute(ctx, args...)
}

// HelpCommand returns a Command which implements "help" for the
// DefaultCommander. Use Register(HelpCommand(), <group>) for it to be
// recognized.
func HelpCommand() Command {
	return DefaultCommander.HelpCommand()
}

// FlagsCommand returns a Command which implements "flags" for the
// DefaultCommander. Use Register(FlagsCommand(), <group>) for it to be
// recognized.
func FlagsCommand() Command {
	return DefaultCommander.FlagsCommand()
}

// CommandsCommand returns Command which implements a "commands" subcommand.
func CommandsCommand() Command {
	return DefaultCommander.CommandsCommand()
}
<commit_msg>Fix formatting<commit_after>/*
Copyright 2016 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package subcommands implements a simple way for a single command to have many
// subcommands, each of which takes arguments and so forth.
package subcommands

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path"
	"sort"

	"golang.org/x/net/context"
)

// A Command represents a single command.
type Command interface {
	// Name returns the name of the command.
	Name() string

	// Synopsis returns a short string (less than one line) describing the command.
	Synopsis() string

	// Usage returns a long string explaining the command and giving usage
	// information.
	Usage() string

	// SetFlags adds the flags for this command to the specified set.
	SetFlags(*flag.FlagSet)

	// Execute executes the command and returns an ExitStatus.
	Execute(ctx context.Context, f *flag.FlagSet, args ...interface{}) ExitStatus
}

// A Commander represents a set of commands.
type Commander struct {
	commands  []*commandGroup
	topFlags  *flag.FlagSet // top-level flags
	important []string      // important top-level flags
	name      string        // normally path.Base(os.Args[0])
}

// A commandGroup represents a set of commands about a common topic.
type commandGroup struct {
	name     string
	commands []Command
}

// An ExitStatus represents a Posix exit status that a subcommand
// expects to be returned to the shell.
type ExitStatus int

const (
	ExitSuccess ExitStatus = iota
	ExitFailure
	ExitUsageError
)

// NewCommander returns a new commander with the specified top-level
// flags and command name. The Usage function for the topLevelFlags
// will be set as well.
func NewCommander(topLevelFlags *flag.FlagSet, name string) *Commander {
	cdr := &Commander{
		topFlags: topLevelFlags,
		name:     name,
	}
	topLevelFlags.Usage = func() { cdr.explain(os.Stderr) }
	return cdr
}

// Register adds a subcommand to the supported subcommands in the
// specified group. (Help output is sorted and arranged by group name.)
// The empty string is an acceptable group name; such subcommands are
// explained first before named groups.
func (cdr *Commander) Register(cmd Command, group string) {
	for _, g := range cdr.commands {
		if g.name == group {
			g.commands = append(g.commands, cmd)
			return
		}
	}
	cdr.commands = append(cdr.commands, &commandGroup{
		name:     group,
		commands: []Command{cmd},
	})
}

// ImportantFlag marks a top-level flag as important, which means it
// will be printed out as part of the output of an ordinary "help"
// subcommand.  (All flags, important or not, are printed by the
// "flags" subcommand.)
func (cdr *Commander) ImportantFlag(name string) {
	cdr.important = append(cdr.important, name)
}

// Execute should be called once the top-level-flags on a Commander
// have been initialized. It finds the correct subcommand and executes
// it, and returns an ExitStatus with the result. On a usage error, an
// appropriate message is printed to os.Stderr, and ExitUsageError is
// returned. The additional args are provided as-is to the Execute method
// of the selected Command.
func (cdr *Commander) Execute(ctx context.Context, args ...interface{}) ExitStatus {
	if cdr.topFlags.NArg() < 1 {
		cdr.topFlags.Usage()
		return ExitUsageError
	}

	name := cdr.topFlags.Arg(0)

	for _, group := range cdr.commands {
		for _, cmd := range group.commands {
			if name != cmd.Name() {
				continue
			}
			f := flag.NewFlagSet(name, flag.ContinueOnError)
			f.Usage = func() { explain(os.Stderr, cmd) }
			cmd.SetFlags(f)
			if f.Parse(cdr.topFlags.Args()[1:]) != nil {
				return ExitUsageError
			}
			return cmd.Execute(ctx, f, args...)
		}
	}

	// Cannot find this command.
	cdr.topFlags.Usage()
	return ExitUsageError
}

// Sorting of a slice of command groups.
type byGroupName []*commandGroup

func (p byGroupName) Len() int           { return len(p) }
func (p byGroupName) Less(i, j int) bool { return p[i].name < p[j].name }
func (p byGroupName) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// explain prints a brief description of all the subcommands and the
// important top-level flags.
func (cdr *Commander) explain(w io.Writer) {
	fmt.Fprintf(w, "Usage: %s <flags> <subcommand> <subcommand args>\n\n", cdr.name)
	sort.Sort(byGroupName(cdr.commands))
	for _, group := range cdr.commands {
		explainGroup(w, group)
	}
	if cdr.topFlags == nil {
		fmt.Fprintln(w, "\nNo top level flags.")
		return
	}
	if len(cdr.important) == 0 {
		fmt.Fprintf(w, "\nUse \"%s flags\" for a list of top-level flags\n", cdr.name)
		return
	}

	fmt.Fprintf(w, "\nTop-level flags (use \"%s flags\" for a full list):\n", cdr.name)
	for _, name := range cdr.important {
		f := cdr.topFlags.Lookup(name)
		if f == nil {
			panic("Important flag is not defined")
		}
		fmt.Fprintf(w, "  -%s=%s: %s\n", f.Name, f.DefValue, f.Usage)
	}
}

// Sorting of the commands within a group.
func (g commandGroup) Len() int           { return len(g.commands) }
func (g commandGroup) Less(i, j int) bool { return g.commands[i].Name() < g.commands[j].Name() }
func (g commandGroup) Swap(i, j int)      { g.commands[i], g.commands[j] = g.commands[j], g.commands[i] }

// explainGroup explains all the subcommands for a particular group.
func explainGroup(w io.Writer, group *commandGroup) {
	if len(group.commands) == 0 {
		return
	}
	if group.name == "" {
		fmt.Fprintf(w, "Subcommands:\n")
	} else {
		fmt.Fprintf(w, "Subcommands for %s:\n", group.name)
	}
	sort.Sort(group)
	for _, cmd := range group.commands {
		fmt.Fprintf(w, "\t%-15s  %s\n", cmd.Name(), cmd.Synopsis())
	}
	fmt.Fprintln(w)
}

// explainCmd prints a brief description of a single command.
func explain(w io.Writer, cmd Command) {
	fmt.Fprintf(w, "%s", cmd.Usage())
	subflags := flag.NewFlagSet(cmd.Name(), flag.PanicOnError)
	subflags.SetOutput(w)
	cmd.SetFlags(subflags)
	subflags.PrintDefaults()
}

// A helper is a Command implementing a "help" command for
// a given Commander.
type helper Commander

func (h *helper) Name() string           { return "help" }
func (h *helper) Synopsis() string       { return "describe subcommands and their syntax" }
func (h *helper) SetFlags(*flag.FlagSet) {}
func (h *helper) Usage() string {
	return `help [<subcommand>]:
	With an argument, prints detailed information on the use of
	the specified subcommand. With no argument, print a list of
	all commands and a brief description of each.
`
}
func (h *helper) Execute(_ context.Context, f *flag.FlagSet, args ...interface{}) ExitStatus {
	switch f.NArg() {
	case 0:
		(*Commander)(h).explain(os.Stdout)
		return ExitSuccess

	case 1:
		for _, group := range h.commands {
			for _, cmd := range group.commands {
				if f.Arg(0) != cmd.Name() {
					continue
				}
				explain(os.Stdout, cmd)
				return ExitSuccess
			}
		}
		fmt.Fprintf(os.Stderr, "Subcommand %s not understood\n", f.Arg(0))
	}

	f.Usage()
	return ExitUsageError
}

// HelpCommand returns a Command which implements a "help" subcommand.
func (cdr *Commander) HelpCommand() Command {
	return (*helper)(cdr)
}

// A flagger is a Command implementing a "flags" command for a given Commander.
type flagger Commander

func (flg *flagger) Name() string           { return "flags" }
func (flg *flagger) Synopsis() string       { return "describe all known top-level flags" }
func (flg *flagger) SetFlags(*flag.FlagSet) {}
func (flg *flagger) Usage() string {
	return `flags [<subcommand>]:
	With an argument, print all flags of <subcommand>. Else,
	print a description of all known top-level flags.  (The basic
	help information only discusses the most generally important
	top-level flags.)
`
}
func (flg *flagger) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) ExitStatus {
	if f.NArg() > 1 {
		f.Usage()
		return ExitUsageError
	}

	if f.NArg() == 0 {
		if flg.topFlags == nil {
			fmt.Println("No top-level flags are defined.")
		} else {
			flg.topFlags.PrintDefaults()
		}
		return ExitSuccess
	}

	for _, group := range flg.commands {
		for _, cmd := range group.commands {
			if f.Arg(0) != cmd.Name() {
				continue
			}
			subflags := flag.NewFlagSet(cmd.Name(), flag.PanicOnError)
			subflags.SetOutput(os.Stdout)
			cmd.SetFlags(subflags)
			subflags.PrintDefaults()
			return ExitSuccess
		}
	}
	fmt.Fprintf(os.Stderr, "Subcommand %s not understood\n", f.Arg(0))
	return ExitFailure
}

// FlagsCommand returns a Command which implements a "flags" subcommand.
func (cdr *Commander) FlagsCommand() Command {
	return (*flagger)(cdr)
}

// A lister is a Command implementing a "commands" command for a given Commander.
type lister Commander

func (l *lister) Name() string           { return "commands" }
func (l *lister) Synopsis() string       { return "list all command names" }
func (l *lister) SetFlags(*flag.FlagSet) {}
func (l *lister) Usage() string {
	return `commands:
	Print a list of all commands.
`
}
func (l *lister) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) ExitStatus {
	if f.NArg() != 0 {
		f.Usage()
		return ExitUsageError
	}

	for _, group := range l.commands {
		for _, cmd := range group.commands {
			fmt.Printf("%s\n", cmd.Name())
		}
	}
	return ExitSuccess
}

// CommandsCommand returns Command which implements a "commands" subcommand.
func (cdr *Commander) CommandsCommand() Command {
	return (*lister)(cdr)
}

// DefaultCommander is the default commander using flag.CommandLine for flags
// and os.Args[0] for the command name.
var DefaultCommander *Commander

func init() {
	DefaultCommander = NewCommander(flag.CommandLine, path.Base(os.Args[0]))
}

// Register adds a subcommand to the supported subcommands in the
// specified group. (Help output is sorted and arranged by group
// name.)  The empty string is an acceptable group name; such
// subcommands are explained first before named groups. It is a
// wrapper around DefaultCommander.Register.
func Register(cmd Command, group string) {
	DefaultCommander.Register(cmd, group)
}

// ImportantFlag marks a top-level flag as important, which means it
// will be printed out as part of the output of an ordinary "help"
// subcommand.  (All flags, important or not, are printed by the
// "flags" subcommand.) It is a wrapper around
// DefaultCommander.ImportantFlag.
func ImportantFlag(name string) {
	DefaultCommander.ImportantFlag(name)
}

// Execute should be called once the default flags have been
// initialized by flag.Parse. It finds the correct subcommand and
// executes it, and returns an ExitStatus with the result. On a usage
// error, an appropriate message is printed to os.Stderr, and
// ExitUsageError is returned. The additional args are provided as-is
// to the Execute method of the selected Command. It is a wrapper
// around DefaultCommander.Execute.
func Execute(ctx context.Context, args ...interface{}) ExitStatus {
	return DefaultCommander.Execute(ctx, args...)
}

// HelpCommand returns a Command which implements "help" for the
// DefaultCommander. Use Register(HelpCommand(), <group>) for it to be
// recognized.
func HelpCommand() Command {
	return DefaultCommander.HelpCommand()
}

// FlagsCommand returns a Command which implements "flags" for the
// DefaultCommander. Use Register(FlagsCommand(), <group>) for it to be
// recognized.
func FlagsCommand() Command {
	return DefaultCommander.FlagsCommand()
}

// CommandsCommand returns Command which implements a "commands" subcommand.
func CommandsCommand() Command {
	return DefaultCommander.CommandsCommand()
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// The camnetdns server serves camlistore.net's DNS server and its
// DNS challenges
package main

import (
	"errors"
	"flag"
	"log"
	"net"
	"strings"

	"camlistore.org/pkg/sorted"

	"cloud.google.com/go/compute/metadata"
	"cloud.google.com/go/datastore"
	"cloud.google.com/go/logging"
	"github.com/miekg/dns"
	"go4.org/cloud/cloudlaunch"
	compute "google.golang.org/api/compute/v1"
)

var flagServerIP = flag.String("server_ip", "104.154.231.160", "The IP address of the authoritative name server for camlistore.net, i.e. the address where this program will run.")

// TODO(mpl): pass the server ip to the launchConfig, so we create the instance
// with this specific IP. Which means, we'll have to book it as a static address in
// Google Cloud I suppose?
// Or, we hope we're lucky and we never have to destroy the camnet-dns VM (and lose
// its current IP)?

var launchConfig = &cloudlaunch.Config{
	Name:         "camnetdns",
	BinaryBucket: "camlistore-dnsserver-resource",
	GCEProjectID: "camlistore-website",
	Scopes: []string{
		compute.ComputeScope,
		logging.Scope,
		datastore.ScopeDatastore,
	},
}

// DefaultResponseTTL is the record TTL in seconds
const DefaultResponseTTL = 300

var ErrRecordNotFound = errors.New("record not found")

func defaultListenAddr() string {
	if metadata.OnGCE() {
		return ":53"
	}
	return ":5300"
}

// DNSServer implements the dns.Handler interface to serve A and AAAA
// records using a sorted.KeyValue for the lookups.
type DNSServer struct {
	dataSource sorted.KeyValue
}

func NewDNSServer(src sorted.KeyValue) *DNSServer {
	return &DNSServer{
		dataSource: src,
	}
}

func (ds *DNSServer) HandleLookup(name string) (string, error) {
	// Lowercase it all, to satisfy https://tools.ietf.org/html/draft-vixie-dnsext-dns0x20-00
	return ds.dataSource.Get(strings.ToLower(name))
}

const (
	domain      = "camlistore.net."
	authorityNS = "camnetdns.camlistore.org."
	// Increment after every change with format YYYYMMDDnn.
	soaSerial = 2016102101
)

var (
	authoritySection = &dns.NS{
		Ns: authorityNS,
		Hdr: dns.RR_Header{
			Name:   domain,
			Rrtype: dns.TypeNS,
			Class:  dns.ClassINET,
			Ttl:    DefaultResponseTTL,
		},
	}
	additionalSection = &dns.A{
		A: net.ParseIP(*flagServerIP),
		Hdr: dns.RR_Header{
			Name:   authorityNS,
			Rrtype: dns.TypeA,
			Class:  dns.ClassINET,
			Ttl:    DefaultResponseTTL,
		},
	}
	startOfAuthoritySection = &dns.SOA{
		Hdr: dns.RR_Header{
			Name:   domain,
			Rrtype: dns.TypeSOA,
			Class:  dns.ClassINET,
			Ttl:    DefaultResponseTTL,
		},
		Ns:      authorityNS,
		Mbox:    "admin.camlistore.org.",
		Serial:  soaSerial,
		Refresh: 3600, // TODO(mpl): set them lower once we got everything right.
		Retry:   3600,
		Expire:  86500,
		Minttl:  DefaultResponseTTL,
	}
)

func commonHeader(q dns.Question) dns.RR_Header {
	return dns.RR_Header{
		Name:   q.Name,
		Rrtype: q.Qtype,
		Class:  dns.ClassINET,
		Ttl:    DefaultResponseTTL,
	}
}

func (ds *DNSServer) ServeDNS(rw dns.ResponseWriter, mes *dns.Msg) {
	resp := new(dns.Msg)

	if mes.IsEdns0() != nil {
		// Because apparently, if we're not going to handle EDNS
		// properly, i.e. by returning an OPT section as well, we should
		// return an RcodeFormatError. Not seen in the RFC, but doing that
		// addresses some of the warnings from
		// http://dnsviz.net/d/granivo.re/dnssec/
		log.Print("unhandled EDNS message\n")
		resp.SetRcode(mes, dns.RcodeFormatError)
		if err := rw.WriteMsg(resp); err != nil {
			log.Printf("error responding to DNS query: %s", err)
		}
		return
	}
	resp.SetReply(mes)
	// TODO(mpl): Should we make sure that at least q.Name ends in
	// "camlistore.net" before claiming we're authoritative on that response?
	resp.Authoritative = true

	for _, q := range mes.Question {
		log.Printf("DNS request from %s: %s", rw.RemoteAddr(), &q)

		answer, err := ds.HandleLookup(q.Name)
		if err == sorted.ErrNotFound {
			resp.SetRcode(mes, dns.RcodeNameError)
			if err := rw.WriteMsg(resp); err != nil {
				log.Printf("error responding to DNS query: %s", err)
			}
			return
		}
		if err != nil {
			log.Printf("error looking up %q: %v", q.Name, err)
			continue
		}

		if q.Qclass != dns.ClassINET {
			log.Printf("error: got invalid DNS question class %d\n", q.Qclass)
			continue
		}

		switch q.Qtype {
		// As long as we send a reply (even an empty one), we apparently
		// look compliant. Or at least more than if we replied with
		// RcodeNotImplemented.
		case dns.TypeDNSKEY, dns.TypeTXT, dns.TypeMX:
			break

		case dns.TypeSOA:
			resp.Answer = []dns.RR{startOfAuthoritySection}
			resp.Extra = []dns.RR{additionalSection}

		case dns.TypeNS:
			resp.Answer = []dns.RR{authoritySection}
			resp.Extra = []dns.RR{additionalSection}

		case dns.TypeCAA:
			header := commonHeader(q)
			rr := &dns.CAA{
				Hdr:   header,
				Flag:  1,
				Tag:   "issue",
				Value: "letsencrypt.org",
			}
			resp.Answer = []dns.RR{rr}

		case dns.TypeA, dns.TypeAAAA:
			val := answer
			ip := net.ParseIP(val)
			// TODO(mpl): maybe we should have a distinct memstore for each type?
			isIP6 := strings.Contains(ip.String(), ":")
			header := commonHeader(q)
			var rr dns.RR
			if q.Qtype == dns.TypeA {
				if isIP6 {
					break
				}
				rr = &dns.A{
					A:   ip,
					Hdr: header,
				}
			} else if q.Qtype == dns.TypeAAAA {
				if !isIP6 {
					break
				}
				rr = &dns.AAAA{
					AAAA: ip,
					Hdr:  header,
				}
			} else {
				panic("unreachable")
			}
			resp.Answer = []dns.RR{rr}
			// Not necessary, but I think they help.
			resp.Ns = []dns.RR{authoritySection}
			resp.Extra = []dns.RR{additionalSection}

		default:
			log.Printf("unhandled qtype: %d\n", q.Qtype)
			resp.SetRcode(mes, dns.RcodeNotImplemented)
			if err := rw.WriteMsg(resp); err != nil {
				log.Printf("error responding to DNS query: %s", err)
			}
			return
		}
		break
	}

	if err := rw.WriteMsg(resp); err != nil {
		log.Printf("error responding to DNS query: %s", err)
	}
}

func main() {
	launchConfig.MaybeDeploy()
	addr := flag.String("addr", defaultListenAddr(), "specify address for server to listen on")
	flag.Parse()

	memkv := sorted.NewMemoryKeyValue()
	if err := memkv.Set("6401800c.camlistore.net.", "159.203.246.79"); err != nil {
		panic(err)
	}
	if err := memkv.Set(domain, *flagServerIP); err != nil {
		panic(err)
	}
	if err := memkv.Set("www.camlistore.net.", *flagServerIP); err != nil {
		panic(err)
	}
	if err := memkv.Set("wip.camlistore.net.", "104.199.42.193"); err != nil {
		panic(err)
	}

	ds := NewDNSServer(memkv)

	log.Printf("serving DNS on %s\n", *addr)
	tcperr := make(chan error, 1)
	udperr := make(chan error, 1)
	go func() {
		tcperr <- dns.ListenAndServe(*addr, "tcp", ds)
	}()
	go func() {
		udperr <- dns.ListenAndServe(*addr, "udp", ds)
	}()
	select {
	case err := <-tcperr:
		log.Fatalf("DNS over TCP error: %v", err)
	case err := <-udperr:
		log.Fatalf("DNS error: %v", err)
	}
}
<commit_msg>server/camnetdns: persist records in datastore<commit_after>/*
Copyright 2016 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// The camnetdns server serves camlistore.net's DNS server and its
// DNS challenges
package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"strings"
	"time"

	"camlistore.org/pkg/gpgchallenge"
	"camlistore.org/pkg/lru"
	"camlistore.org/pkg/sorted"

	"cloud.google.com/go/compute/metadata"
	"cloud.google.com/go/datastore"
	"cloud.google.com/go/logging"
	"github.com/miekg/dns"
	"go4.org/cloud/cloudlaunch"
	"golang.org/x/net/context"
	compute "google.golang.org/api/compute/v1"
)

var (
	addr         = flag.String("addr", defaultListenAddr(), "specify address for server to listen on")
	flagServerIP = flag.String("server_ip", "104.154.231.160", "The IP address of the authoritative name server for camlistore.net, i.e. the address where this program will run.")
)

var launchConfig = &cloudlaunch.Config{
	Name:         "camnetdns",
	BinaryBucket: "camlistore-dnsserver-resource",
	GCEProjectID: GCEProjectID,
	Scopes: []string{
		compute.ComputeScope,
		logging.Scope,
		datastore.ScopeDatastore,
	},
}

const (
	GCEProjectID = "camlistore-website"
	// DefaultResponseTTL is the record TTL in seconds
	DefaultResponseTTL = 300
	// even if record already existed in store, we overwrite it if it is older than 30 days, for analytics.
	staleRecord = 30 * 24 * time.Hour
	// max number of records in the lru cache
	cacheSize = 1e6
)

var errRecordNotFound = errors.New("record not found")

func defaultListenAddr() string {
	if metadata.OnGCE() {
		return ":53"
	}
	return ":5300"
}

type keyValue interface {
	// Get fetches the value for key. It returns errRecordNotFound when
	// there is no such record.
	Get(key string) (string, error)
	Set(key, value string) error
}

// cachedStore is a keyValue implementation that stores in Google's datastore
// with dsClient. It automatically stores to cache as well on writes, and always
// tries to read from cache first.
type cachedStore struct {
	// datastore client to store the records. It should not be nil.
	dsClient *datastore.Client
	// cache stores the most recent records. It should not be nil.
	cache *lru.Cache
}

// dsValue is the value type written to the datastore
type dsValue struct {
	// Record is the RHS of an A or AAAA DNS record, i.e. an IPV4 or IPV6
	// address.
	Record string
	// Updated is the last time this key value pair was inserted. Values
	// older than 30 days are rewritten on writes.
	Updated time.Time
}

func (cs cachedStore) Get(key string) (string, error) {
	val, ok := cs.cache.Get(key)
	if ok {
		return val.(string), nil
	}
	// Cache Miss. hit the datastore.
	ctx := context.Background()
	dk := datastore.NewKey(ctx, "camnetdns", key, 0, nil)
	var value dsValue
	if err := cs.dsClient.Get(ctx, dk, &value); err != nil {
		if err != datastore.ErrNoSuchEntity {
			return "", fmt.Errorf("error getting value for %q from datastore: %v", key, err)
		}
		return "", errRecordNotFound
	}
	// And cache it.
	cs.cache.Add(key, value.Record)
	return value.Record, nil
}

func (cs cachedStore) put(ctx context.Context, key, value string) error {
	dk := datastore.NewKey(ctx, "camnetdns", key, 0, nil)
	val := &dsValue{
		Record:  value,
		Updated: time.Now(),
	}
	if _, err := cs.dsClient.Put(ctx, dk, val); err != nil {
		return fmt.Errorf("error writing (%q : %q) record to datastore: %v", key, value, err)
	}
	// and cache it.
	cs.cache.Add(key, value)
	return nil
}

// Set writes the key, value pair to cs. But it does not actually write if the
// value already exists, is up to date, and is more recent than 30 days.
func (cs cachedStore) Set(key, value string) error {
	// check if record already exists
	ctx := context.Background()
	dk := datastore.NewKey(ctx, "camnetdns", key, 0, nil)
	var oldValue dsValue
	if err := cs.dsClient.Get(ctx, dk, &oldValue); err != nil {
		if err != datastore.ErrNoSuchEntity {
			return fmt.Errorf("error checking if record exists for %q from datastore: %v", key, err)
		}
		// record does not exist, write it.
		return cs.put(ctx, key, value)
	}
	// record already exists
	if oldValue.Record != value {
		// new record is different, overwrite old one.
		return cs.put(ctx, key, value)
	}
	// record is the same as before
	if oldValue.Updated.Add(staleRecord).After(time.Now()) {
		// record is still fresh, nothing to do.
		return nil
	}
	// record is older than 30 days, so we rewrite it, for analytics.
	return cs.put(ctx, key, value)
}

type memkv struct {
	skv sorted.KeyValue
}

func (kv memkv) Get(key string) (string, error) {
	val, err := kv.skv.Get(key)
	if err != nil {
		if err != sorted.ErrNotFound {
			return "", err
		}
		return "", errRecordNotFound
	}
	return val, nil
}

func (kv memkv) Set(key, value string) error {
	return kv.skv.Set(key, value)
}

// DNSServer implements the dns.Handler interface to serve A and AAAA
// records, using a KeyValue store for the lookups.
type DNSServer struct {
	dataSource keyValue
}

func newDNSServer(src keyValue) *DNSServer {
	return &DNSServer{
		dataSource: src,
	}
}

func (ds *DNSServer) HandleLookup(name string) (string, error) {
	// Lowercase it all, to satisfy https://tools.ietf.org/html/draft-vixie-dnsext-dns0x20-00
	return ds.dataSource.Get(strings.ToLower(name))
}

const (
	domain      = "camlistore.net."
	authorityNS = "camnetdns.camlistore.org."
	// Increment after every change with format YYYYMMDDnn.
	soaSerial = 2016121401
)

var (
	authoritySection = &dns.NS{
		Ns: authorityNS,
		Hdr: dns.RR_Header{
			Name:   domain,
			Rrtype: dns.TypeNS,
			Class:  dns.ClassINET,
			Ttl:    DefaultResponseTTL,
		},
	}
	additionalSection = &dns.A{
		A: net.ParseIP(*flagServerIP),
		Hdr: dns.RR_Header{
			Name:   authorityNS,
			Rrtype: dns.TypeA,
			Class:  dns.ClassINET,
			Ttl:    DefaultResponseTTL,
		},
	}
	startOfAuthoritySection = &dns.SOA{
		Hdr: dns.RR_Header{
			Name:   domain,
			Rrtype: dns.TypeSOA,
			Class:  dns.ClassINET,
			Ttl:    DefaultResponseTTL,
		},
		Ns:      authorityNS,
		Mbox:    "admin.camlistore.org.",
		Serial:  soaSerial,
		Refresh: 3600, // TODO(mpl): set them lower once we got everything right.
		Retry:   3600,
		Expire:  86500,
		Minttl:  DefaultResponseTTL,
	}
)

func commonHeader(q dns.Question) dns.RR_Header {
	return dns.RR_Header{
		Name:   q.Name,
		Rrtype: q.Qtype,
		Class:  dns.ClassINET,
		Ttl:    DefaultResponseTTL,
	}
}

func (ds *DNSServer) ServeDNS(rw dns.ResponseWriter, mes *dns.Msg) {
	resp := new(dns.Msg)

	if mes.IsEdns0() != nil {
		// Because apparently, if we're not going to handle EDNS
		// properly, i.e. by returning an OPT section as well, we should
		// return an RcodeFormatError. Not seen in the RFC, but doing that
		// addresses some of the warnings from
		// http://dnsviz.net/d/granivo.re/dnssec/
		log.Print("unhandled EDNS message\n")
		resp.SetRcode(mes, dns.RcodeFormatError)
		if err := rw.WriteMsg(resp); err != nil {
			log.Printf("error responding to DNS query: %s", err)
		}
		return
	}
	resp.SetReply(mes)
	// TODO(mpl): Should we make sure that at least q.Name ends in
	// "camlistore.net" before claiming we're authoritative on that response?
	resp.Authoritative = true

	for _, q := range mes.Question {
		log.Printf("DNS request from %s: %s", rw.RemoteAddr(), &q)

		answer, err := ds.HandleLookup(q.Name)
		if err == errRecordNotFound {
			resp.SetRcode(mes, dns.RcodeNameError)
			if err := rw.WriteMsg(resp); err != nil {
				log.Printf("error responding to DNS query: %s", err)
			}
			return
		}
		if err != nil {
			log.Printf("error looking up %q: %v", q.Name, err)
			continue
		}

		if q.Qclass != dns.ClassINET {
			log.Printf("error: got invalid DNS question class %d\n", q.Qclass)
			continue
		}

		switch q.Qtype {
		// As long as we send a reply (even an empty one), we apparently
		// look compliant. Or at least more than if we replied with
		// RcodeNotImplemented.
		case dns.TypeDNSKEY, dns.TypeTXT, dns.TypeMX:
			break

		case dns.TypeSOA:
			resp.Answer = []dns.RR{startOfAuthoritySection}
			resp.Extra = []dns.RR{additionalSection}

		case dns.TypeNS:
			resp.Answer = []dns.RR{authoritySection}
			resp.Extra = []dns.RR{additionalSection}

		case dns.TypeCAA:
			header := commonHeader(q)
			rr := &dns.CAA{
				Hdr:   header,
				Flag:  1,
				Tag:   "issue",
				Value: "letsencrypt.org",
			}
			resp.Answer = []dns.RR{rr}

		case dns.TypeA, dns.TypeAAAA:
			val := answer
			ip := net.ParseIP(val)
			// TODO(mpl): maybe we should have a distinct memstore for each type?
			isIP6 := strings.Contains(ip.String(), ":")
			header := commonHeader(q)
			var rr dns.RR
			if q.Qtype == dns.TypeA {
				if isIP6 {
					break
				}
				rr = &dns.A{
					A:   ip,
					Hdr: header,
				}
			} else if q.Qtype == dns.TypeAAAA {
				if !isIP6 {
					break
				}
				rr = &dns.AAAA{
					AAAA: ip,
					Hdr:  header,
				}
			} else {
				panic("unreachable")
			}
			resp.Answer = []dns.RR{rr}
			// Not necessary, but I think they help.
			resp.Ns = []dns.RR{authoritySection}
			resp.Extra = []dns.RR{additionalSection}

		default:
			log.Printf("unhandled qtype: %d\n", q.Qtype)
			resp.SetRcode(mes, dns.RcodeNotImplemented)
			if err := rw.WriteMsg(resp); err != nil {
				log.Printf("error responding to DNS query: %s", err)
			}
			return
		}
		break
	}

	if err := rw.WriteMsg(resp); err != nil {
		log.Printf("error responding to DNS query: %s", err)
	}
}

func main() {
	launchConfig.MaybeDeploy()
	flag.Parse()

	var kv keyValue
	var httpsListenAddr string
	if metadata.OnGCE() {
		httpsListenAddr = ":443"
		dsClient, err := datastore.NewClient(context.Background(), GCEProjectID)
		if err != nil {
			log.Fatalf("Error creating datastore client for records: %v", err)
		}
		kv = cachedStore{
			dsClient: dsClient,
			cache:    lru.New(cacheSize),
		}
	} else {
		httpsListenAddr = ":4430"
		kv = memkv{skv: sorted.NewMemoryKeyValue()}
	}
	if err := kv.Set("6401800c.camlistore.net.", "159.203.246.79"); err != nil {
		log.Fatalf("Error adding %v:%v record", "6401800c.camlistore.net.", "159.203.246.79")
	}
	if err := kv.Set(domain, *flagServerIP); err != nil {
		log.Fatalf("Error adding %v:%v record", domain, *flagServerIP)
	}
	if err := kv.Set("www.camlistore.net.", *flagServerIP); err != nil {
		log.Fatalf("Error adding %v:%v record", "www.camlistore.net.", *flagServerIP)
	}

	ds := newDNSServer(kv)
	cs := &gpgchallenge.Server{
		OnSuccess: func(identity string, address string) error {
			log.Printf("Adding %v.camlistore.net. as %v", identity, address)
			return ds.dataSource.Set(strings.ToLower(identity+".camlistore.net."), address)
		},
	}

	tcperr := make(chan error, 1)
	udperr := make(chan error, 1)
	httperr := make(chan error, 1)
	log.Printf("serving DNS on %s\n", *addr)
	go func() {
		tcperr <- dns.ListenAndServe(*addr, "tcp", ds)
	}()
	go func() {
		udperr <- dns.ListenAndServe(*addr, "udp", ds)
	}()
	go func() {
		// TODO(mpl): get a cert for camnetdns.camlistore.org from Let's
		// Encrypt. Then we can switch to ListenAndServeTLS here. Next CL.
		httperr <- http.ListenAndServe(httpsListenAddr, cs)
	}()
	select {
	case err := <-tcperr:
		log.Fatalf("DNS over TCP error: %v", err)
	case err := <-udperr:
		log.Fatalf("DNS error: %v", err)
	case err := <-httperr:
		log.Fatalf("HTTP server error: %v", err)
	}
}
<|endoftext|>


<commit_before>package controllers

import (
	"net/http"
	"strconv"

	"github.com/martini-contrib/render"

	"github.com/l2x/wolffy/server/models"
)

type Cluster struct{}

func (c Cluster) GetAll(r render.Render, req *http.Request) {
	res := NewRes()

	clusters, err := models.ClusterModel.GetAll()
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, clusters)
}

func (c Cluster) Get(r render.Render, req *http.Request) {
	res := NewRes()

	id := req.URL.Query().Get("id")
	idint, err := strconv.Atoi(id)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	cluster, err := models.ClusterModel.GetOne(idint)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, cluster)
}

func (c Cluster) Add(r render.Render, req *http.Request) {
	res := NewRes()

	name := req.URL.Query().Get("name")
	tags := req.URL.Query().Get("tags")
	machines := req.URL.Query().Get("machines")
	note := req.URL.Query().Get("note")

	cluster, err := models.ClusterModel.Add(name, tags, machines, note)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, cluster)
}

func (c Cluster) Del(r render.Render, req *http.Request) {
	res := NewRes()
	id := req.URL.Query().Get("id")
	idint, err := strconv.Atoi(id)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	err = models.ClusterModel.Del(idint)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, map[string]string{})
}

func (c Cluster) Update(r render.Render, req *http.Request) {
	res := NewRes()
	name := req.URL.Query().Get("name")
	tags := req.URL.Query().Get("tags")
	machines := req.URL.Query().Get("machines")
	note := req.URL.Query().Get("note")
	id := req.URL.Query().Get("id")

	idint, err := strconv.Atoi(id)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	cluster, err := models.ClusterModel.Update(idint, name, tags, machines, note)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, cluster)
}
<commit_msg>update<commit_after>package controllers

import (
	"net/http"
	"strconv"

	"github.com/martini-contrib/render"

	"github.com/l2x/wolffy/server/models"
)

type Cluster struct{}

func (c Cluster) GetAll(r render.Render, req *http.Request) {
	res := NewRes()

	clusters, err := models.ClusterModel.GetAll()
	if err != nil {
		RenderError(r, 1, err.Error())
		return
	}
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, clusters)
}

func (c Cluster) Get(r render.Render, req *http.Request) {
	res := NewRes()

	id := req.URL.Query().Get("id")
	idint, err := strconv.Atoi(id)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	cluster, err := models.ClusterModel.GetOne(idint)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, cluster)
}

func (c Cluster) Add(r render.Render, req *http.Request) {
	res := NewRes()

	name := req.URL.Query().Get("name")
	tags := req.URL.Query().Get("tags")
	machines := req.URL.Query().Get("machines")
	note := req.URL.Query().Get("note")

	cluster, err := models.ClusterModel.Add(name, tags, machines, note)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, cluster)
}

func (c Cluster) Del(r render.Render, req *http.Request) {
	res := NewRes()
	id := req.URL.Query().Get("id")
	idint, err := strconv.Atoi(id)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	err = models.ClusterModel.Del(idint)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, map[string]string{})
}

func (c Cluster) Update(r render.Render, req *http.Request) {
	res := NewRes()
	name := req.URL.Query().Get("name")
	tags := req.URL.Query().Get("tags")
	machines := req.URL.Query().Get("machines")
	note := req.URL.Query().Get("note")
	id := req.URL.Query().Get("id")

	idint, err := strconv.Atoi(id)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	cluster, err := models.ClusterModel.Update(idint, name, tags, machines, note)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, cluster)
}
<|endoftext|>


<commit_before>// Copyright 2017 zebra project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bgp

// ip community-list standard WORD {permit, deny} 100:1 100:2 no-export
// ip community-list expanded WORD {permit, deny} _100:1_

import (
	"fmt"

	"github.com/coreswitch/zebra/policy"
)

type CommunityListType int

const (
	CommunityListStandard CommunityListType = iota
	CommunityListExpanded
)

type CommunityListMap map[string]*CommunityList

type CommunityList struct {
	Type  CommunityListType
	Name  string
	Entry []*CommunityListEntry
}

type CommunityListEntry struct {
	Type      CommunityListType
	Policy    policy.Type
	Community Community
	Regstr    string
}

func (typ CommunityListType) String() string {
	switch typ {
	case CommunityListStandard:
		return "standard"
	case CommunityListExpanded:
		return "expanded"
	default:
		return "unknown"
	}
}

func (list *CommunityList) String() string {
	str := ""
	for _, e := range list.Entry {
		str += fmt.Sprintf("ip community-list %s %s %s ",
			list.Name, list.Type.String(), e.Policy.String())
		switch e.Type {
		case CommunityListStandard:
			str += e.Community.String()
		case CommunityListExpanded:
			str += e.Regstr
		}
		str += "\n"
	}
	return str
}

func NewCommunityListMap() CommunityListMap {
	return CommunityListMap{}
}

func (clist CommunityListMap) CommunityListGet(name string) *CommunityList {
	if list, ok := clist[name]; ok {
		return list
	}
	list := &CommunityList{Name: name}
	clist[name] = list
	return list
}

func (lhs *CommunityListEntry) Equal(rhs *CommunityListEntry) bool {
	if lhs.Policy != rhs.Policy {
		return false
	}
	if lhs.Type != rhs.Type {
		return false
	}
	switch lhs.Type {
	case CommunityListStandard:
		if !lhs.Community.Equal(rhs.Community) {
			return false
		}
	case CommunityListExpanded:
		if lhs.Regstr != rhs.Regstr {
			return false
		}
	}
	return true
}

func NewCommunityListEntry(typ CommunityListType, policy policy.Type, str string) (*CommunityListEntry, error) {
	entry := &CommunityListEntry{
		Type:   typ,
		Policy: policy,
	}
	switch typ {
	case CommunityListStandard:
		com, err := CommunityParse(str)
		if err != nil {
			return nil, err
		}
		entry.Community = com
	case CommunityListExpanded:
		entry.Regstr = str
	}
	return entry, nil
}

func (clist CommunityListMap) CommunityListAdd(name string, typ CommunityListType, policy policy.Type, str string) error {
	list := clist.CommunityListGet(name)
	if list.Type != typ {
		return fmt.Errorf("community-list type mismatch")
	}
	entry, err := NewCommunityListEntry(typ, policy, str)
	if err != nil {
		return err
	}
	for _, e := range list.Entry {
		if e.Equal(entry) {
			return nil
		}
	}
	list.Entry = append(list.Entry, entry)
	return nil
}

func (clist CommunityListMap) CommunityListDelete(name string, typ CommunityListType, policy policy.Type, str string) error {
	list := clist[name]
	if list == nil {
		return fmt.Errorf("community-list name %s dose not exist", name)
	}
	if list.Type != typ {
		return fmt.Errorf("community-list type mismatch")
	}
	entry, err := NewCommunityListEntry(typ, policy, str)
	if err != nil {
		return err
	}
	elist := []*CommunityListEntry{}
	for _, e := range list.Entry {
		if !e.Equal(entry) {
			elist = append(elist, e)
		}
	}
	list.Entry = elist
	return nil
}
<commit_msg>fix compile error<commit_after>// Copyright 2017 zebra project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bgp

// ip community-list standard WORD {permit, deny} 100:1 100:2 no-export
// ip community-list expanded WORD {permit, deny} _100:1_

import (
	"fmt"

	"github.com/coreswitch/zebra/policy"
)

type CommunityListType int

const (
	CommunityListStandard CommunityListType = iota
	CommunityListExpanded
)

type CommunityListMap map[string]*CommunityList

type CommunityList struct {
	Type  CommunityListType
	Name  string
	Entry []*CommunityListEntry
}

type CommunityListEntry struct {
	Type      CommunityListType
	Policy    policy.Action
	Community Community
	Regstr    string
}

func (typ CommunityListType) String() string {
	switch typ {
	case CommunityListStandard:
		return "standard"
	case CommunityListExpanded:
		return "expanded"
	default:
		return "unknown"
	}
}

func (list *CommunityList) String() string {
	str := ""
	for _, e := range list.Entry {
		str += fmt.Sprintf("ip community-list %s %s %s ",
			list.Name, list.Type.String(), e.Policy.String())
		switch e.Type {
		case CommunityListStandard:
			str += e.Community.String()
		case CommunityListExpanded:
			str += e.Regstr
		}
		str += "\n"
	}
	return str
}

func NewCommunityListMap() CommunityListMap {
	return CommunityListMap{}
}

func (clist CommunityListMap) CommunityListGet(name string) *CommunityList {
	if list, ok := clist[name]; ok {
		return list
	}
	list := &CommunityList{Name: name}
	clist[name] = list
	return list
}

func (lhs *CommunityListEntry) Equal(rhs *CommunityListEntry) bool {
	if lhs.Policy != rhs.Policy {
		return false
	}
	if lhs.Type != rhs.Type {
		return false
	}
	switch lhs.Type {
	case CommunityListStandard:
		if !lhs.Community.Equal(rhs.Community) {
			return false
		}
	case CommunityListExpanded:
		if lhs.Regstr != rhs.Regstr {
			return false
		}
	}
	return true
}

func NewCommunityListEntry(typ CommunityListType, policy policy.Action, str string) (*CommunityListEntry, error) {
	entry := &CommunityListEntry{
		Type:   typ,
		Policy: policy,
	}
	switch typ {
	case CommunityListStandard:
		com, err := CommunityParse(str)
		if err != nil {
			return nil, err
		}
		entry.Community = com
	case CommunityListExpanded:
		entry.Regstr = str
	}
	return entry, nil
}

func (clist CommunityListMap) CommunityListAdd(name string, typ CommunityListType, policy policy.Action, str string) error {
	list := clist.CommunityListGet(name)
	if list.Type != typ {
		return fmt.Errorf("community-list type mismatch")
	}
	entry, err := NewCommunityListEntry(typ, policy, str)
	if err != nil {
		return err
	}
	for _, e := range list.Entry {
		if e.Equal(entry) {
			return nil
		}
	}
	list.Entry = append(list.Entry, entry)
	return nil
}

func (clist CommunityListMap) CommunityListDelete(name string, typ CommunityListType, policy policy.Action, str string) error {
	list := clist[name]
	if list == nil {
		return fmt.Errorf("community-list name %s dose not exist", name)
	}
	if list.Type != typ {
		return fmt.Errorf("community-list type mismatch")
	}
	entry, err := NewCommunityListEntry(typ, policy, str)
	if err != nil {
		return err
	}
	elist := []*CommunityListEntry{}
	for _, e := range list.Entry {
		if !e.Equal(entry) {
			elist = append(elist, e)
		}
	}
	list.Entry = elist
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mock

import (
	"net/http"
	"net/http/httptest"

	"io"

	"github.com/ligato/cn-infra/rpc/rest"
)

// HTTPMock is supposed to be used to mock real HTTP server but have the ability
// to test all other httpmux plugin related code
//
// Example:
//
//    httpmux.FromExistingServer(mock.SetHandler)
//	  mock.NewRequest("GET", "/v1/a", nil)
//
type HTTPMock struct {
	handler http.Handler
}

// SetHandler is called from httpmux plugin during startup (handler is usually the gorilla mux instance)
func (mock *HTTPMock) SetHandler(config rest.Config, handler http.Handler) (httpServer io.Closer, err error) {
	mock.handler = handler
	return &doNothingCloser{}, nil
}

// NewRequest propagates the request to the httpmux
func (mock *HTTPMock) NewRequest(method, url string, body io.Reader) (*http.Response, error) {
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		return nil, err
	}

	recorder := &httptest.ResponseRecorder{}
	mock.handler.ServeHTTP(recorder, req)
	return recorder.Result(), nil
}

type doNothingCloser struct{}

// Close does nothing
func (*doNothingCloser) Close() error {
	return nil
}
<commit_msg>ODPM-633 fix http mock<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mock

import (
	"net/http"
	"net/http/httptest"

	"io"

	"github.com/ligato/cn-infra/rpc/rest"
)

// HTTPMock is supposed to be used to mock real HTTP server but have the ability
// to test all other httpmux plugin related code
//
// Example:
//
//    httpmux.FromExistingServer(mock.SetHandler)
//	  mock.NewRequest("GET", "/v1/a", nil)
//
type HTTPMock struct {
	handler http.Handler
}

// SetHandler is called from httpmux plugin during startup (handler is usually the gorilla mux instance)
func (mock *HTTPMock) SetHandler(config rest.Config, handler http.Handler) (httpServer io.Closer, err error) {
	mock.handler = handler
	return &doNothingCloser{}, nil
}

// NewRequest propagates the request to the httpmux
func (mock *HTTPMock) NewRequest(method, url string, body io.Reader) (*http.Response, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	recorder := httptest.NewRecorder()
	mock.handler.ServeHTTP(recorder, req)
	return recorder.Result(), nil
}

type doNothingCloser struct{}

// Close does nothing
func (*doNothingCloser) Close() error {
	return nil
}
<|endoftext|>


<commit_before>package hooks

import (
	"sync"

	"github.com/gernest/ngorm/engine"
	"github.com/gernest/ngorm/model"
)

//Hook is a nterface that is executed at a particular point in time. This allows
//doing additional transformation of any portion of the engine.Engine, and it
//can be a way to overide default behaviour.
type Hook interface {
	Name() string
	Exec(h *Book, e *engine.Engine) error
}

//Hooks a safe struct that holds a map of hooks. Use this to provide a safe
//group of related hooks.
type Hooks struct {
	h  map[string]Hook
	mu sync.RWMutex
}

//Set saves the hook.
func (h *Hooks) Set(hk Hook) {
	h.mu.Lock()
	h.h[hk.Name()] = hk
	h.mu.Unlock()
}

//Get returns a saved hook.
func (h *Hooks) Get(name string) (Hook, bool) {
	h.mu.RLock()
	hk, ok := h.h[name]
	h.mu.RUnlock()
	return hk, ok
}

//NewHooks retruns an initalized Hooks instance.
func NewHooks() *Hooks {
	return &Hooks{h: make(map[string]Hook)}
}

type simpleHook struct {
	name string
	e    func(*Book, *engine.Engine) error
}

func (s *simpleHook) Name() string {
	return s.name
}

func (s *simpleHook) Exec(h *Book, e *engine.Engine) error {
	return s.e(h, e)
}

//HookFunc wraps the function f into a struct that statisfies Hook interface.
func HookFunc(name string, f func(*Book, *engine.Engine) error) Hook {
	return &simpleHook{name: name, e: f}
}

//Book a collection of hooks used by ngorm.
type Book struct {
	Create *Hooks
	Delete *Hooks
	Update *Hooks
	Save   *Hooks
	Query  *Hooks
}

func DefaultBook() *Book {
	b := &Book{
		Create: NewHooks(),
		Delete: NewHooks(),
		Update: NewHooks(),
		Save:   NewHooks(),
		Query:  NewHooks(),
	}
	b.Create.Set(HookFunc(model.Create, Create))
	b.Create.Set(HookFunc(model.HookAfterCreate, AfterCreate))
	b.Create.Set(HookFunc(model.HookBeforeCreate, BeforeCreate))

	b.Query.Set(HookFunc(model.Query, Query))
	b.Query.Set(HookFunc(model.HookAfterQuery, AfterQuery))
	return b
}
<commit_msg>[hooks] Fix misspell<commit_after>package hooks

import (
	"sync"

	"github.com/gernest/ngorm/engine"
	"github.com/gernest/ngorm/model"
)

//Hook is a nterface that is executed at a particular point in time. This allows
//doing additional transformation of any portion of the engine.Engine, and it
//can be a way to overide default behaviour.
type Hook interface {
	Name() string
	Exec(h *Book, e *engine.Engine) error
}

//Hooks a safe struct that holds a map of hooks. Use this to provide a safe
//group of related hooks.
type Hooks struct {
	h  map[string]Hook
	mu sync.RWMutex
}

//Set saves the hook.
func (h *Hooks) Set(hk Hook) {
	h.mu.Lock()
	h.h[hk.Name()] = hk
	h.mu.Unlock()
}

//Get returns a saved hook.
func (h *Hooks) Get(name string) (Hook, bool) {
	h.mu.RLock()
	hk, ok := h.h[name]
	h.mu.RUnlock()
	return hk, ok
}

//NewHooks retruns an initialized Hooks instance.
func NewHooks() *Hooks {
	return &Hooks{h: make(map[string]Hook)}
}

type simpleHook struct {
	name string
	e    func(*Book, *engine.Engine) error
}

func (s *simpleHook) Name() string {
	return s.name
}

func (s *simpleHook) Exec(h *Book, e *engine.Engine) error {
	return s.e(h, e)
}

//HookFunc wraps the function f into a struct that statisfies Hook interface.
func HookFunc(name string, f func(*Book, *engine.Engine) error) Hook {
	return &simpleHook{name: name, e: f}
}

//Book a collection of hooks used by ngorm.
type Book struct {
	Create *Hooks
	Delete *Hooks
	Update *Hooks
	Save   *Hooks
	Query  *Hooks
}

func DefaultBook() *Book {
	b := &Book{
		Create: NewHooks(),
		Delete: NewHooks(),
		Update: NewHooks(),
		Save:   NewHooks(),
		Query:  NewHooks(),
	}
	b.Create.Set(HookFunc(model.Create, Create))
	b.Create.Set(HookFunc(model.HookAfterCreate, AfterCreate))
	b.Create.Set(HookFunc(model.HookBeforeCreate, BeforeCreate))

	b.Query.Set(HookFunc(model.Query, Query))
	b.Query.Set(HookFunc(model.HookAfterQuery, AfterQuery))
	return b
}
<|endoftext|>


<commit_before>package jsoniter

import (
	"io"
	"bytes"
)

// Unmarshal adapts to json/encoding APIs
func Unmarshal(data []byte, v interface{}) error {
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	iter.ReadVal(v)
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return nil
	}
	if iter.Error == nil {
		iter.reportError("Unmarshal", "there are bytes left after unmarshal")
	}
	return iter.Error
}

func UnmarshalAny(data []byte) (Any, error) {
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	any := iter.ReadAny()
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return any, nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalAny", "there are bytes left after unmarshal")
	}
	return any, iter.Error
}

func lastNotSpacePos(data []byte) int {
	for i := len(data) - 1; i >= 0; i-- {
		if data[i] != ' ' && data[i] != '\t' && data[i] != '\r' && data[i] != '\n' {
			return i + 1
		}
	}
	return 0
}

func UnmarshalFromString(str string, v interface{}) error {
	data := []byte(str)
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	iter.ReadVal(v)
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalFromString", "there are bytes left after unmarshal")
	}
	return iter.Error
}

func UnmarshalAnyFromString(str string) (Any, error) {
	data := []byte(str)
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	any := iter.ReadAny()
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return any, nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalAnyFromString", "there are bytes left after unmarshal")
	}
	return nil, iter.Error
}

// Marshal is an adapter to json.Marshal
//
// Marshal returns the JSON encoding of v, adapts to json/encoding Marshal API
func Marshal(v interface{}) ([]byte, error) {
	buf := &bytes.Buffer{}
	stream := NewStream(buf, 512)
	stream.WriteVal(v)
	stream.Flush()
	if stream.Error != nil {
		return nil, stream.Error
	}
	return buf.Bytes(), nil
}

func MarshalToString(v interface{}) (string, error) {
	buf, err := Marshal(v)
	if err != nil {
		return "", err
	}
	return string(buf), nil
}

func NewDecoder(reader io.Reader) *AdaptedDecoder {
	iter := Parse(reader, 512)
	return &AdaptedDecoder{iter}
}

type AdaptedDecoder struct {
	iter *Iterator
}

func (adapter *AdaptedDecoder) Decode(obj interface{}) error {
	adapter.iter.ReadVal(obj)
	err := adapter.iter.Error
	if err == io.EOF {
		return nil
	}
	return adapter.iter.Error
}

func (adapter *AdaptedDecoder) More() bool {
	return adapter.iter.head != adapter.iter.tail
}

func (adapter *AdaptedDecoder) Buffered() io.Reader {
	remaining := adapter.iter.buf[adapter.iter.head:adapter.iter.tail]
	return bytes.NewReader(remaining)
}

func NewEncoder(writer io.Writer) *AdaptedEncoder {
	stream := NewStream(writer, 512)
	return &AdaptedEncoder{stream}
}

type AdaptedEncoder struct {
	stream *Stream
}

func (adapter *AdaptedEncoder) Encode(val interface{}) error {
	adapter.stream.WriteVal(val)
	adapter.stream.Flush()
	return adapter.stream.Error
}

func (adapter *AdaptedEncoder) SetIndent(prefix, indent string) {
	// not implemented yet
}<commit_msg>Marshal comment<commit_after>package jsoniter

import (
	"io"
	"bytes"
)

// Unmarshal adapts to json/encoding APIs
func Unmarshal(data []byte, v interface{}) error {
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	iter.ReadVal(v)
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return nil
	}
	if iter.Error == nil {
		iter.reportError("Unmarshal", "there are bytes left after unmarshal")
	}
	return iter.Error
}

func UnmarshalAny(data []byte) (Any, error) {
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	any := iter.ReadAny()
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return any, nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalAny", "there are bytes left after unmarshal")
	}
	return any, iter.Error
}

func lastNotSpacePos(data []byte) int {
	for i := len(data) - 1; i >= 0; i-- {
		if data[i] != ' ' && data[i] != '\t' && data[i] != '\r' && data[i] != '\n' {
			return i + 1
		}
	}
	return 0
}

func UnmarshalFromString(str string, v interface{}) error {
	data := []byte(str)
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	iter.ReadVal(v)
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalFromString", "there are bytes left after unmarshal")
	}
	return iter.Error
}

func UnmarshalAnyFromString(str string) (Any, error) {
	data := []byte(str)
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	any := iter.ReadAny()
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return any, nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalAnyFromString", "there are bytes left after unmarshal")
	}
	return nil, iter.Error
}

// Marshal is an adapter to json.Marshal
//
// Marshal returns the JSON encoding of v, adapts to json/encoding Marshal API
func Marshal(v interface{}) ([]byte, error) {
	buf := &bytes.Buffer{}
	stream := NewStream(buf, 512)
	stream.WriteVal(v)
	stream.Flush()
	if stream.Error != nil {
		return nil, stream.Error
	}
	return buf.Bytes(), nil
}

// notbeginwithfuncname MarshalToString does not show in doc
func MarshalToString(v interface{}) (string, error) {
	buf, err := Marshal(v)
	if err != nil {
		return "", err
	}
	return string(buf), nil
}

// NewDecoder show in doc
func NewDecoder(reader io.Reader) *AdaptedDecoder {
	iter := Parse(reader, 512)
	return &AdaptedDecoder{iter}
}

type AdaptedDecoder struct {
	iter *Iterator
}

func (adapter *AdaptedDecoder) Decode(obj interface{}) error {
	adapter.iter.ReadVal(obj)
	err := adapter.iter.Error
	if err == io.EOF {
		return nil
	}
	return adapter.iter.Error
}

func (adapter *AdaptedDecoder) More() bool {
	return adapter.iter.head != adapter.iter.tail
}

func (adapter *AdaptedDecoder) Buffered() io.Reader {
	remaining := adapter.iter.buf[adapter.iter.head:adapter.iter.tail]
	return bytes.NewReader(remaining)
}

func NewEncoder(writer io.Writer) *AdaptedEncoder {
	stream := NewStream(writer, 512)
	return &AdaptedEncoder{stream}
}

type AdaptedEncoder struct {
	stream *Stream
}

func (adapter *AdaptedEncoder) Encode(val interface{}) error {
	adapter.stream.WriteVal(val)
	adapter.stream.Flush()
	return adapter.stream.Error
}

func (adapter *AdaptedEncoder) SetIndent(prefix, indent string) {
	// not implemented yet
}<|endoftext|>