<commit_before>package reflect

import (
	lua "github.com/anaminus/gopher-lua"
	"github.com/anaminus/rbxmk"
	"github.com/anaminus/rbxmk/dump"
	"github.com/anaminus/rbxmk/dump/dt"
	"github.com/anaminus/rbxmk/rtypes"
	"github.com/robloxapi/rbxdump"
	"github.com/robloxapi/rbxdump/diff"
	"github.com/robloxapi/types"
)

func init() { register(DescAction) }
func DescAction() rbxmk.Reflector {
	return rbxmk.Reflector{
		Name:     "DescAction",
		PushTo:   rbxmk.PushPtrTypeTo("DescAction"),
		PullFrom: rbxmk.PullTypeFrom("DescAction"),
		Metatable: rbxmk.Metatable{
			"__tostring": func(s rbxmk.State) int {
				v := s.Pull(1, "DescAction").(*rtypes.DescAction)
				s.L.Push(lua.LString(v.String()))
				return 1
			},
		},
		Properties: rbxmk.Properties{
			"Type": {
				Get: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					enum := s.MustEnum("DescActionType")
					value := int(action.Action.Type)
					item := enum.Value(value)
					if item == nil {
						s.RaiseError("invalid value %d for %s", value, enum.Name())
					}
					return s.Push(item)
				},
				Set: func(s rbxmk.State, v types.Value) {
					action := v.(*rtypes.DescAction)
					enum := s.MustEnum("DescActionType")
					value := s.L.Get(3)
					item := enum.Pull(value)
					if item == nil {
						s.RaiseError("invalid value %s for %s", value, enum.Name())
						return
					}
					action.Action.Type = diff.Type(item.Value())
				},
				Dump: func() dump.Property {
					return dump.Property{
						ValueType:   dt.Prim("Enum.DescActionType"),
						Summary:     "Types/DescAction:Properties/Type/Summary",
						Description: "Types/DescAction:Properties/Type/Description",
					}
				},
			},
			"Element": {
				Get: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					enum := s.MustEnum("DescActionElement")
					value := int(action.Action.Element)
					item := enum.Value(value)
					if item == nil {
						s.RaiseError("invalid value %d for %s", value, enum.Name())
					}
					return s.Push(item)
				},
				Set: func(s rbxmk.State, v types.Value) {
					action := v.(*rtypes.DescAction)
					enum := s.MustEnum("DescActionElement")
					value := s.L.Get(3)
					item := enum.Pull(value)
					if item == nil {
						s.RaiseError("invalid value %s for %s", value, enum.Name())
						return
					}
					action.Action.Element = diff.Element(item.Value())
				},
				Dump: func() dump.Property {
					return dump.Property{
						ValueType:   dt.Prim("Enum.DescActionElement"),
						Summary:     "Types/DescAction:Properties/Element/Summary",
						Description: "Types/DescAction:Properties/Element/Description",
					}
				},
			},
			"Primary": {
				Get: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					return s.Push(types.String(action.Action.Primary))
				},
				Set: func(s rbxmk.State, v types.Value) {
					action := v.(*rtypes.DescAction)
					action.Action.Primary = string(s.Pull(3, "string").(types.String))
				},
				Dump: func() dump.Property {
					return dump.Property{
						ValueType:   dt.Prim("string"),
						Summary:     "Types/DescAction:Properties/Primary/Summary",
						Description: "Types/DescAction:Properties/Primary/Description",
					}
				},
			},
			"Secondary": {
				Get: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					return s.Push(types.String(action.Action.Secondary))
				},
				Set: func(s rbxmk.State, v types.Value) {
					action := v.(*rtypes.DescAction)
					action.Action.Secondary = string(s.Pull(3, "string").(types.String))
				},
				Dump: func() dump.Property {
					return dump.Property{
						ValueType:   dt.Prim("string"),
						Summary:     "Types/DescAction:Properties/Secondary/Summary",
						Description: "Types/DescAction:Properties/Secondary/Description",
					}
				},
			},
		},
		Methods: rbxmk.Methods{
			"Field": {
				Func: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					name := string(s.Pull(3, "string").(types.String))
					fvalue, ok := action.Fields[name]
					if !ok {
						return s.Push(rtypes.Nil)
					}
					value := pushDescActionField(fvalue)
					if value == nil {
						return s.Push(rtypes.Nil)
					}
					return s.Push(value)
				},
				Dump: func() dump.Function {
					return dump.Function{
						Parameters: dump.Parameters{
							{Name: "name", Type: dt.Prim("string")},
						},
						Returns: dump.Parameters{
							{Type: dt.Prim("any")},
						},
						Summary:     "Types/DescAction:Methods/Field/Summary",
						Description: "Types/DescAction:Methods/Field/Description",
					}
				},
			},
			"Fields": {
				Func: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					dict := make(rtypes.Dictionary, len(action.Fields))
					for k, v := range action.Fields {
						value := pushDescActionField(v)
						if value == nil {
							//TODO: Emit a warning or something.
							continue
						}
						dict[k] = value
					}
					return s.Push(dict)
				},
				Dump: func() dump.Function {
					return dump.Function{
						Returns: dump.Parameters{
							{Type: dt.Dictionary{V: dt.Prim("any")}},
						},
						Summary:     "Types/DescAction:Methods/Fields/Summary",
						Description: "Types/DescAction:Methods/Fields/Description",
					}
				},
			},
			"SetField": {
				Func: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					name := string(s.Pull(3, "string").(types.String))
					value := pullDescActionField(name, s.Pull(4, "Variant"))
					if value == nil {
						return s.RaiseError("unexpected type %s", v.Type())
					}
					action.Fields[name] = value
					return 0
				},
				Dump: func() dump.Function {
					return dump.Function{
						Parameters: dump.Parameters{
							{Name: "name", Type: dt.Prim("string")},
							{Name: "value", Type: dt.Prim("any")},
						},
						Summary:     "Types/DescAction:Methods/SetField/Summary",
						Description: "Types/DescAction:Methods/SetField/Description",
					}
				},
			},
			"SetFields": {
				Func: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					values := s.Pull(3, "Dictionary").(rtypes.Dictionary)
					fields := make(rbxdump.Fields, len(values))
					for k, v := range values {
						value := pullDescActionField(k, v)
						if value == nil {
							return s.RaiseError("field %q: unexpected type %s", k, v.Type())
						}
						fields[k] = value
					}
					action.Fields = fields
					return 0
				},
				Dump: func() dump.Function {
					return dump.Function{
						Parameters: dump.Parameters{
							{Name: "fields", Type: dt.Dictionary{V: dt.Prim("any")}},
						},
						Summary:     "Types/DescAction:Methods/SetFields/Summary",
						Description: "Types/DescAction:Methods/SetFields/Description",
					}
				},
			},
		},
		Constructors: rbxmk.Constructors{
			"new": rbxmk.Constructor{
				Func: func(s rbxmk.State) int {
					var v rtypes.DescAction

					typeEnum := s.MustEnum("DescActionType")
					actionType := typeEnum.Pull(s.L.Get(1))
					if actionType == nil {
						return s.ArgError(1, "invalid value %s for %s", actionType, typeEnum.Name())
					}
					v.Action.Type = diff.Type(actionType.Value())

					elementEnum := s.MustEnum("DescActionElement")
					actionElement := elementEnum.Pull(s.L.Get(2))
					if actionElement == nil {
						return s.ArgError(2, "invalid value %s for %s", actionElement, elementEnum.Name())
					}
					v.Action.Element = diff.Element(actionElement.Value())

					return s.Push(&v)
				},
				Dump: func() dump.MultiFunction {
					return dump.MultiFunction{
						dump.Function{
							Parameters: dump.Parameters{
								{Name: "type", Type: dt.Prim("Enum.DescActionType")},
								{Name: "element", Type: dt.Prim("Enum.DescActionElement")},
							},
							Returns: dump.Parameters{
								{Type: dt.Prim("DescAction")},
							},
							Summary:     "Types/DescAction:Constructors/new/Summary",
							Description: "Types/DescAction:Constructors/new/Description",
						},
					}
				},
			},
		},
		Dump: func() dump.TypeDef {
			return dump.TypeDef{
				Summary:     "Types/DescAction:Summary",
				Description: "Types/DescAction:Description",
			}
		},
		Enums: []*rtypes.Enum{
			rtypes.NewEnum("DescActionType",
				rtypes.NewItem{Name: diff.Remove.String(), Value: int(diff.Remove)},
				rtypes.NewItem{Name: diff.Change.String(), Value: int(diff.Change)},
				rtypes.NewItem{Name: diff.Add.String(), Value: int(diff.Add)},
			),
			rtypes.NewEnum("DescActionElement",
				rtypes.NewItem{Name: diff.Class.String(), Value: int(diff.Class)},
				rtypes.NewItem{Name: diff.Property.String(), Value: int(diff.Property)},
				rtypes.NewItem{Name: diff.Function.String(), Value: int(diff.Function)},
				rtypes.NewItem{Name: diff.Event.String(), Value: int(diff.Event)},
				rtypes.NewItem{Name: diff.Callback.String(), Value: int(diff.Callback)},
				rtypes.NewItem{Name: diff.Enum.String(), Value: int(diff.Enum)},
				rtypes.NewItem{Name: diff.EnumItem.String(), Value: int(diff.EnumItem)},
			),
		},
		Types: []func() rbxmk.Reflector{
			Array,
			Bool,
			Dictionary,
			Int,
			ParameterDesc,
			String,
			TypeDesc,
		},
	}
}

func pushDescActionField(v interface{}) types.Value {
	switch v := v.(type) {
	case bool:
		return types.Bool(v)
	case int:
		return types.Int(v)
	case string:
		return types.String(v)
	case rbxdump.Tags:
		a := make(rtypes.Array, len(v))
		for i, v := range v {
			a[i] = types.String(v)
		}
		return a
	case rbxdump.Type:
		return rtypes.TypeDesc{Embedded: v}
	case []rbxdump.Parameter:
		a := make(rtypes.Array, len(v))
		for i, v := range v {
			a[i] = rtypes.ParameterDesc{Parameter: v}
		}
		return a
	default:
		return nil
	}
}

func pullDescActionField(k string, v types.Value) interface{} {
	switch v := v.(type) {
	case types.Bool:
		return bool(v)
	case types.Int:
		return int(v)
	case types.String:
		return string(v)
	case rtypes.TypeDesc:
		return v.Embedded
	case rtypes.Array:
		switch k {
		case "Parameters":
			a := make([]rbxdump.Parameter, len(v))
			for i, v := range v {
				v, ok := v.(rtypes.ParameterDesc)
				if !ok {
					return nil
				}
				a[i] = v.Parameter
			}
			return a
		case "Tags":
			a := make([]string, len(v))
			for i, v := range v {
				v, ok := v.(types.Stringlike)
				if !ok {
					return nil
				}
				a[i] = v.Stringlike()
			}
			return a
		}
		return nil
	default:
		return nil
	}
}
<commit_msg>Fix DescAction field not pushing tags.<commit_after>package reflect

import (
	lua "github.com/anaminus/gopher-lua"
	"github.com/anaminus/rbxmk"
	"github.com/anaminus/rbxmk/dump"
	"github.com/anaminus/rbxmk/dump/dt"
	"github.com/anaminus/rbxmk/rtypes"
	"github.com/robloxapi/rbxdump"
	"github.com/robloxapi/rbxdump/diff"
	"github.com/robloxapi/types"
)

func init() { register(DescAction) }
func DescAction() rbxmk.Reflector {
	return rbxmk.Reflector{
		Name:     "DescAction",
		PushTo:   rbxmk.PushPtrTypeTo("DescAction"),
		PullFrom: rbxmk.PullTypeFrom("DescAction"),
		Metatable: rbxmk.Metatable{
			"__tostring": func(s rbxmk.State) int {
				v := s.Pull(1, "DescAction").(*rtypes.DescAction)
				s.L.Push(lua.LString(v.String()))
				return 1
			},
		},
		Properties: rbxmk.Properties{
			"Type": {
				Get: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					enum := s.MustEnum("DescActionType")
					value := int(action.Action.Type)
					item := enum.Value(value)
					if item == nil {
						s.RaiseError("invalid value %d for %s", value, enum.Name())
					}
					return s.Push(item)
				},
				Set: func(s rbxmk.State, v types.Value) {
					action := v.(*rtypes.DescAction)
					enum := s.MustEnum("DescActionType")
					value := s.L.Get(3)
					item := enum.Pull(value)
					if item == nil {
						s.RaiseError("invalid value %s for %s", value, enum.Name())
						return
					}
					action.Action.Type = diff.Type(item.Value())
				},
				Dump: func() dump.Property {
					return dump.Property{
						ValueType:   dt.Prim("Enum.DescActionType"),
						Summary:     "Types/DescAction:Properties/Type/Summary",
						Description: "Types/DescAction:Properties/Type/Description",
					}
				},
			},
			"Element": {
				Get: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					enum := s.MustEnum("DescActionElement")
					value := int(action.Action.Element)
					item := enum.Value(value)
					if item == nil {
						s.RaiseError("invalid value %d for %s", value, enum.Name())
					}
					return s.Push(item)
				},
				Set: func(s rbxmk.State, v types.Value) {
					action := v.(*rtypes.DescAction)
					enum := s.MustEnum("DescActionElement")
					value := s.L.Get(3)
					item := enum.Pull(value)
					if item == nil {
						s.RaiseError("invalid value %s for %s", value, enum.Name())
						return
					}
					action.Action.Element = diff.Element(item.Value())
				},
				Dump: func() dump.Property {
					return dump.Property{
						ValueType:   dt.Prim("Enum.DescActionElement"),
						Summary:     "Types/DescAction:Properties/Element/Summary",
						Description: "Types/DescAction:Properties/Element/Description",
					}
				},
			},
			"Primary": {
				Get: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					return s.Push(types.String(action.Action.Primary))
				},
				Set: func(s rbxmk.State, v types.Value) {
					action := v.(*rtypes.DescAction)
					action.Action.Primary = string(s.Pull(3, "string").(types.String))
				},
				Dump: func() dump.Property {
					return dump.Property{
						ValueType:   dt.Prim("string"),
						Summary:     "Types/DescAction:Properties/Primary/Summary",
						Description: "Types/DescAction:Properties/Primary/Description",
					}
				},
			},
			"Secondary": {
				Get: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					return s.Push(types.String(action.Action.Secondary))
				},
				Set: func(s rbxmk.State, v types.Value) {
					action := v.(*rtypes.DescAction)
					action.Action.Secondary = string(s.Pull(3, "string").(types.String))
				},
				Dump: func() dump.Property {
					return dump.Property{
						ValueType:   dt.Prim("string"),
						Summary:     "Types/DescAction:Properties/Secondary/Summary",
						Description: "Types/DescAction:Properties/Secondary/Description",
					}
				},
			},
		},
		Methods: rbxmk.Methods{
			"Field": {
				Func: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					name := string(s.Pull(3, "string").(types.String))
					fvalue, ok := action.Fields[name]
					if !ok {
						return s.Push(rtypes.Nil)
					}
					value := pushDescActionField(fvalue)
					if value == nil {
						return s.Push(rtypes.Nil)
					}
					return s.Push(value)
				},
				Dump: func() dump.Function {
					return dump.Function{
						Parameters: dump.Parameters{
							{Name: "name", Type: dt.Prim("string")},
						},
						Returns: dump.Parameters{
							{Type: dt.Prim("any")},
						},
						Summary:     "Types/DescAction:Methods/Field/Summary",
						Description: "Types/DescAction:Methods/Field/Description",
					}
				},
			},
			"Fields": {
				Func: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					dict := make(rtypes.Dictionary, len(action.Fields))
					for k, v := range action.Fields {
						value := pushDescActionField(v)
						if value == nil {
							//TODO: Emit a warning or something.
							continue
						}
						dict[k] = value
					}
					return s.Push(dict)
				},
				Dump: func() dump.Function {
					return dump.Function{
						Returns: dump.Parameters{
							{Type: dt.Dictionary{V: dt.Prim("any")}},
						},
						Summary:     "Types/DescAction:Methods/Fields/Summary",
						Description: "Types/DescAction:Methods/Fields/Description",
					}
				},
			},
			"SetField": {
				Func: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					name := string(s.Pull(3, "string").(types.String))
					value := pullDescActionField(name, s.Pull(4, "Variant"))
					if value == nil {
						return s.RaiseError("unexpected type %s", v.Type())
					}
					action.Fields[name] = value
					return 0
				},
				Dump: func() dump.Function {
					return dump.Function{
						Parameters: dump.Parameters{
							{Name: "name", Type: dt.Prim("string")},
							{Name: "value", Type: dt.Prim("any")},
						},
						Summary:     "Types/DescAction:Methods/SetField/Summary",
						Description: "Types/DescAction:Methods/SetField/Description",
					}
				},
			},
			"SetFields": {
				Func: func(s rbxmk.State, v types.Value) int {
					action := v.(*rtypes.DescAction)
					values := s.Pull(3, "Dictionary").(rtypes.Dictionary)
					fields := make(rbxdump.Fields, len(values))
					for k, v := range values {
						value := pullDescActionField(k, v)
						if value == nil {
							return s.RaiseError("field %q: unexpected type %s", k, v.Type())
						}
						fields[k] = value
					}
					action.Fields = fields
					return 0
				},
				Dump: func() dump.Function {
					return dump.Function{
						Parameters: dump.Parameters{
							{Name: "fields", Type: dt.Dictionary{V: dt.Prim("any")}},
						},
						Summary:     "Types/DescAction:Methods/SetFields/Summary",
						Description: "Types/DescAction:Methods/SetFields/Description",
					}
				},
			},
		},
		Constructors: rbxmk.Constructors{
			"new": rbxmk.Constructor{
				Func: func(s rbxmk.State) int {
					var v rtypes.DescAction

					typeEnum := s.MustEnum("DescActionType")
					actionType := typeEnum.Pull(s.L.Get(1))
					if actionType == nil {
						return s.ArgError(1, "invalid value %s for %s", actionType, typeEnum.Name())
					}
					v.Action.Type = diff.Type(actionType.Value())

					elementEnum := s.MustEnum("DescActionElement")
					actionElement := elementEnum.Pull(s.L.Get(2))
					if actionElement == nil {
						return s.ArgError(2, "invalid value %s for %s", actionElement, elementEnum.Name())
					}
					v.Action.Element = diff.Element(actionElement.Value())

					return s.Push(&v)
				},
				Dump: func() dump.MultiFunction {
					return dump.MultiFunction{
						dump.Function{
							Parameters: dump.Parameters{
								{Name: "type", Type: dt.Prim("Enum.DescActionType")},
								{Name: "element", Type: dt.Prim("Enum.DescActionElement")},
							},
							Returns: dump.Parameters{
								{Type: dt.Prim("DescAction")},
							},
							Summary:     "Types/DescAction:Constructors/new/Summary",
							Description: "Types/DescAction:Constructors/new/Description",
						},
					}
				},
			},
		},
		Dump: func() dump.TypeDef {
			return dump.TypeDef{
				Summary:     "Types/DescAction:Summary",
				Description: "Types/DescAction:Description",
			}
		},
		Enums: []*rtypes.Enum{
			rtypes.NewEnum("DescActionType",
				rtypes.NewItem{Name: diff.Remove.String(), Value: int(diff.Remove)},
				rtypes.NewItem{Name: diff.Change.String(), Value: int(diff.Change)},
				rtypes.NewItem{Name: diff.Add.String(), Value: int(diff.Add)},
			),
			rtypes.NewEnum("DescActionElement",
				rtypes.NewItem{Name: diff.Class.String(), Value: int(diff.Class)},
				rtypes.NewItem{Name: diff.Property.String(), Value: int(diff.Property)},
				rtypes.NewItem{Name: diff.Function.String(), Value: int(diff.Function)},
				rtypes.NewItem{Name: diff.Event.String(), Value: int(diff.Event)},
				rtypes.NewItem{Name: diff.Callback.String(), Value: int(diff.Callback)},
				rtypes.NewItem{Name: diff.Enum.String(), Value: int(diff.Enum)},
				rtypes.NewItem{Name: diff.EnumItem.String(), Value: int(diff.EnumItem)},
			),
		},
		Types: []func() rbxmk.Reflector{
			Array,
			Bool,
			Dictionary,
			Int,
			ParameterDesc,
			String,
			TypeDesc,
		},
	}
}

func pushDescActionField(v interface{}) types.Value {
	switch v := v.(type) {
	case bool:
		return types.Bool(v)
	case int:
		return types.Int(v)
	case string:
		return types.String(v)
	case rbxdump.Tags:
		a := make(rtypes.Array, len(v))
		for i, v := range v {
			a[i] = types.String(v)
		}
		return a
	case rbxdump.Type:
		return rtypes.TypeDesc{Embedded: v}
	case []rbxdump.Parameter:
		a := make(rtypes.Array, len(v))
		for i, v := range v {
			a[i] = rtypes.ParameterDesc{Parameter: v}
		}
		return a
	case []string:
		a := make(rtypes.Array, len(v))
		for i, v := range v {
			a[i] = types.String(v)
		}
		return a
	default:
		return nil
	}
}

func pullDescActionField(k string, v types.Value) interface{} {
	switch v := v.(type) {
	case types.Bool:
		return bool(v)
	case types.Int:
		return int(v)
	case types.String:
		return string(v)
	case rtypes.TypeDesc:
		return v.Embedded
	case rtypes.Array:
		switch k {
		case "Parameters":
			a := make([]rbxdump.Parameter, len(v))
			for i, v := range v {
				v, ok := v.(rtypes.ParameterDesc)
				if !ok {
					return nil
				}
				a[i] = v.Parameter
			}
			return a
		case "Tags":
			a := make([]string, len(v))
			for i, v := range v {
				v, ok := v.(types.Stringlike)
				if !ok {
					return nil
				}
				a[i] = v.Stringlike()
			}
			return a
		}
		return nil
	default:
		return nil
	}
}
<|endoftext|>


<commit_before><commit_msg>Added TestWriteInt16Slice<commit_after><|endoftext|>


<commit_before>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/format"
	"io"
	"os"
	"reflect"
	"regexp"
	"strings"
	"text/template"
)

const templateFunctionHelp = `
ciao-cli adds some new functions to Go's template language

- tojson outputs the specified object in json format, e.g., {{tojson .}}
- filter operates on an slice or array of structures.  It allows the caller
  to filter the input array based on the value of a single field.
  The function returns a slice containing only the objects that satisfy the
  filter, e.g.

  ciao-cli image list -f '{{$x := filter . "Protected" "true"}}{{len $x}}'

  outputs the number of protected images maintained by the image service.
- filterContains operates along the same lines as filter, but returns
  substring matches

  ciao-cli workload list -f '{{$x := filterContains . "Name" "Cloud"}}{{range $x}}{{.ID}}{{end}}'

  outputs the IDs of the workloads which have Cloud in their name.
- filterHasPrefix along the same lines as filter, but returns prefix matches
- filterHasSuffix along the same lines as filter, but returns suffix matches
- filterFolded along the same lines as filter, but  returns matches based
  on equality under Unicode case-folding
- filterRegexp along the same lines as filter, but  returns matches based
  on regular expression matching

  ciao-cli workload list -f '{{$x := filterRegexp . "Name" "^Docker[ a-zA-z]*latest$"}}{{range $x}}{{println .ID .Name}}{{end}}'

  outputs the IDs of the workloads which have Docker prefix and latest suffix
  in their name.
- select operates on a slice of structs.  It outputs the value of a specified
  field for each struct on a new line , e.g.,

  {{select . "Name"}}
`

var funcMap = template.FuncMap{
	"filter":          filterByField,
	"filterContains":  filterByContains,
	"filterHasPrefix": filterByHasPrefix,
	"filterHasSuffix": filterByHasSuffix,
	"filterFolded":    filterByFolded,
	"filterRegexp":    filterByRegexp,
	"tojson":          toJSON,
	"select":          selectField,
}

func findField(fieldPath []string, v reflect.Value) reflect.Value {
	f := v
	for _, seg := range fieldPath {
		f = f.FieldByName(seg)
		if f.Kind() == reflect.Ptr {
			f = reflect.Indirect(f)
		}
	}
	return f
}

func filterField(obj interface{}, field, val string, cmp func(string, string) bool) (retval interface{}) {
	defer func() {
		err := recover()
		if err != nil {
			fatalf("Invalid use of filter: %v", err)
		}
	}()

	list := reflect.ValueOf(obj)
	if list.Kind() == reflect.Ptr {
		list = reflect.Indirect(list)
	}
	filtered := reflect.MakeSlice(list.Type(), 0, list.Len())

	fieldPath := strings.Split(field, ".")

	for i := 0; i < list.Len(); i++ {
		v := list.Index(i)
		if v.Kind() == reflect.Ptr {
			v = reflect.Indirect(v)
		}

		f := findField(fieldPath, v)

		strVal := fmt.Sprintf("%v", f.Interface())
		if cmp(strVal, val) {
			filtered = reflect.Append(filtered, list.Index(i))
		}
	}

	retval = filtered.Interface()
	return

}

func filterByField(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, func(a, b string) bool {
		return a == b
	})
}

func filterByContains(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, strings.Contains)
}

func filterByFolded(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, strings.EqualFold)
}

func filterByHasPrefix(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, strings.HasPrefix)
}

func filterByHasSuffix(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, strings.HasSuffix)
}

func filterByRegexp(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, func(a, b string) bool {
		matched, err := regexp.MatchString(b, a)
		if err != nil {
			fatalf("Invalid regexp: %v", err)
		}
		return matched
	})
}

func selectField(obj interface{}, field string) string {
	defer func() {
		err := recover()
		if err != nil {
			fatalf("Invalid use of select: %v", err)
		}
	}()

	var b bytes.Buffer
	list := reflect.ValueOf(obj)
	if list.Kind() == reflect.Ptr {
		list = reflect.Indirect(list)
	}

	fieldPath := strings.Split(field, ".")

	for i := 0; i < list.Len(); i++ {
		v := list.Index(i)
		if v.Kind() == reflect.Ptr {
			v = reflect.Indirect(v)
		}

		f := findField(fieldPath, v)

		fmt.Fprintf(&b, "%v\n", f.Interface())
	}

	return string(b.Bytes())
}

func toJSON(obj interface{}) string {
	b, err := json.MarshalIndent(obj, "", "\t")
	if err != nil {
		return ""
	}
	return string(b)
}

func outputToTemplate(name, tmplSrc string, obj interface{}) error {
	t, err := template.New(name).Funcs(funcMap).Parse(tmplSrc)
	if err != nil {
		fatalf(err.Error())
	}
	if err = t.Execute(os.Stdout, obj); err != nil {
		fatalf(err.Error())
	}
	return nil
}

func createTemplate(name, tmplSrc string) *template.Template {
	var t *template.Template
	if tmplSrc == "" {
		return nil
	}

	t, err := template.New(name).Funcs(funcMap).Parse(tmplSrc)
	if err != nil {
		fatalf(err.Error())
	}

	return t
}

func exportedFields(typ reflect.Type) bool {
	var i int
	for i = 0; i < typ.NumField(); i++ {
		if typ.Field(i).PkgPath == "" {
			break
		}
	}

	return i < typ.NumField()
}

func ignoreKind(kind reflect.Kind) bool {
	return (kind == reflect.Chan) || (kind == reflect.Invalid)
}

func generateStruct(buf io.Writer, dummy reflect.Type) {
	for i := 0; i < dummy.NumField(); i++ {
		field := dummy.Field(i)
		if field.PkgPath != "" || ignoreKind(field.Type.Kind()) {
			continue
		}
		fmt.Fprintf(buf, "%s ", field.Name)
		tag := ""
		if field.Tag != "" {
			tag = fmt.Sprintf("`%s`", field.Tag)
		}
		generateUsage(buf, field.Type, tag)
	}
}

func generateUsage(buf io.Writer, dummy reflect.Type, tag string) {
	kind := dummy.Kind()
	if ignoreKind(kind) {
		return
	}

	switch kind {
	case reflect.Struct:
		if exportedFields(dummy) {
			fmt.Fprintf(buf, "struct {\n")
			generateStruct(buf, dummy)
			fmt.Fprintf(buf, "}%s\n", tag)
		} else if dummy.Name() != "" {
			fmt.Fprintf(buf, "%s%s\n", dummy.String(), tag)
		} else {
			fmt.Fprintf(buf, "struct {\n}%s\n", tag)
		}
	case reflect.Slice:
		fmt.Fprintf(buf, "[]")
		generateUsage(buf, dummy.Elem(), tag)
	case reflect.Array:
		fmt.Fprintf(buf, "[%d]", dummy.Len())
		generateUsage(buf, dummy.Elem(), tag)
	case reflect.Map:
		fmt.Fprintf(buf, "map[%s]", dummy.Key().String())
		generateUsage(buf, dummy.Elem(), tag)
	default:
		fmt.Fprintf(buf, "%s%s\n", dummy.String(), tag)
	}
}

func formatType(buf *bytes.Buffer, unformattedType []byte) {
	const typePrefix = "type x "
	source := bytes.NewBufferString(typePrefix)
	_, _ = source.Write(unformattedType)
	formattedType, err := format.Source(source.Bytes())
	if err != nil {
		panic(fmt.Errorf("generateIndentedUsage created invalid Go code: %v", err))
	}
	_, _ = buf.Write(formattedType[len(typePrefix):])
}

func generateIndentedUsage(buf *bytes.Buffer, dummy interface{}) {
	var source bytes.Buffer
	generateUsage(&source, reflect.TypeOf(dummy), "")
	formatType(buf, source.Bytes())
}

func generateUsageUndecorated(dummy interface{}) string {
	var buf bytes.Buffer
	generateIndentedUsage(&buf, dummy)
	return buf.String()
}

func generateUsageDecorated(flag string, dummy interface{}) string {
	var buf bytes.Buffer

	fmt.Fprintf(&buf,
		"The template passed to the -%s option operates on a\n\n",
		flag)

	generateIndentedUsage(&buf, dummy)
	fmt.Fprintf(&buf, templateFunctionHelp)
	return buf.String()
}
<commit_msg>ciao-cli: Rename parameters and simplify function in template.go<commit_after>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/format"
	"io"
	"os"
	"reflect"
	"regexp"
	"strings"
	"text/template"
)

const templateFunctionHelp = `
ciao-cli adds some new functions to Go's template language

- tojson outputs the specified object in json format, e.g., {{tojson .}}
- filter operates on an slice or array of structures.  It allows the caller
  to filter the input array based on the value of a single field.
  The function returns a slice containing only the objects that satisfy the
  filter, e.g.

  ciao-cli image list -f '{{$x := filter . "Protected" "true"}}{{len $x}}'

  outputs the number of protected images maintained by the image service.
- filterContains operates along the same lines as filter, but returns
  substring matches

  ciao-cli workload list -f '{{$x := filterContains . "Name" "Cloud"}}{{range $x}}{{.ID}}{{end}}'

  outputs the IDs of the workloads which have Cloud in their name.
- filterHasPrefix along the same lines as filter, but returns prefix matches
- filterHasSuffix along the same lines as filter, but returns suffix matches
- filterFolded along the same lines as filter, but  returns matches based
  on equality under Unicode case-folding
- filterRegexp along the same lines as filter, but  returns matches based
  on regular expression matching

  ciao-cli workload list -f '{{$x := filterRegexp . "Name" "^Docker[ a-zA-z]*latest$"}}{{range $x}}{{println .ID .Name}}{{end}}'

  outputs the IDs of the workloads which have Docker prefix and latest suffix
  in their name.
- select operates on a slice of structs.  It outputs the value of a specified
  field for each struct on a new line , e.g.,

  {{select . "Name"}}
`

var funcMap = template.FuncMap{
	"filter":          filterByField,
	"filterContains":  filterByContains,
	"filterHasPrefix": filterByHasPrefix,
	"filterHasSuffix": filterByHasSuffix,
	"filterFolded":    filterByFolded,
	"filterRegexp":    filterByRegexp,
	"tojson":          toJSON,
	"select":          selectField,
}

func findField(fieldPath []string, v reflect.Value) reflect.Value {
	f := v
	for _, seg := range fieldPath {
		f = f.FieldByName(seg)
		if f.Kind() == reflect.Ptr {
			f = reflect.Indirect(f)
		}
	}
	return f
}

func filterField(obj interface{}, field, val string, cmp func(string, string) bool) (retval interface{}) {
	defer func() {
		err := recover()
		if err != nil {
			fatalf("Invalid use of filter: %v", err)
		}
	}()

	list := reflect.ValueOf(obj)
	if list.Kind() == reflect.Ptr {
		list = reflect.Indirect(list)
	}
	filtered := reflect.MakeSlice(list.Type(), 0, list.Len())

	fieldPath := strings.Split(field, ".")

	for i := 0; i < list.Len(); i++ {
		v := list.Index(i)
		if v.Kind() == reflect.Ptr {
			v = reflect.Indirect(v)
		}

		f := findField(fieldPath, v)

		strVal := fmt.Sprintf("%v", f.Interface())
		if cmp(strVal, val) {
			filtered = reflect.Append(filtered, list.Index(i))
		}
	}

	retval = filtered.Interface()
	return

}

func filterByField(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, func(a, b string) bool {
		return a == b
	})
}

func filterByContains(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, strings.Contains)
}

func filterByFolded(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, strings.EqualFold)
}

func filterByHasPrefix(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, strings.HasPrefix)
}

func filterByHasSuffix(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, strings.HasSuffix)
}

func filterByRegexp(obj interface{}, field, val string) (retval interface{}) {
	return filterField(obj, field, val, func(a, b string) bool {
		matched, err := regexp.MatchString(b, a)
		if err != nil {
			fatalf("Invalid regexp: %v", err)
		}
		return matched
	})
}

func selectField(obj interface{}, field string) string {
	defer func() {
		err := recover()
		if err != nil {
			fatalf("Invalid use of select: %v", err)
		}
	}()

	var b bytes.Buffer
	list := reflect.ValueOf(obj)
	if list.Kind() == reflect.Ptr {
		list = reflect.Indirect(list)
	}

	fieldPath := strings.Split(field, ".")

	for i := 0; i < list.Len(); i++ {
		v := list.Index(i)
		if v.Kind() == reflect.Ptr {
			v = reflect.Indirect(v)
		}

		f := findField(fieldPath, v)

		fmt.Fprintf(&b, "%v\n", f.Interface())
	}

	return string(b.Bytes())
}

func toJSON(obj interface{}) string {
	b, err := json.MarshalIndent(obj, "", "\t")
	if err != nil {
		return ""
	}
	return string(b)
}

func outputToTemplate(name, tmplSrc string, obj interface{}) error {
	t, err := template.New(name).Funcs(funcMap).Parse(tmplSrc)
	if err != nil {
		fatalf(err.Error())
	}
	if err = t.Execute(os.Stdout, obj); err != nil {
		fatalf(err.Error())
	}
	return nil
}

func createTemplate(name, tmplSrc string) *template.Template {
	var t *template.Template
	if tmplSrc == "" {
		return nil
	}

	t, err := template.New(name).Funcs(funcMap).Parse(tmplSrc)
	if err != nil {
		fatalf(err.Error())
	}

	return t
}

func exportedFields(typ reflect.Type) bool {
	for i := 0; i < typ.NumField(); i++ {
		if typ.Field(i).PkgPath == "" {
			return true
		}
	}

	return false
}

func ignoreKind(kind reflect.Kind) bool {
	return (kind == reflect.Chan) || (kind == reflect.Invalid)
}

func generateStruct(buf io.Writer, typ reflect.Type) {
	for i := 0; i < typ.NumField(); i++ {
		field := typ.Field(i)
		if field.PkgPath != "" || ignoreKind(field.Type.Kind()) {
			continue
		}
		fmt.Fprintf(buf, "%s ", field.Name)
		tag := ""
		if field.Tag != "" {
			tag = fmt.Sprintf("`%s`", field.Tag)
		}
		generateUsage(buf, field.Type, tag)
	}
}

func generateUsage(buf io.Writer, typ reflect.Type, tag string) {
	kind := typ.Kind()
	if ignoreKind(kind) {
		return
	}

	switch kind {
	case reflect.Struct:
		if exportedFields(typ) {
			fmt.Fprintf(buf, "struct {\n")
			generateStruct(buf, typ)
			fmt.Fprintf(buf, "}%s\n", tag)
		} else if typ.Name() != "" {
			fmt.Fprintf(buf, "%s%s\n", typ.String(), tag)
		} else {
			fmt.Fprintf(buf, "struct {\n}%s\n", tag)
		}
	case reflect.Slice:
		fmt.Fprintf(buf, "[]")
		generateUsage(buf, typ.Elem(), tag)
	case reflect.Array:
		fmt.Fprintf(buf, "[%d]", typ.Len())
		generateUsage(buf, typ.Elem(), tag)
	case reflect.Map:
		fmt.Fprintf(buf, "map[%s]", typ.Key().String())
		generateUsage(buf, typ.Elem(), tag)
	default:
		fmt.Fprintf(buf, "%s%s\n", typ.String(), tag)
	}
}

func formatType(buf *bytes.Buffer, unformattedType []byte) {
	const typePrefix = "type x "
	source := bytes.NewBufferString(typePrefix)
	_, _ = source.Write(unformattedType)
	formattedType, err := format.Source(source.Bytes())
	if err != nil {
		panic(fmt.Errorf("generateIndentedUsage created invalid Go code: %v", err))
	}
	_, _ = buf.Write(formattedType[len(typePrefix):])
}

func generateIndentedUsage(buf *bytes.Buffer, i interface{}) {
	var source bytes.Buffer
	generateUsage(&source, reflect.TypeOf(i), "")
	formatType(buf, source.Bytes())
}

func generateUsageUndecorated(i interface{}) string {
	var buf bytes.Buffer
	generateIndentedUsage(&buf, i)
	return buf.String()
}

func generateUsageDecorated(flag string, i interface{}) string {
	var buf bytes.Buffer

	fmt.Fprintf(&buf,
		"The template passed to the -%s option operates on a\n\n",
		flag)

	generateIndentedUsage(&buf, i)
	fmt.Fprintf(&buf, templateFunctionHelp)
	return buf.String()
}
<|endoftext|>


<commit_before>package perfTestUtils

import (
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"io"
	"io/ioutil"
	"os"
	"sort"
	"strconv"
	"time"
)

//FileSystem is an interface to access os filesystem or mock it
type FileSystem interface {
	Open(name string) (File, error)
	Create(name string) (File, error)
}

//File is an interface to access os.File or mock it
type File interface {
	Readdir(n int) (fi []os.FileInfo, err error)
	io.WriteCloser
	Read(p []byte) (n int, err error)
}

// OsFS implements fileSystem using the local disk.
type OsFS struct{}

//Open calls os function
func (OsFS) Open(name string) (File, error) { return os.Open(name) }

//Create calls os function
func (OsFS) Create(name string) (File, error) { return os.Create(name) }

//=============================
//Test run utility functions
//=============================
//This function reads a base perf and converts it to a base perf struct
func ReadBasePerfFile(r io.Reader) (*BasePerfStats, error) {
	basePerfstats := &BasePerfStats{
		BaseServiceResponseTimes: make(map[string]int64),
		MemoryAudit:              make([]uint64, 0),
	}
	var errorFound error

	content, err := ioutil.ReadAll(r)
	if err != nil {
		errorFound = err
	} else {
		jsonError := json.Unmarshal(content, basePerfstats)
		if jsonError != nil {
			errorFound = jsonError
		}
	}
	return basePerfstats, errorFound
}

func ValidateTestDefinitionAmount(baselineAmount int, configurationSettings *Config, fs FileSystem) bool {

	d, err := fs.Open(configurationSettings.TestCaseDir)
	if err != nil {
		log.Error("Failed to open test definitions directory. Error:", err)
		os.Exit(1)
	}
	defer d.Close()
	fi, err := d.Readdir(-1)
	if err != nil {
		log.Error("Failed to read files in test definitions directory. Error:", err)
		os.Exit(1)
	}

	definitionAmount := len(fi)

	log.Info("Number of defined test cases:", definitionAmount)
	log.Info("Number of base line test cases:", baselineAmount)
	if definitionAmount != baselineAmount {
		log.Error(fmt.Sprintf("Amount of test definition: %d does not equal to baseline amount: %d.\n", definitionAmount, baselineAmount))
		return false
	}
	return true
}

func GetExecutionTimeDisplay(executionTime int64) string {
	timeInMilliSeconds := executionTime / 1000000
	seconds := (timeInMilliSeconds / 1000)
	secondsDisplay := seconds % 60
	minutes := seconds / 60
	minutesDisplay := minutes % 60

	displayStatement := []byte("")
	displayStatement = append(displayStatement, []byte(strconv.FormatInt(minutesDisplay, 10))...)
	displayStatement = append(displayStatement, []byte(":")...)
	if secondsDisplay <= 9 {
		displayStatement = append(displayStatement, []byte("0")...)
	}
	displayStatement = append(displayStatement, []byte(strconv.FormatInt(secondsDisplay, 10))...)
	return string(displayStatement)
}

func IsReadyForTest(configurationSettings *Config, osFileSystem OsFS, testSuiteName string) (bool, *BasePerfStats) {

	//1) read in perf base stats
	f, err := os.Open(configurationSettings.BaseStatsOutputDir + "/" + configurationSettings.ExecutionHost + "-" + testSuiteName + "-perfBaseStats")
	if err != nil {
		log.Errorf("Failed to open env stats for %v. Error: %v.", configurationSettings.ExecutionHost, err)
		return false, nil
	}
	basePerfstats, err := ReadBasePerfFile(f)
	if err != nil {
		log.Error("Failed to read env stats for " + configurationSettings.ExecutionHost + ". Error:" + err.Error() + ".")
		return false, nil
	}

	//2) validate content  of base stats file
	isBasePerfStatsValid := validateBasePerfStat(basePerfstats)
	if !isBasePerfStatsValid {
		log.Error("Base Perf stats are not fully populated for  " + configurationSettings.ExecutionHost + ".")
		return false, nil
	}
	//3) Verify the number of base test cases is equal to the number of service test cases.
	correctNumberOfTests := ValidateTestDefinitionAmount(len(basePerfstats.BaseServiceResponseTimes), configurationSettings, osFileSystem)
	if !correctNumberOfTests {
		return false, nil
	}

	return true, basePerfstats
}

func validateBasePerfStat(basePerfstats *BasePerfStats) bool {
	isBasePerfStatsValid := true

	if basePerfstats.BasePeakMemory <= 0 {
		isBasePerfStatsValid = false
	}
	if basePerfstats.GenerationDate == "" {
		isBasePerfStatsValid = false
	}
	if basePerfstats.ModifiedDate == "" {
		isBasePerfStatsValid = false
	}
	if len(basePerfstats.MemoryAudit) <= 0 {
		isBasePerfStatsValid = false
	}
	if basePerfstats.BaseServiceResponseTimes != nil {
		for _, baseResponseTime := range basePerfstats.BaseServiceResponseTimes {
			if baseResponseTime <= 0 {
				isBasePerfStatsValid = false
				break
			}
		}
	} else {
		isBasePerfStatsValid = false
	}
	return isBasePerfStatsValid
}

//=====================
//Calc Memory functions
//=====================
func CalcPeakMemoryVariancePercentage(basePeakMemory uint64, peakMemory uint64) float64 {

	peakMemoryVariancePercentage := float64(0)

	if basePeakMemory < peakMemory {
		peakMemoryDelta := peakMemory - basePeakMemory
		temp := float64(float64(peakMemoryDelta) / float64(basePeakMemory))
		peakMemoryVariancePercentage = temp * 100
	} else {
		peakMemoryDelta := basePeakMemory - peakMemory
		temp := float64(float64(peakMemoryDelta) / float64(basePeakMemory))
		peakMemoryVariancePercentage = (temp * 100) * -1
	}

	return peakMemoryVariancePercentage
}

//============================
//Calc Response time functions
//============================
func CalcAverageResponseTime(responseTimes RspTimes, numIterations int) int64 {

	averageResponseTime := int64(0)

	//Remove the highest =10% to take out anomolies
	sort.Sort(responseTimes)
	numberToRemove := int(float32(numIterations) * float32(0.1))
	responseTimes = responseTimes[0 : len(responseTimes)-numberToRemove]

	totalOfAllresponseTimes := int64(0)
	for _, val := range responseTimes {
		totalOfAllresponseTimes = totalOfAllresponseTimes + val
	}
	averageResponseTime = int64(float64(totalOfAllresponseTimes) / float64(numIterations-numberToRemove))

	return averageResponseTime
}

func CalcAverageResponseVariancePercentage(averageResponseTime int64, baseResponseTime int64) float64 {

	responseTimeVariancePercentage := float64(0)

	if baseResponseTime < averageResponseTime {
		delta := uint64(averageResponseTime) - uint64(baseResponseTime)
		temp := float64(float64(delta) / float64(baseResponseTime))
		responseTimeVariancePercentage = temp * 100
	} else {
		delta := baseResponseTime - averageResponseTime
		temp := float64(float64(delta) / float64(baseResponseTime))
		responseTimeVariancePercentage = (temp * 100) * -1
	}

	return responseTimeVariancePercentage
}

//=====================================
//Service response validation functions
//=====================================
func ValidateResponseStatusCode(responseStatusCode int, expectedStatusCode int, testName string) bool {

	isResponseStatusCodeValid := false
	if responseStatusCode == expectedStatusCode {
		isResponseStatusCodeValid = true
	} else {
		log.Error(fmt.Sprintf("Incorrect status code of %d retruned for service %s. %d expected", responseStatusCode, testName, expectedStatusCode))
	}
	return isResponseStatusCodeValid
}

func ValidateServiceResponseTime(responseTime int64, testName string) bool {

	isResponseTimeValid := false
	if responseTime > 0 {
		isResponseTimeValid = true
	} else {
		log.Error(fmt.Sprintf("Time taken to complete request %s was 0 nanoseconds", testName))
	}
	return isResponseTimeValid
}

//=====================================
//Test Assertion functions
//=====================================
func ValidatePeakMemoryVariance(allowablePeakMemoryVariance float64, peakMemoryVariancePercentage float64) bool {

	if allowablePeakMemoryVariance >= peakMemoryVariancePercentage {
		return true
	} else {
		return false
	}
}

func ValidateAverageServiceResponeTimeVariance(allowableServiceResponseTimeVariance float64, serviceResponseTimeVariancePercentage float64, serviceName string) bool {
	if allowableServiceResponseTimeVariance >= serviceResponseTimeVariancePercentage {
		return true
	} else {
		return false
	}
}

//=====================================
//Response times sort functions
//=====================================
func (a RspTimes) Len() int           { return len(a) }
func (a RspTimes) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a RspTimes) Less(i, j int) bool { return a[i] < a[j] }

//==============================================
//Generate base environment stats file functions
//==============================================
func populateBasePerfStats(perfStatsForTest *PerfStats, basePerfstats *BasePerfStats, reBaseMemory bool) {
	modified := false

	//Setting memory data
	if basePerfstats.BasePeakMemory == 0 || reBaseMemory {
		basePerfstats.BasePeakMemory = perfStatsForTest.PeakMemory
		modified = true
	}
	if basePerfstats.MemoryAudit == nil || len(basePerfstats.MemoryAudit) == 0 || reBaseMemory {
		basePerfstats.MemoryAudit = perfStatsForTest.MemoryAudit
		modified = true
	}

	//Setting service response time data
	for serviceName, responseTime := range perfStatsForTest.ServiceResponseTimes {
		serviceBaseResponseTime := basePerfstats.BaseServiceResponseTimes[serviceName]
		if serviceBaseResponseTime == 0 {
			basePerfstats.BaseServiceResponseTimes[serviceName] = responseTime
			modified = true
		}
	}

	//Setting time stamps
	currentTime := time.Now().Format(time.RFC850)
	if basePerfstats.GenerationDate == "" {
		basePerfstats.GenerationDate = currentTime
	}
	if modified {
		basePerfstats.ModifiedDate = currentTime
	}
}

func GenerateEnvBasePerfOutputFile(perfStatsForTest *PerfStats, basePerfstats *BasePerfStats, configurationSettings *Config, exit func(code int), fs FileSystem, testSuiteName string) {

	//Set base performance based on training test run
	populateBasePerfStats(perfStatsForTest, basePerfstats, configurationSettings.ReBaseMemory)

	//Convert base perf stat to Json and write out to file
	basePerfstatsJson, err := json.Marshal(basePerfstats)
	if err != nil {
		log.Error("Failed to marshal to Json. Error:", err)
		exit(1)
	}
	file, err := fs.Create(configurationSettings.BaseStatsOutputDir + "/" + configurationSettings.ExecutionHost + "-" + testSuiteName + "-perfBaseStats")
	if err != nil {
		log.Error("Failed to create output file. Error:", err)
		exit(1)
	}
	if file != nil {
		defer file.Close()
		file.Write(basePerfstatsJson)
	}
}
<commit_msg>Updated utils.go to create the output directory if it doesn't exist.<commit_after>package perfTestUtils

import (
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"io"
	"io/ioutil"
	"os"
	"sort"
	"strconv"
	"time"
)

//FileSystem is an interface to access os filesystem or mock it
type FileSystem interface {
	Open(name string) (File, error)
	Create(name string) (File, error)
}

//File is an interface to access os.File or mock it
type File interface {
	Readdir(n int) (fi []os.FileInfo, err error)
	io.WriteCloser
	Read(p []byte) (n int, err error)
}

// OsFS implements fileSystem using the local disk.
type OsFS struct{}

//Open calls os function
func (OsFS) Open(name string) (File, error) { return os.Open(name) }

//Create calls os function
func (OsFS) Create(name string) (File, error) { return os.Create(name) }

//=============================
//Test run utility functions
//=============================
//This function reads a base perf and converts it to a base perf struct
func ReadBasePerfFile(r io.Reader) (*BasePerfStats, error) {
	basePerfstats := &BasePerfStats{
		BaseServiceResponseTimes: make(map[string]int64),
		MemoryAudit:              make([]uint64, 0),
	}
	var errorFound error

	content, err := ioutil.ReadAll(r)
	if err != nil {
		errorFound = err
	} else {
		jsonError := json.Unmarshal(content, basePerfstats)
		if jsonError != nil {
			errorFound = jsonError
		}
	}
	return basePerfstats, errorFound
}

func ValidateTestDefinitionAmount(baselineAmount int, configurationSettings *Config, fs FileSystem) bool {

	d, err := fs.Open(configurationSettings.TestCaseDir)
	if err != nil {
		log.Error("Failed to open test definitions directory. Error:", err)
		os.Exit(1)
	}
	defer d.Close()
	fi, err := d.Readdir(-1)
	if err != nil {
		log.Error("Failed to read files in test definitions directory. Error:", err)
		os.Exit(1)
	}

	definitionAmount := len(fi)

	log.Info("Number of defined test cases:", definitionAmount)
	log.Info("Number of base line test cases:", baselineAmount)
	if definitionAmount != baselineAmount {
		log.Error(fmt.Sprintf("Amount of test definition: %d does not equal to baseline amount: %d.\n", definitionAmount, baselineAmount))
		return false
	}
	return true
}

func GetExecutionTimeDisplay(executionTime int64) string {
	timeInMilliSeconds := executionTime / 1000000
	seconds := (timeInMilliSeconds / 1000)
	secondsDisplay := seconds % 60
	minutes := seconds / 60
	minutesDisplay := minutes % 60

	displayStatement := []byte("")
	displayStatement = append(displayStatement, []byte(strconv.FormatInt(minutesDisplay, 10))...)
	displayStatement = append(displayStatement, []byte(":")...)
	if secondsDisplay <= 9 {
		displayStatement = append(displayStatement, []byte("0")...)
	}
	displayStatement = append(displayStatement, []byte(strconv.FormatInt(secondsDisplay, 10))...)
	return string(displayStatement)
}

func IsReadyForTest(configurationSettings *Config, osFileSystem OsFS, testSuiteName string) (bool, *BasePerfStats) {

	//1) read in perf base stats
	f, err := os.Open(configurationSettings.BaseStatsOutputDir + "/" + configurationSettings.ExecutionHost + "-" + testSuiteName + "-perfBaseStats")
	if err != nil {
		log.Errorf("Failed to open env stats for %v. Error: %v.", configurationSettings.ExecutionHost, err)
		return false, nil
	}
	basePerfstats, err := ReadBasePerfFile(f)
	if err != nil {
		log.Error("Failed to read env stats for " + configurationSettings.ExecutionHost + ". Error:" + err.Error() + ".")
		return false, nil
	}

	//2) validate content  of base stats file
	isBasePerfStatsValid := validateBasePerfStat(basePerfstats)
	if !isBasePerfStatsValid {
		log.Error("Base Perf stats are not fully populated for  " + configurationSettings.ExecutionHost + ".")
		return false, nil
	}
	//3) Verify the number of base test cases is equal to the number of service test cases.
	correctNumberOfTests := ValidateTestDefinitionAmount(len(basePerfstats.BaseServiceResponseTimes), configurationSettings, osFileSystem)
	if !correctNumberOfTests {
		return false, nil
	}

	return true, basePerfstats
}

func validateBasePerfStat(basePerfstats *BasePerfStats) bool {
	isBasePerfStatsValid := true

	if basePerfstats.BasePeakMemory <= 0 {
		isBasePerfStatsValid = false
	}
	if basePerfstats.GenerationDate == "" {
		isBasePerfStatsValid = false
	}
	if basePerfstats.ModifiedDate == "" {
		isBasePerfStatsValid = false
	}
	if len(basePerfstats.MemoryAudit) <= 0 {
		isBasePerfStatsValid = false
	}
	if basePerfstats.BaseServiceResponseTimes != nil {
		for _, baseResponseTime := range basePerfstats.BaseServiceResponseTimes {
			if baseResponseTime <= 0 {
				isBasePerfStatsValid = false
				break
			}
		}
	} else {
		isBasePerfStatsValid = false
	}
	return isBasePerfStatsValid
}

//=====================
//Calc Memory functions
//=====================
func CalcPeakMemoryVariancePercentage(basePeakMemory uint64, peakMemory uint64) float64 {

	peakMemoryVariancePercentage := float64(0)

	if basePeakMemory < peakMemory {
		peakMemoryDelta := peakMemory - basePeakMemory
		temp := float64(float64(peakMemoryDelta) / float64(basePeakMemory))
		peakMemoryVariancePercentage = temp * 100
	} else {
		peakMemoryDelta := basePeakMemory - peakMemory
		temp := float64(float64(peakMemoryDelta) / float64(basePeakMemory))
		peakMemoryVariancePercentage = (temp * 100) * -1
	}

	return peakMemoryVariancePercentage
}

//============================
//Calc Response time functions
//============================
func CalcAverageResponseTime(responseTimes RspTimes, numIterations int) int64 {

	averageResponseTime := int64(0)

	//Remove the highest =10% to take out anomolies
	sort.Sort(responseTimes)
	numberToRemove := int(float32(numIterations) * float32(0.1))
	responseTimes = responseTimes[0 : len(responseTimes)-numberToRemove]

	totalOfAllresponseTimes := int64(0)
	for _, val := range responseTimes {
		totalOfAllresponseTimes = totalOfAllresponseTimes + val
	}
	averageResponseTime = int64(float64(totalOfAllresponseTimes) / float64(numIterations-numberToRemove))

	return averageResponseTime
}

func CalcAverageResponseVariancePercentage(averageResponseTime int64, baseResponseTime int64) float64 {

	responseTimeVariancePercentage := float64(0)

	if baseResponseTime < averageResponseTime {
		delta := uint64(averageResponseTime) - uint64(baseResponseTime)
		temp := float64(float64(delta) / float64(baseResponseTime))
		responseTimeVariancePercentage = temp * 100
	} else {
		delta := baseResponseTime - averageResponseTime
		temp := float64(float64(delta) / float64(baseResponseTime))
		responseTimeVariancePercentage = (temp * 100) * -1
	}

	return responseTimeVariancePercentage
}

//=====================================
//Service response validation functions
//=====================================
func ValidateResponseStatusCode(responseStatusCode int, expectedStatusCode int, testName string) bool {

	isResponseStatusCodeValid := false
	if responseStatusCode == expectedStatusCode {
		isResponseStatusCodeValid = true
	} else {
		log.Error(fmt.Sprintf("Incorrect status code of %d retruned for service %s. %d expected", responseStatusCode, testName, expectedStatusCode))
	}
	return isResponseStatusCodeValid
}

func ValidateServiceResponseTime(responseTime int64, testName string) bool {

	isResponseTimeValid := false
	if responseTime > 0 {
		isResponseTimeValid = true
	} else {
		log.Error(fmt.Sprintf("Time taken to complete request %s was 0 nanoseconds", testName))
	}
	return isResponseTimeValid
}

//=====================================
//Test Assertion functions
//=====================================
func ValidatePeakMemoryVariance(allowablePeakMemoryVariance float64, peakMemoryVariancePercentage float64) bool {

	if allowablePeakMemoryVariance >= peakMemoryVariancePercentage {
		return true
	} else {
		return false
	}
}

func ValidateAverageServiceResponeTimeVariance(allowableServiceResponseTimeVariance float64, serviceResponseTimeVariancePercentage float64, serviceName string) bool {
	if allowableServiceResponseTimeVariance >= serviceResponseTimeVariancePercentage {
		return true
	} else {
		return false
	}
}

//=====================================
//Response times sort functions
//=====================================
func (a RspTimes) Len() int           { return len(a) }
func (a RspTimes) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a RspTimes) Less(i, j int) bool { return a[i] < a[j] }

//==============================================
//Generate base environment stats file functions
//==============================================
func populateBasePerfStats(perfStatsForTest *PerfStats, basePerfstats *BasePerfStats, reBaseMemory bool) {
	modified := false

	//Setting memory data
	if basePerfstats.BasePeakMemory == 0 || reBaseMemory {
		basePerfstats.BasePeakMemory = perfStatsForTest.PeakMemory
		modified = true
	}
	if basePerfstats.MemoryAudit == nil || len(basePerfstats.MemoryAudit) == 0 || reBaseMemory {
		basePerfstats.MemoryAudit = perfStatsForTest.MemoryAudit
		modified = true
	}

	//Setting service response time data
	for serviceName, responseTime := range perfStatsForTest.ServiceResponseTimes {
		serviceBaseResponseTime := basePerfstats.BaseServiceResponseTimes[serviceName]
		if serviceBaseResponseTime == 0 {
			basePerfstats.BaseServiceResponseTimes[serviceName] = responseTime
			modified = true
		}
	}

	//Setting time stamps
	currentTime := time.Now().Format(time.RFC850)
	if basePerfstats.GenerationDate == "" {
		basePerfstats.GenerationDate = currentTime
	}
	if modified {
		basePerfstats.ModifiedDate = currentTime
	}
}

func GenerateEnvBasePerfOutputFile(perfStatsForTest *PerfStats, basePerfstats *BasePerfStats, configurationSettings *Config, exit func(code int), fs FileSystem, testSuiteName string) {

	//Set base performance based on training test run
	populateBasePerfStats(perfStatsForTest, basePerfstats, configurationSettings.ReBaseMemory)

	//Convert base perf stat to Json
	basePerfstatsJson, err := json.Marshal(basePerfstats)
	if err != nil {
		log.Error("Failed to marshal to Json. Error:", err)
		exit(1)
	}

	// Check for existence of output dir and create if needed.
	if os.MkdirAll( configurationSettings.BaseStatsOutputDir, os.ModePerm ); err != nil {
		log.Errorf( "Failed to create path: [%s]. Error: %s\n", configurationSettings.BaseStatsOutputDir, err )
		exit(1)
	}

	// Write base perf stat to file.
	file_name := configurationSettings.ExecutionHost + "-" + testSuiteName + "-perfBaseStats"
	file, err := fs.Create( configurationSettings.BaseStatsOutputDir + "/" + file_name )
	if err != nil {
		log.Error("Failed to create output file. Error:", err)
		exit(1)
	}
	if file != nil {
		defer file.Close()
		file.Write(basePerfstatsJson)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"syscall"

	"github.com/golang/protobuf/proto"
	"github.com/checkpoint-restore/criu/lib/go/src/criu"
	"github.com/checkpoint-restore/criu/lib/go/src/rpc"
	"github.com/checkpoint-restore/criu/phaul/src/phaul"
)

type testLocal struct {
	criu.CriuNoNotify
	r *testRemote
}

type testRemote struct {
	srv *phaul.PhaulServer
}

/* Dir where test will put dump images */
const images_dir = "test_images"

func prepareImages() error {
	err := os.Mkdir(images_dir, 0700)
	if err != nil {
		return err
	}

	/* Work dir for PhaulClient */
	err = os.Mkdir(images_dir+"/local", 0700)
	if err != nil {
		return err
	}

	/* Work dir for PhaulServer */
	err = os.Mkdir(images_dir+"/remote", 0700)
	if err != nil {
		return err
	}

	/* Work dir for DumpCopyRestore */
	err = os.Mkdir(images_dir+"/test", 0700)
	if err != nil {
		return err
	}

	return nil
}

func mergeImages(dump_dir, last_pre_dump_dir string) error {
	idir, err := os.Open(dump_dir)
	if err != nil {
		return err
	}

	defer idir.Close()

	imgs, err := idir.Readdirnames(0)
	if err != nil {
		return err
	}

	for _, fname := range imgs {
		if !strings.HasSuffix(fname, ".img") {
			continue
		}

		fmt.Printf("\t%s -> %s/\n", fname, last_pre_dump_dir)
		err = syscall.Link(dump_dir+"/"+fname, last_pre_dump_dir+"/"+fname)
		if err != nil {
			return err
		}
	}

	return nil
}

func (r *testRemote) doRestore() error {
	last_srv_images_dir := r.srv.LastImagesDir()
	/*
	 * In images_dir we have images from dump, in the
	 * last_srv_images_dir -- where server-side images
	 * (from page server, with pages and pagemaps) are.
	 * Need to put former into latter and restore from
	 * them.
	 */
	err := mergeImages(images_dir+"/test", last_srv_images_dir)
	if err != nil {
		return err
	}

	img_dir, err := os.Open(last_srv_images_dir)
	if err != nil {
		return err
	}
	defer img_dir.Close()

	opts := rpc.CriuOpts{
		LogLevel:    proto.Int32(4),
		LogFile:     proto.String("restore.log"),
		ImagesDirFd: proto.Int32(int32(img_dir.Fd())),
	}

	cr := r.srv.GetCriu()
	fmt.Printf("Do restore\n")
	return cr.Restore(opts, nil)
}

func (l *testLocal) PostDump() error {
	return l.r.doRestore()
}

func (l *testLocal) DumpCopyRestore(cr *criu.Criu, cfg phaul.PhaulConfig, last_cln_images_dir string) error {
	fmt.Printf("Final stage\n")

	img_dir, err := os.Open(images_dir + "/test")
	if err != nil {
		return err
	}
	defer img_dir.Close()

	psi := rpc.CriuPageServerInfo{
		Fd: proto.Int32(int32(cfg.Memfd)),
	}

	opts := rpc.CriuOpts{
		Pid:         proto.Int32(int32(cfg.Pid)),
		LogLevel:    proto.Int32(4),
		LogFile:     proto.String("dump.log"),
		ImagesDirFd: proto.Int32(int32(img_dir.Fd())),
		TrackMem:    proto.Bool(true),
		ParentImg:   proto.String(last_cln_images_dir),
		Ps:          &psi,
	}

	fmt.Printf("Do dump\n")
	return cr.Dump(opts, l)
}

func main() {
	pid, _ := strconv.Atoi(os.Args[1])
	fds, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_STREAM, 0)
	if err != nil {
		fmt.Printf("Can't make socketpair: %v\n", err)
		return
	}

	err = prepareImages()
	if err != nil {
		fmt.Printf("Can't prepare dirs for images: %v\n", err)
		return
	}

	fmt.Printf("Make server part (socket %d)\n", fds[1])
	srv, err := phaul.MakePhaulServer(phaul.PhaulConfig{
		Pid:   pid,
		Memfd: fds[1],
		Wdir:  images_dir + "/remote"})
	if err != nil {
		return
	}

	r := &testRemote{srv}

	fmt.Printf("Make client part (socket %d)\n", fds[0])
	cln, err := phaul.MakePhaulClient(&testLocal{r: r}, srv,
		phaul.PhaulConfig{
			Pid:   pid,
			Memfd: fds[0],
			Wdir:  images_dir + "/local"})
	if err != nil {
		return
	}

	fmt.Printf("Migrate\n")
	err = cln.Migrate()
	if err != nil {
		fmt.Printf("Failed: %v\n", err)
		return
	}

	fmt.Printf("SUCCESS!\n")
}
<commit_msg>phaul/test: exit with a non-zero code in error cases<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"syscall"

	"github.com/golang/protobuf/proto"
	"github.com/checkpoint-restore/criu/lib/go/src/criu"
	"github.com/checkpoint-restore/criu/lib/go/src/rpc"
	"github.com/checkpoint-restore/criu/phaul/src/phaul"
)

type testLocal struct {
	criu.CriuNoNotify
	r *testRemote
}

type testRemote struct {
	srv *phaul.PhaulServer
}

/* Dir where test will put dump images */
const images_dir = "test_images"

func prepareImages() error {
	err := os.Mkdir(images_dir, 0700)
	if err != nil {
		return err
	}

	/* Work dir for PhaulClient */
	err = os.Mkdir(images_dir+"/local", 0700)
	if err != nil {
		return err
	}

	/* Work dir for PhaulServer */
	err = os.Mkdir(images_dir+"/remote", 0700)
	if err != nil {
		return err
	}

	/* Work dir for DumpCopyRestore */
	err = os.Mkdir(images_dir+"/test", 0700)
	if err != nil {
		return err
	}

	return nil
}

func mergeImages(dump_dir, last_pre_dump_dir string) error {
	idir, err := os.Open(dump_dir)
	if err != nil {
		return err
	}

	defer idir.Close()

	imgs, err := idir.Readdirnames(0)
	if err != nil {
		return err
	}

	for _, fname := range imgs {
		if !strings.HasSuffix(fname, ".img") {
			continue
		}

		fmt.Printf("\t%s -> %s/\n", fname, last_pre_dump_dir)
		err = syscall.Link(dump_dir+"/"+fname, last_pre_dump_dir+"/"+fname)
		if err != nil {
			return err
		}
	}

	return nil
}

func (r *testRemote) doRestore() error {
	last_srv_images_dir := r.srv.LastImagesDir()
	/*
	 * In images_dir we have images from dump, in the
	 * last_srv_images_dir -- where server-side images
	 * (from page server, with pages and pagemaps) are.
	 * Need to put former into latter and restore from
	 * them.
	 */
	err := mergeImages(images_dir+"/test", last_srv_images_dir)
	if err != nil {
		return err
	}

	img_dir, err := os.Open(last_srv_images_dir)
	if err != nil {
		return err
	}
	defer img_dir.Close()

	opts := rpc.CriuOpts{
		LogLevel:    proto.Int32(4),
		LogFile:     proto.String("restore.log"),
		ImagesDirFd: proto.Int32(int32(img_dir.Fd())),
	}

	cr := r.srv.GetCriu()
	fmt.Printf("Do restore\n")
	return cr.Restore(opts, nil)
}

func (l *testLocal) PostDump() error {
	return l.r.doRestore()
}

func (l *testLocal) DumpCopyRestore(cr *criu.Criu, cfg phaul.PhaulConfig, last_cln_images_dir string) error {
	fmt.Printf("Final stage\n")

	img_dir, err := os.Open(images_dir + "/test")
	if err != nil {
		return err
	}
	defer img_dir.Close()

	psi := rpc.CriuPageServerInfo{
		Fd: proto.Int32(int32(cfg.Memfd)),
	}

	opts := rpc.CriuOpts{
		Pid:         proto.Int32(int32(cfg.Pid)),
		LogLevel:    proto.Int32(4),
		LogFile:     proto.String("dump.log"),
		ImagesDirFd: proto.Int32(int32(img_dir.Fd())),
		TrackMem:    proto.Bool(true),
		ParentImg:   proto.String(last_cln_images_dir),
		Ps:          &psi,
	}

	fmt.Printf("Do dump\n")
	return cr.Dump(opts, l)
}

func main() {
	pid, _ := strconv.Atoi(os.Args[1])
	fds, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_STREAM, 0)
	if err != nil {
		fmt.Printf("Can't make socketpair: %v\n", err)
		os.Exit(1)
	}

	err = prepareImages()
	if err != nil {
		fmt.Printf("Can't prepare dirs for images: %v\n", err)
		os.Exit(1)
		return
	}

	fmt.Printf("Make server part (socket %d)\n", fds[1])
	srv, err := phaul.MakePhaulServer(phaul.PhaulConfig{
		Pid:   pid,
		Memfd: fds[1],
		Wdir:  images_dir + "/remote"})
	if err != nil {
		fmt.Printf("Unable to run a server: %v", err)
		os.Exit(1)
		return
	}

	r := &testRemote{srv}

	fmt.Printf("Make client part (socket %d)\n", fds[0])
	cln, err := phaul.MakePhaulClient(&testLocal{r: r}, srv,
		phaul.PhaulConfig{
			Pid:   pid,
			Memfd: fds[0],
			Wdir:  images_dir + "/local"})
	if err != nil {
		fmt.Printf("Unable to run a client: %v\n", err);
		os.Exit(1)
	}

	fmt.Printf("Migrate\n")
	err = cln.Migrate()
	if err != nil {
		fmt.Printf("Failed: %v\n", err)
		os.Exit(1)
	}

	fmt.Printf("SUCCESS!\n")
}
<|endoftext|>


<commit_before>package philarios

import (
  "regexp"
  "github.com/wangjohn/quickselect"

  "fmt"
  "strings"
)

type WordVector struct {
  Word string
  Score float64
}

type WordVectorCollection []WordVector

func (t WordVectorCollection) Len() int {
  return len(t)
}

func (t WordVectorCollection) Less(i, j int) bool {
  return synonymScore(t[i].Score) < synonymScore(t[j].Score)
}

func (t WordVectorCollection) Swap(i, j int) {
  t[i], t[j] = t[j], t[i]
}

/*
AlternativeWords returns alternative words that can be used in place of the
current word. The words that are returned will have close meanings to the word
used as an argument, but which are usually used in place of that word.

The maxWords parameter specifies the maximum number of words to return.
*/
func AlternativeWords(word string, maxWords int) ([]string, error) {
  var alternativeWords []string
  wordVectors := make(WordVectorCollection, 0)

  targetVectors, err := TargetVectors(word)
  if err != nil {
    return alternativeWords, err
  }
  wordVectors = append(wordVectors, targetVectors...)

  synonyms, err := Synonyms(word)
  if err != nil {
    return alternativeWords, err
  }

  for _, synonym := range synonyms {
    synonymVectors, err := TargetVectors(synonym)
    wordVectors = append(wordVectors, synonymVectors...)
  }

  var wordsToSelect int
  if len(wordVectors) < maxWords {
    wordsToSelect = len(wordVectors)
  } else {
    wordsToSelect = maxWords
  }
  quickselect.QuickSelect(wordVectors, wordsToSelect)

  alternativeWords = make([]string, wordsToSelect)
  for i := 0; i < wordsToSelect; i++ {
    alternativeWords[i] = wordVectors[i].Word
  }

  return alternativeWords, nil
}

/*
The synonymScore method specifies the mapping between a word's score and its
rank relative to other words. This function allows finer control over the words
that are returned.
*/
func synonymScore(score float64) (float64) {
  return score
}

func TargetVectors(word string) ([]WordVector, error) {
  var wordVectors []WordVector

  rows, err := QueryForWord(word)
  if err != nil {
    return wordVectors, err
  }
  defer rows.Close()

  for rows.Next() {
    var body string
    if err = rows.Scan(&body); err != nil {
      return wordVectors, err
    }

    newVectors, err := paragraphTargetVectors(body, word)
    if err != nil {
      return wordVectors, err
    }
    wordVectors = append(wordVectors, newVectors...)
  }

  return wordVectors, nil
}

func paragraphTargetVectors(paragraph, word string) ([]WordVector, error) {
  wordVectors := make([]WordVector, 0)

  regexString := fmt.Sprintf(`[\s[[:punct:]]]*%s[\s[[:punct:]]]*`, word)
  compiledRegex, err := regexp.Compile(regexString)
  if err != nil {
    return wordVectors, err
  }

  allIndices := compiledRegex.FindAllStringIndex(paragraph, -1)
  var potentialVector string
  var currentVector WordVector

  if allIndices != nil {
    for i := 0; i < len(allIndices); i++ {
      matchedIndexPair := allIndices[i]
      start, end := matchedIndexPair[0], matchedIndexPair[1]

      potentialVector, err = lookForWord(paragraph[:start], false)
      if err != nil {
        return wordVectors, err
      }
      if potentialVector != "" {
        currentVector = WordVector{potentialVector, 1.0}
        wordVectors = append(wordVectors, currentVector)
      }

      potentialVector, err := lookForWord(paragraph[end:], true)
      if err != nil {
        return wordVectors, err
      }
      if potentialVector != "" {
        currentVector = WordVector{potentialVector, 1.0}
        wordVectors = append(wordVectors, currentVector)
      }
    }
  }

  return wordVectors, nil
}

func lookForWord(paragraph string, atBeginning bool) (string, error) {
  if strings.Trim(paragraph, " ") == "" {
    return "", nil
  }

  var compiledRegex regexp.Regexp
  var err error

  if atBeginning {
    compiledRegex, err = regexp.Compile(`^(\w+)[\s[[:punct:]]]?`)
  } else {
    compiledRegex, err = regexp.Compile(`[\s[[:punct:]]]?(\w+)$`)
  }

  if err != nil {
    return "", err
  }

  return compiledRegex.FindString(paragraph), nil
}

func Synonyms(word string) ([]string, error) {
  // TODO: implement
  return nil, nil
}
<commit_msg>Creating method for alternative word vectors.<commit_after>package philarios

import (
  "regexp"
  "github.com/wangjohn/quickselect"

  "fmt"
  "strings"
)

type WordVector struct {
  Word string
  Score float64
}

type WordVectorCollection []WordVector

func (t WordVectorCollection) Len() int {
  return len(t)
}

func (t WordVectorCollection) Less(i, j int) bool {
  return synonymScore(t[i].Score) < synonymScore(t[j].Score)
}

func (t WordVectorCollection) Swap(i, j int) {
  t[i], t[j] = t[j], t[i]
}

/*
AlternativeWords returns alternative words that can be used in place of the
current word. The words that are returned will have close meanings to the word
used as an argument, but which are usually used in place of that word.

The maxWords parameter specifies the maximum number of words to return.
*/
func AlternativeWords(word string, maxWords int) ([]string, error) {
  alternativeWords = make([]string, 0)
  alternativeWordVectors, err := AlternativeWordVectors(word, maxWords)
  if err != nil {
    return alternativeWords, err
  }

  for _, wordVector := range alternativeWordVectors {
    alternativeWords = append(alternativeWords, wordVector.word)
  }

  return alternativeWords, nil
}

/*
AlternativeWordVectors returns alternative word vectors for a particular word.
*/
func AlternativeWordVectors(word string, maxWords int) ([]WordVector, error) {
  var alternativeWords []string
  wordVectors := make(WordVectorCollection, 0)

  targetVectors, err := TargetVectors(word)
  if err != nil {
    return alternativeWords, err
  }
  wordVectors = append(wordVectors, targetVectors...)

  synonyms, err := Synonyms(word)
  if err != nil {
    return alternativeWords, err
  }

  for _, synonym := range synonyms {
    synonymVectors, err := TargetVectors(synonym)
    wordVectors = append(wordVectors, synonymVectors...)
  }

  var wordsToSelect int
  if len(wordVectors) < maxWords {
    wordsToSelect = len(wordVectors)
  } else {
    wordsToSelect = maxWords
  }
  quickselect.QuickSelect(wordVectors, wordsToSelect)

  return wordVectors[wordsToSelect:], nil
}

/*
The synonymScore method specifies the mapping between a word's score and its
rank relative to other words. This function allows finer control over the words
that are returned.
*/
func synonymScore(score float64) (float64) {
  return score
}

func TargetVectors(word string) ([]WordVector, error) {
  var wordVectors []WordVector

  rows, err := QueryForWord(word)
  if err != nil {
    return wordVectors, err
  }
  defer rows.Close()

  for rows.Next() {
    var body string
    if err = rows.Scan(&body); err != nil {
      return wordVectors, err
    }

    newVectors, err := paragraphTargetVectors(body, word)
    if err != nil {
      return wordVectors, err
    }
    wordVectors = append(wordVectors, newVectors...)
  }

  return wordVectors, nil
}

func paragraphTargetVectors(paragraph, word string) ([]WordVector, error) {
  wordVectors := make([]WordVector, 0)

  regexString := fmt.Sprintf(`[\s[[:punct:]]]*%s[\s[[:punct:]]]*`, word)
  compiledRegex, err := regexp.Compile(regexString)
  if err != nil {
    return wordVectors, err
  }

  allIndices := compiledRegex.FindAllStringIndex(paragraph, -1)
  var potentialVector string
  var currentVector WordVector

  if allIndices != nil {
    for i := 0; i < len(allIndices); i++ {
      matchedIndexPair := allIndices[i]
      start, end := matchedIndexPair[0], matchedIndexPair[1]

      potentialVector, err = lookForWord(paragraph[:start], false)
      if err != nil {
        return wordVectors, err
      }
      if potentialVector != "" {
        currentVector = WordVector{potentialVector, 1.0}
        wordVectors = append(wordVectors, currentVector)
      }

      potentialVector, err := lookForWord(paragraph[end:], true)
      if err != nil {
        return wordVectors, err
      }
      if potentialVector != "" {
        currentVector = WordVector{potentialVector, 1.0}
        wordVectors = append(wordVectors, currentVector)
      }
    }
  }

  return wordVectors, nil
}

func lookForWord(paragraph string, atBeginning bool) (string, error) {
  if strings.Trim(paragraph, " ") == "" {
    return "", nil
  }

  var compiledRegex regexp.Regexp
  var err error

  if atBeginning {
    compiledRegex, err = regexp.Compile(`^(\w+)[\s[[:punct:]]]?`)
  } else {
    compiledRegex, err = regexp.Compile(`[\s[[:punct:]]]?(\w+)$`)
  }

  if err != nil {
    return "", err
  }

  return compiledRegex.FindString(paragraph), nil
}

func Synonyms(word string) ([]string, error) {
  // TODO: implement
  return nil, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package driver

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	// Preload proto definitions
	_ "github.com/cncf/udpa/go/udpa/type/v1"
	bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
	core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"

	// Preload proto definitions
	_ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"

	"istio.io/proxy/test/envoye2e/env"
)

// Envoy starts up a Envoy process locally.
type Envoy struct {
	// template for the bootstrap
	Bootstrap string

	// Istio proxy version to download.
	// This could be either a patch version (x.y.z), or a minor version (x.y), or master.
	// When minor version or master is provided, proxy binary will downloaded based on
	// the latest proxy SHA that istio minor version branch points to.
	// DownloadVersion will be ignored if proxy binary already exists at the
	// default bazel-bin location, or ENVOY_PATH env var is set.
	DownloadVersion string

	tmpFile   string
	cmd       *exec.Cmd
	adminPort uint32

	done chan error
}

var _ Step = &Envoy{}

// Run starts a Envoy process.
func (e *Envoy) Run(p *Params) error {
	bootstrap, err := p.Fill(e.Bootstrap)
	if err != nil {
		return err
	}
	log.Printf("envoy bootstrap:\n%s\n", bootstrap)

	e.adminPort, err = getAdminPort(bootstrap)
	if err != nil {
		return err
	}
	log.Printf("admin port %d", e.adminPort)

	tmp, err := ioutil.TempFile(os.TempDir(), "envoy-bootstrap-*.yaml")
	if err != nil {
		return err
	}
	if _, err := tmp.Write([]byte(bootstrap)); err != nil {
		return err
	}
	e.tmpFile = tmp.Name()

	debugLevel, ok := os.LookupEnv("ENVOY_DEBUG")
	if !ok {
		debugLevel = "info"
	}
	args := []string{
		"-c", e.tmpFile,
		"-l", debugLevel,
		"--concurrency", "1",
		"--disable-hot-restart",
		"--drain-time-s", "4", // this affects how long draining listenrs are kept alive
	}
	envoyPath := filepath.Join(env.GetDefaultEnvoyBinOrDie(), "envoy")
	if path, exists := os.LookupEnv("ENVOY_PATH"); exists {
		envoyPath = path
	} else if _, err := os.Stat(envoyPath); os.IsNotExist(err) && e.DownloadVersion != "" {
		envoyPath, err = downloadEnvoy(e.DownloadVersion)
		if err != nil {
			return fmt.Errorf("failed to download Envoy binary %v", err)
		}
	}
	cmd := exec.Command(envoyPath, args...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Dir = BazelWorkspace()

	log.Printf("envoy cmd %v", cmd.Args)
	e.cmd = cmd
	if err = cmd.Start(); err != nil {
		return err
	}
	e.done = make(chan error, 1)
	go func() {
		err := e.cmd.Wait()
		if err != nil {
			log.Printf("envoy process error: %v\n", err)
			if strings.Contains(err.Error(), "segmentation fault") {
				panic(err)
			}
		}
		e.done <- err
	}()

	url := fmt.Sprintf("http://127.0.0.1:%v/ready", e.adminPort)
	return env.WaitForHTTPServer(url)
}

// Cleanup stops the Envoy process.
func (e *Envoy) Cleanup() {
	log.Printf("stop envoy ...\n")
	defer os.Remove(e.tmpFile)
	if e.cmd != nil {
		url := fmt.Sprintf("http://127.0.0.1:%v/quitquitquit", e.adminPort)
		_, _, _ = env.HTTPPost(url, "", "")
		select {
		case <-time.After(3 * time.Second):
			log.Println("envoy killed as timeout reached")
			log.Println(e.cmd.Process.Kill())
		case <-e.done:
			log.Printf("stop envoy ... done\n")
		}
	}
}

func getAdminPort(bootstrap string) (uint32, error) {
	pb := &bootstrap_v3.Bootstrap{}
	if err := ReadYAML(bootstrap, pb); err != nil {
		return 0, err
	}
	if pb.Admin == nil || pb.Admin.Address == nil {
		return 0, fmt.Errorf("missing admin section in bootstrap: %v", bootstrap)
	}
	socket, ok := pb.Admin.Address.Address.(*core.Address_SocketAddress)
	if !ok {
		return 0, fmt.Errorf("missing socket in bootstrap: %v", bootstrap)
	}
	port, ok := socket.SocketAddress.PortSpecifier.(*core.SocketAddress_PortValue)
	if !ok {
		return 0, fmt.Errorf("missing port in bootstrap: %v", bootstrap)
	}
	return port.PortValue, nil
}

// downloads env based on the given branch name. Return location of downloaded envoy.
func downloadEnvoy(ver string) (string, error) {
	var proxyDepURL string
	if regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`).Match([]byte(ver)) {
		// this is a patch version string
		proxyDepURL = fmt.Sprintf("https://raw.githubusercontent.com/istio/istio/%v/istio.deps", ver)
	} else if regexp.MustCompile(`[0-9]+\.[0-9]+`).Match([]byte(ver)) {
		// this is a minor version string
		proxyDepURL = fmt.Sprintf("https://raw.githubusercontent.com/istio/istio/release-%v/istio.deps", ver)
	} else if ver == "master" {
		proxyDepURL = "https://raw.githubusercontent.com/istio/istio/master/istio.deps"
	} else {
		return "", fmt.Errorf("envoy version %v is neither minor version nor patch version", ver)
	}
	resp, err := http.Get(proxyDepURL)
	if err != nil {
		return "", fmt.Errorf("cannot get envoy sha from %v: %v", proxyDepURL, err)
	}
	defer resp.Body.Close()
	istioDeps, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("cannot read body of istio deps: %v", err)
	}

	var deps []interface{}
	if err := json.Unmarshal(istioDeps, &deps); err != nil {
		return "", err
	}
	proxySHA := ""
	for _, d := range deps {
		if dm, ok := d.(map[string]interface{}); ok && dm["repoName"].(string) == "proxy" {
			proxySHA = dm["lastStableSHA"].(string)
		}
	}
	if proxySHA == "" {
		return "", errors.New("cannot identify proxy SHA to download")
	}

	dir := fmt.Sprintf("%s/%s", os.TempDir(), "istio-proxy")
	dst := fmt.Sprintf("%v/envoy-%v", dir, proxySHA)
	if _, err := os.Stat(dst); err == nil {
		// If the desired envoy binary is already downloaded, skip downloading and return.
		return dst, nil
	}

	// clean up the tmp dir before downloading to remove stale envoy binary.
	_ = os.RemoveAll(dir)

	// make temp directory to put downloaded envoy binary.
	if err := os.MkdirAll(dir, 0755); err != nil {
		return "", err
	}

	envoyURL := fmt.Sprintf("https://storage.googleapis.com/istio-build/proxy/envoy-alpha-%v.tar.gz", proxySHA)
	downloadCmd := exec.Command("bash", "-c", fmt.Sprintf("curl -fLSs %v | tar xz", envoyURL))
	downloadCmd.Stderr = os.Stderr
	downloadCmd.Stdout = os.Stdout
	err = downloadCmd.Run()
	if err != nil {
		return "", fmt.Errorf("fail to run envoy download command: %v", err)
	}
	src := "usr/local/bin/envoy"
	if _, err := os.Stat(src); err != nil {
		return "", fmt.Errorf("fail to find downloaded envoy: %v", err)
	}
	defer os.RemoveAll("usr/")

	cpCmd := exec.Command("cp", src, dst)
	cpCmd.Stderr = os.Stderr
	cpCmd.Stdout = os.Stdout
	if err := cpCmd.Run(); err != nil {
		return "", fmt.Errorf("fail to copy envoy binary from %v to %v: %v", src, dst, err)
	}

	return dst, nil
}
<commit_msg>Do not override downloaded envoy if already exists. (#3225)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package driver

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	// Preload proto definitions
	_ "github.com/cncf/udpa/go/udpa/type/v1"
	bootstrap_v3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
	core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"

	// Preload proto definitions
	_ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"

	"istio.io/proxy/test/envoye2e/env"
)

// Envoy starts up a Envoy process locally.
type Envoy struct {
	// template for the bootstrap
	Bootstrap string

	// Istio proxy version to download.
	// This could be either a patch version (x.y.z), or a minor version (x.y), or master.
	// When minor version or master is provided, proxy binary will downloaded based on
	// the latest proxy SHA that istio minor version branch points to.
	// DownloadVersion will be ignored if proxy binary already exists at the
	// default bazel-bin location, or ENVOY_PATH env var is set.
	DownloadVersion string

	tmpFile   string
	cmd       *exec.Cmd
	adminPort uint32

	done chan error
}

var _ Step = &Envoy{}

// Run starts a Envoy process.
func (e *Envoy) Run(p *Params) error {
	bootstrap, err := p.Fill(e.Bootstrap)
	if err != nil {
		return err
	}
	log.Printf("envoy bootstrap:\n%s\n", bootstrap)

	e.adminPort, err = getAdminPort(bootstrap)
	if err != nil {
		return err
	}
	log.Printf("admin port %d", e.adminPort)

	tmp, err := ioutil.TempFile(os.TempDir(), "envoy-bootstrap-*.yaml")
	if err != nil {
		return err
	}
	if _, err := tmp.Write([]byte(bootstrap)); err != nil {
		return err
	}
	e.tmpFile = tmp.Name()

	debugLevel, ok := os.LookupEnv("ENVOY_DEBUG")
	if !ok {
		debugLevel = "info"
	}
	args := []string{
		"-c", e.tmpFile,
		"-l", debugLevel,
		"--concurrency", "1",
		"--disable-hot-restart",
		"--drain-time-s", "4", // this affects how long draining listenrs are kept alive
	}
	envoyPath := filepath.Join(env.GetDefaultEnvoyBinOrDie(), "envoy")
	if path, exists := os.LookupEnv("ENVOY_PATH"); exists {
		envoyPath = path
	} else if _, err := os.Stat(envoyPath); os.IsNotExist(err) && e.DownloadVersion != "" {
		envoyPath, err = downloadEnvoy(e.DownloadVersion)
		if err != nil {
			return fmt.Errorf("failed to download Envoy binary %v", err)
		}
	}
	cmd := exec.Command(envoyPath, args...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Dir = BazelWorkspace()

	log.Printf("envoy cmd %v", cmd.Args)
	e.cmd = cmd
	if err = cmd.Start(); err != nil {
		return err
	}
	e.done = make(chan error, 1)
	go func() {
		err := e.cmd.Wait()
		if err != nil {
			log.Printf("envoy process error: %v\n", err)
			if strings.Contains(err.Error(), "segmentation fault") {
				panic(err)
			}
		}
		e.done <- err
	}()

	url := fmt.Sprintf("http://127.0.0.1:%v/ready", e.adminPort)
	return env.WaitForHTTPServer(url)
}

// Cleanup stops the Envoy process.
func (e *Envoy) Cleanup() {
	log.Printf("stop envoy ...\n")
	defer os.Remove(e.tmpFile)
	if e.cmd != nil {
		url := fmt.Sprintf("http://127.0.0.1:%v/quitquitquit", e.adminPort)
		_, _, _ = env.HTTPPost(url, "", "")
		select {
		case <-time.After(3 * time.Second):
			log.Println("envoy killed as timeout reached")
			log.Println(e.cmd.Process.Kill())
		case <-e.done:
			log.Printf("stop envoy ... done\n")
		}
	}
}

func getAdminPort(bootstrap string) (uint32, error) {
	pb := &bootstrap_v3.Bootstrap{}
	if err := ReadYAML(bootstrap, pb); err != nil {
		return 0, err
	}
	if pb.Admin == nil || pb.Admin.Address == nil {
		return 0, fmt.Errorf("missing admin section in bootstrap: %v", bootstrap)
	}
	socket, ok := pb.Admin.Address.Address.(*core.Address_SocketAddress)
	if !ok {
		return 0, fmt.Errorf("missing socket in bootstrap: %v", bootstrap)
	}
	port, ok := socket.SocketAddress.PortSpecifier.(*core.SocketAddress_PortValue)
	if !ok {
		return 0, fmt.Errorf("missing port in bootstrap: %v", bootstrap)
	}
	return port.PortValue, nil
}

// downloads env based on the given branch name. Return location of downloaded envoy.
func downloadEnvoy(ver string) (string, error) {
	var proxyDepURL string
	if regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`).Match([]byte(ver)) {
		// this is a patch version string
		proxyDepURL = fmt.Sprintf("https://raw.githubusercontent.com/istio/istio/%v/istio.deps", ver)
	} else if regexp.MustCompile(`[0-9]+\.[0-9]+`).Match([]byte(ver)) {
		// this is a minor version string
		proxyDepURL = fmt.Sprintf("https://raw.githubusercontent.com/istio/istio/release-%v/istio.deps", ver)
	} else if ver == "master" {
		proxyDepURL = "https://raw.githubusercontent.com/istio/istio/master/istio.deps"
	} else {
		return "", fmt.Errorf("envoy version %v is neither minor version nor patch version", ver)
	}
	resp, err := http.Get(proxyDepURL)
	if err != nil {
		return "", fmt.Errorf("cannot get envoy sha from %v: %v", proxyDepURL, err)
	}
	defer resp.Body.Close()
	istioDeps, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("cannot read body of istio deps: %v", err)
	}

	var deps []interface{}
	if err := json.Unmarshal(istioDeps, &deps); err != nil {
		return "", err
	}
	proxySHA := ""
	for _, d := range deps {
		if dm, ok := d.(map[string]interface{}); ok && dm["repoName"].(string) == "proxy" {
			proxySHA = dm["lastStableSHA"].(string)
		}
	}
	if proxySHA == "" {
		return "", errors.New("cannot identify proxy SHA to download")
	}

	dir := fmt.Sprintf("%s/%s", os.TempDir(), "istio-proxy")
	dst := fmt.Sprintf("%v/envoy-%v", dir, proxySHA)
	if _, err := os.Stat(dst); err == nil {
		// If the desired envoy binary is already downloaded, skip downloading and return.
		return dst, nil
	}

	// clean up the tmp dir before downloading to remove stale envoy binary.
	_ = os.RemoveAll(dir)

	// make temp directory to put downloaded envoy binary.
	if err := os.MkdirAll(dir, 0755); err != nil {
		return "", err
	}

	envoyURL := fmt.Sprintf("https://storage.googleapis.com/istio-build/proxy/envoy-alpha-%v.tar.gz", proxySHA)
	downloadCmd := exec.Command("bash", "-c", fmt.Sprintf("curl -fLSs %v | tar xz", envoyURL))
	downloadCmd.Stderr = os.Stderr
	downloadCmd.Stdout = os.Stdout
	err = downloadCmd.Run()
	if err != nil {
		return "", fmt.Errorf("fail to run envoy download command: %v", err)
	}
	src := "usr/local/bin/envoy"
	if _, err := os.Stat(src); err != nil {
		return "", fmt.Errorf("fail to find downloaded envoy: %v", err)
	}
	defer os.RemoveAll("usr/")

	cpCmd := exec.Command("cp", "-n", src, dst)
	cpCmd.Stderr = os.Stderr
	cpCmd.Stdout = os.Stdout
	if err := cpCmd.Run(); err != nil {
		return "", fmt.Errorf("fail to copy envoy binary from %v to %v: %v", src, dst, err)
	}

	return dst, nil
}
<|endoftext|>


<commit_before>package api

import (
	"expvar"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"time"

	"github.com/zserge/metric"

	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/daemon/config"
	"nimona.io/pkg/discovery"
	"nimona.io/pkg/exchange"
	"nimona.io/pkg/http/router"
	"nimona.io/pkg/log"
	"nimona.io/pkg/net"
	"nimona.io/pkg/object"
	"nimona.io/pkg/orchestrator"
	"nimona.io/pkg/peer"
	"nimona.io/pkg/sqlobjectstore"
)

// API for HTTP
type API struct {
	config *config.Config

	router    *router.Router
	net       net.Network
	discovery discovery.Discoverer
	exchange  exchange.Exchange

	objectStore  *sqlobjectstore.Store
	orchestrator orchestrator.Orchestrator
	local        *peer.LocalPeer

	token string

	version      string
	commit       string
	buildDate    string
	gracefulStop chan bool
	srv          *http.Server
}

// New HTTP API
func New(
	cfg *config.Config,
	k crypto.PrivateKey,
	n net.Network,
	d discovery.Discoverer,
	x exchange.Exchange,
	linf *peer.LocalPeer,
	sst *sqlobjectstore.Store,
	or orchestrator.Orchestrator,
	version string,
	commit string,
	buildDate string,
	token string,
) *API {
	r := router.New()

	api := &API{
		config: cfg,

		router:      r,
		net:         n,
		discovery:   d,
		exchange:    x,
		objectStore: sst,

		orchestrator: or,

		local: linf,

		version:      version,
		commit:       commit,
		buildDate:    buildDate,
		token:        token,
		gracefulStop: make(chan bool),
	}

	r.Use(api.Cors())
	r.Use(api.TokenAuth())

	r.Handle(
		"GET",
		"/api/v1/version$",
		api.HandleVersion,
	)
	r.Handle(
		"GET",
		"/api/v1/local$",
		api.HandleGetLocal,
	)
	r.Handle(
		"GET",
		"/api/v1/peers",
		api.HandleGetLookup,
	)

	r.Handle(
		"GET",
		"/api/v1/identities$",
		api.HandleGetIdentities,
	)
	r.Handle(
		"GET",
		"/api/v1/identities/(?P<fingerprint>.+)$",
		api.HandleGetIdentity,
	)
	r.Handle(
		"POST",
		"/api/v1/identities$",
		api.HandlePostIdentities,
	)

	r.Handle(
		"GET",
		"/api/v1/peers$",
		api.HandleGetPeers,
	)
	r.Handle(
		"GET",
		"/api/v1/peers/(?P<fingerprint>.+)$",
		api.HandleGetPeer,
	)

	r.Handle(
		"GET",
		"/api/v1/objects$",
		api.HandleGetObjects,
	)
	r.Handle(
		"GET",
		"/api/v1/objects/(?P<objectHash>.+)$",
		api.HandleGetObject,
	)
	r.Handle(
		"POST",
		"/api/v1/objects$",
		api.HandlePostObjects,
	)
	r.Handle(
		"POST",
		"/api/v1/objects/(?P<rootObjectHash>.+)$",
		api.HandlePostObject,
	)

	r.Handle(
		"GET",
		"/api/v1/streams/(?P<ns>.+)/(?P<pattern>.*)$",
		api.HandleGetStreams,
	)

	r.Handle(
		"POST",
		"/api/v1/stop$",
		api.Stop,
	)

	if y, _ := strconv.ParseBool(os.Getenv("NIMONA_EXPVAR")); y {
		expvar.Publish(
			"go:goroutine",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:cgocall",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:alloc",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:alloc.total",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		go func() {
			for range time.Tick(100 * time.Millisecond) {
				m := &runtime.MemStats{}
				runtime.ReadMemStats(m)
				expvar.Get("go:goroutine").(metric.Metric).Add(
					float64(runtime.NumGoroutine()),
				)
				expvar.Get("go:cgocall").(metric.Metric).Add(
					float64(runtime.NumCgoCall()),
				)
				expvar.Get("go:alloc").(metric.Metric).Add(
					float64(m.Alloc) / 1000000,
				)
				expvar.Get("go:alloc.total").(metric.Metric).Add(
					float64(m.TotalAlloc) / 1000000,
				)
			}
		}()
		r.Handle(
			"GET",
			"/debug/metrics",
			func(c *router.Context) {
				metric.Handler(metric.Exposed).ServeHTTP(c.Writer, c.Request)
			},
		)
		r.Handle(
			"GET",
			"/debug/expvar",
			func(c *router.Context) {
				expvar.Handler().ServeHTTP(c.Writer, c.Request)
			},
		)
	}

	return api
}

// Serve HTTP API
func (api *API) Serve(address string) error {
	ctx := context.Background()
	logger := log.FromContext(ctx).Named("api")

	api.srv = &http.Server{
		Addr:    address,
		Handler: api.router,
	}

	go func() {
		if err := api.srv.ListenAndServe(); err != nil &&
			err != http.ErrServerClosed {
			logger.Error("Error serving", log.Error(err))
		}
	}()

	<-api.gracefulStop

	if err := api.srv.Shutdown(ctx); err != nil {
		logger.Error("Failed to shutdown", log.Error(err))
	}

	return nil
}

func (api *API) Stop(c *router.Context) {
	c.Status(http.StatusOK)
	go func() {
		api.gracefulStop <- true
	}()
}

func (api *API) mapObject(o object.Object) map[string]interface{} {
	m := o.ToMap()
	m["_hash:s"] = object.NewHash(o).String()
	return m
}

func (api *API) mapObjects(objects []object.Object) []map[string]interface{} {
	ms := []map[string]interface{}{}
	for _, o := range objects {
		ms = append(ms, api.mapObject(o))
	}
	return ms
}
<commit_msg>feat(api): add goroutine stack endpoint<commit_after>package api

import (
	"expvar"
	"net/http"
	"os"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"time"

	"github.com/zserge/metric"

	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/daemon/config"
	"nimona.io/pkg/discovery"
	"nimona.io/pkg/exchange"
	"nimona.io/pkg/http/router"
	"nimona.io/pkg/log"
	"nimona.io/pkg/net"
	"nimona.io/pkg/object"
	"nimona.io/pkg/orchestrator"
	"nimona.io/pkg/peer"
	"nimona.io/pkg/sqlobjectstore"
)

// API for HTTP
type API struct {
	config *config.Config

	router    *router.Router
	net       net.Network
	discovery discovery.Discoverer
	exchange  exchange.Exchange

	objectStore  *sqlobjectstore.Store
	orchestrator orchestrator.Orchestrator
	local        *peer.LocalPeer

	token string

	version      string
	commit       string
	buildDate    string
	gracefulStop chan bool
	srv          *http.Server
}

// New HTTP API
func New(
	cfg *config.Config,
	k crypto.PrivateKey,
	n net.Network,
	d discovery.Discoverer,
	x exchange.Exchange,
	linf *peer.LocalPeer,
	sst *sqlobjectstore.Store,
	or orchestrator.Orchestrator,
	version string,
	commit string,
	buildDate string,
	token string,
) *API {
	r := router.New()

	api := &API{
		config: cfg,

		router:      r,
		net:         n,
		discovery:   d,
		exchange:    x,
		objectStore: sst,

		orchestrator: or,

		local: linf,

		version:      version,
		commit:       commit,
		buildDate:    buildDate,
		token:        token,
		gracefulStop: make(chan bool),
	}

	r.Use(api.Cors())
	r.Use(api.TokenAuth())

	r.Handle(
		"GET",
		"/api/v1/version$",
		api.HandleVersion,
	)
	r.Handle(
		"GET",
		"/api/v1/local$",
		api.HandleGetLocal,
	)
	r.Handle(
		"GET",
		"/api/v1/peers",
		api.HandleGetLookup,
	)

	r.Handle(
		"GET",
		"/api/v1/identities$",
		api.HandleGetIdentities,
	)
	r.Handle(
		"GET",
		"/api/v1/identities/(?P<fingerprint>.+)$",
		api.HandleGetIdentity,
	)
	r.Handle(
		"POST",
		"/api/v1/identities$",
		api.HandlePostIdentities,
	)

	r.Handle(
		"GET",
		"/api/v1/peers$",
		api.HandleGetPeers,
	)
	r.Handle(
		"GET",
		"/api/v1/peers/(?P<fingerprint>.+)$",
		api.HandleGetPeer,
	)

	r.Handle(
		"GET",
		"/api/v1/objects$",
		api.HandleGetObjects,
	)
	r.Handle(
		"GET",
		"/api/v1/objects/(?P<objectHash>.+)$",
		api.HandleGetObject,
	)
	r.Handle(
		"POST",
		"/api/v1/objects$",
		api.HandlePostObjects,
	)
	r.Handle(
		"POST",
		"/api/v1/objects/(?P<rootObjectHash>.+)$",
		api.HandlePostObject,
	)

	r.Handle(
		"GET",
		"/api/v1/streams/(?P<ns>.+)/(?P<pattern>.*)$",
		api.HandleGetStreams,
	)

	r.Handle(
		"POST",
		"/api/v1/stop$",
		api.Stop,
	)

	if y, _ := strconv.ParseBool(os.Getenv("NIMONA_EXPVAR")); y {
		expvar.Publish(
			"go:goroutine",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:cgocall",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:alloc",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		expvar.Publish(
			"go:alloc.total",
			metric.NewGauge("2m1s", "15m30s", "1h1m"),
		)
		go func() {
			for range time.Tick(100 * time.Millisecond) {
				m := &runtime.MemStats{}
				runtime.ReadMemStats(m)
				expvar.Get("go:goroutine").(metric.Metric).Add(
					float64(runtime.NumGoroutine()),
				)
				expvar.Get("go:cgocall").(metric.Metric).Add(
					float64(runtime.NumCgoCall()),
				)
				expvar.Get("go:alloc").(metric.Metric).Add(
					float64(m.Alloc) / 1000000,
				)
				expvar.Get("go:alloc.total").(metric.Metric).Add(
					float64(m.TotalAlloc) / 1000000,
				)
			}
		}()
		r.Handle(
			"GET",
			"/debug/metrics",
			func(c *router.Context) {
				metric.Handler(metric.Exposed).ServeHTTP(c.Writer, c.Request)
			},
		)
		goroutineStackHandler := func(c *router.Context) {
			stack := debug.Stack()
			c.Writer.Write(stack)                          // nolint: errcheck
			pprof.Lookup("goroutine").WriteTo(c.Writer, 2) // nolint: errcheck
		}
		r.Handle(
			"GET",
			"/debug/stack/goroutine",
			goroutineStackHandler,
		)
		r.Handle(
			"GET",
			"/debug/expvar",
			func(c *router.Context) {
				expvar.Handler().ServeHTTP(c.Writer, c.Request)
			},
		)
	}

	return api
}

// Serve HTTP API
func (api *API) Serve(address string) error {
	ctx := context.Background()
	logger := log.FromContext(ctx).Named("api")

	api.srv = &http.Server{
		Addr:    address,
		Handler: api.router,
	}

	go func() {
		if err := api.srv.ListenAndServe(); err != nil &&
			err != http.ErrServerClosed {
			logger.Error("Error serving", log.Error(err))
		}
	}()

	<-api.gracefulStop

	if err := api.srv.Shutdown(ctx); err != nil {
		logger.Error("Failed to shutdown", log.Error(err))
	}

	return nil
}

func (api *API) Stop(c *router.Context) {
	c.Status(http.StatusOK)
	go func() {
		api.gracefulStop <- true
	}()
}

func (api *API) mapObject(o object.Object) map[string]interface{} {
	m := o.ToMap()
	m["_hash:s"] = object.NewHash(o).String()
	return m
}

func (api *API) mapObjects(objects []object.Object) []map[string]interface{} {
	ms := []map[string]interface{}{}
	for _, o := range objects {
		ms = append(ms, api.mapObject(o))
	}
	return ms
}
<|endoftext|>