<commit_before>package guber

// Common
//==============================================================================
type ResourceDefinition struct {
	Kind       string `json:"kind"`
	ApiVersion string `json:"apiVersion"`
}

type Metadata struct {
	Name              string            `json:"name"`
	Namespace         string            `json:"namespace"`
	Labels            map[string]string `json:"labels,omitempty"`
	CreationTimestamp string            `json:"creationTimestamp,omitempty"`
}

// Namespace
//==============================================================================
type Namespace struct {
	*ResourceDefinition
	Metadata *Metadata `json:"metadata"`
}

type NamespaceList struct {
	Items []*Namespace `json:"items"`
}

// Node
//==============================================================================
type NodeSpec struct {
	ExternalID string `json:"externalID"`
}

type NodeStatusCapacity struct {
	CPU    string `json:"cpu"`
	Memory string `json:"memory"`
}

type NodeStatusCondition struct {
	Type   string `json:"type"`
	Status string `json:"status"`
}

type NodeStatus struct {
	Capacity   *NodeStatusCapacity    `json:"capacity"`
	Conditions []*NodeStatusCondition `json:"conditions"`
}

type Node struct {
	*ResourceDefinition
	Metadata *Metadata   `json:"metadata"`
	Spec     *NodeSpec   `json:"spec"`
	Status   *NodeStatus `json:"status"`
}

// ReplicationController
//==============================================================================
type PodTemplate struct {
	Metadata *Metadata `json:"metadata"`
	Spec     *PodSpec  `json:"spec"`
}

type ReplicationControllerSpec struct {
	Selector map[string]string `json:"selector"`
	Replicas int               `json:"replicas"`
	Template *PodTemplate      `json:"template"`
}

type ReplicationControllerStatus struct {
	Replicas int `json:"replicas"`
}

type ReplicationController struct {
	*ResourceDefinition
	Metadata *Metadata                    `json:"metadata"`
	Spec     *ReplicationControllerSpec   `json:"spec"`
	Status   *ReplicationControllerStatus `json:"status,omitempty"`
}

type ReplicationControllerList struct {
	Items []*ReplicationController `json:"items"`
}

// Pod
//==============================================================================
type AwsElasticBlockStore struct {
	VolumeID string `json:"volumeID"`
	FSType   string `json:"fsType"`
}

type Volume struct {
	Name                 string                `json:"name"`
	AwsElasticBlockStore *AwsElasticBlockStore `json:"awsElasticBlockStore"`
}

type VolumeMount struct {
	Name      string `json:"name"`
	MountPath string `json:"mountPath"`
}

type ResourceValues struct {
	Memory string `json:"memory"`
	CPU    string `json:"cpu"`
}

type Resources struct {
	Limits   *ResourceValues `json:"limits"`
	Requests *ResourceValues `json:"requests"`
}

type ContainerPort struct {
	Name          string `json:"name,omitempty"`
	ContainerPort int    `json:"containerPort"`
	Protocol      string `json:"protocol,omitempty"`
}

type EnvVar struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

type SecurityContext struct {
	Privileged bool `json:"privileged"`
}

type Container struct {
	Name            string           `json:"name"`
	Image           string           `json:"image"`
	Command         []string         `json:"command"`
	Resources       *Resources       `json:"resources"`
	Ports           []*ContainerPort `json:"ports"`
	VolumeMounts    []*VolumeMount   `json:"volumeMounts"`
	Env             []*EnvVar        `json:"env"`
	SecurityContext *SecurityContext `json:"securityContext"`
}

type ImagePullSecret struct {
	Name string `json:"name"`
}

type PodSpec struct {
	Volumes                       []*Volume          `json:"volumes"`
	Containers                    []*Container       `json:"containers"`
	ImagePullSecrets              []*ImagePullSecret `json:"imagePullSecrets"`
	TerminationGracePeriodSeconds int                `json:"terminationGracePeriodSeconds"`
	ImagePullPolicy               string             `json:"imagePullPolicy"`
}

type PodStatus struct {
	Phase string `json:"phase"`
}

type Pod struct {
	*ResourceDefinition
	Metadata *Metadata  `json:"metadata"`
	Spec     *PodSpec   `json:"spec"`
	Status   *PodStatus `json:"status"`
}

type PodList struct {
	Items []*Pod
}

// Service
//==============================================================================
type ServicePort struct {
	Name       string `json:"name"`
	Port       int    `json:"port"`
	Protocol   string `json:"protocol,omitempty"`
	NodePort   int    `json:"nodePort,omitempty"`
	TargetPort int    `json:"targetPort,omitempty"`
}

type ServiceSpec struct {
	Type     string            `json:"type,omitempty"`
	Selector map[string]string `json:"selector"`
	Ports    []*ServicePort    `json:"ports"`

	ClusterIP string `json:"clusterIP,omitempty"`
}

type Service struct {
	*ResourceDefinition
	Metadata *Metadata    `json:"metadata"`
	Spec     *ServiceSpec `json:"spec"`
	// Status   *ServiceStatus `json:"status"`
}

type ServiceList struct {
	Items []*Service `json:"items"`
}

// Secret
//==============================================================================
type Secret struct {
	*ResourceDefinition
	Metadata *Metadata         `json:"metadata"`
	Type     string            `json:"type"`
	Data     map[string]string `json:"data"`
}

type SecretList struct {
	Items []*Secret `json:"items"`
}

// Event
//==============================================================================
type Source struct {
	Host string `json:"host"`
}

type Event struct {
	*ResourceDefinition
	Message string  `json:"message"`
	Count   int     `json:"count"`
	Source  *Source `json:"source"`
}

type EventList struct {
	Items []*Event `json:"items"`
}

// TODO not sure if this should be in the types file.. related to queries, but is a Kube-specific thing
type QueryParams struct {
	LabelSelector string
	FieldSelector string
}
<commit_msg>Omit empty resource values<commit_after>package guber

// Common
//==============================================================================
type ResourceDefinition struct {
	Kind       string `json:"kind"`
	ApiVersion string `json:"apiVersion"`
}

type Metadata struct {
	Name              string            `json:"name"`
	Namespace         string            `json:"namespace"`
	Labels            map[string]string `json:"labels,omitempty"`
	CreationTimestamp string            `json:"creationTimestamp,omitempty"`
}

// Namespace
//==============================================================================
type Namespace struct {
	*ResourceDefinition
	Metadata *Metadata `json:"metadata"`
}

type NamespaceList struct {
	Items []*Namespace `json:"items"`
}

// Node
//==============================================================================
type NodeSpec struct {
	ExternalID string `json:"externalID"`
}

type NodeStatusCapacity struct {
	CPU    string `json:"cpu"`
	Memory string `json:"memory"`
}

type NodeStatusCondition struct {
	Type   string `json:"type"`
	Status string `json:"status"`
}

type NodeStatus struct {
	Capacity   *NodeStatusCapacity    `json:"capacity"`
	Conditions []*NodeStatusCondition `json:"conditions"`
}

type Node struct {
	*ResourceDefinition
	Metadata *Metadata   `json:"metadata"`
	Spec     *NodeSpec   `json:"spec"`
	Status   *NodeStatus `json:"status"`
}

// ReplicationController
//==============================================================================
type PodTemplate struct {
	Metadata *Metadata `json:"metadata"`
	Spec     *PodSpec  `json:"spec"`
}

type ReplicationControllerSpec struct {
	Selector map[string]string `json:"selector"`
	Replicas int               `json:"replicas"`
	Template *PodTemplate      `json:"template"`
}

type ReplicationControllerStatus struct {
	Replicas int `json:"replicas"`
}

type ReplicationController struct {
	*ResourceDefinition
	Metadata *Metadata                    `json:"metadata"`
	Spec     *ReplicationControllerSpec   `json:"spec"`
	Status   *ReplicationControllerStatus `json:"status,omitempty"`
}

type ReplicationControllerList struct {
	Items []*ReplicationController `json:"items"`
}

// Pod
//==============================================================================
type AwsElasticBlockStore struct {
	VolumeID string `json:"volumeID"`
	FSType   string `json:"fsType"`
}

type Volume struct {
	Name                 string                `json:"name"`
	AwsElasticBlockStore *AwsElasticBlockStore `json:"awsElasticBlockStore"`
}

type VolumeMount struct {
	Name      string `json:"name"`
	MountPath string `json:"mountPath"`
}

type ResourceValues struct {
	Memory string `json:"memory,omitempty"`
	CPU    string `json:"cpu,omitempty"`
}

type Resources struct {
	Limits   *ResourceValues `json:"limits"`
	Requests *ResourceValues `json:"requests"`
}

type ContainerPort struct {
	Name          string `json:"name,omitempty"`
	ContainerPort int    `json:"containerPort"`
	Protocol      string `json:"protocol,omitempty"`
}

type EnvVar struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

type SecurityContext struct {
	Privileged bool `json:"privileged"`
}

type Container struct {
	Name            string           `json:"name"`
	Image           string           `json:"image"`
	Command         []string         `json:"command"`
	Resources       *Resources       `json:"resources"`
	Ports           []*ContainerPort `json:"ports"`
	VolumeMounts    []*VolumeMount   `json:"volumeMounts"`
	Env             []*EnvVar        `json:"env"`
	SecurityContext *SecurityContext `json:"securityContext"`
}

type ImagePullSecret struct {
	Name string `json:"name"`
}

type PodSpec struct {
	Volumes                       []*Volume          `json:"volumes"`
	Containers                    []*Container       `json:"containers"`
	ImagePullSecrets              []*ImagePullSecret `json:"imagePullSecrets"`
	TerminationGracePeriodSeconds int                `json:"terminationGracePeriodSeconds"`
	ImagePullPolicy               string             `json:"imagePullPolicy"`
}

type PodStatus struct {
	Phase string `json:"phase"`
}

type Pod struct {
	*ResourceDefinition
	Metadata *Metadata  `json:"metadata"`
	Spec     *PodSpec   `json:"spec"`
	Status   *PodStatus `json:"status"`
}

type PodList struct {
	Items []*Pod
}

// Service
//==============================================================================
type ServicePort struct {
	Name       string `json:"name"`
	Port       int    `json:"port"`
	Protocol   string `json:"protocol,omitempty"`
	NodePort   int    `json:"nodePort,omitempty"`
	TargetPort int    `json:"targetPort,omitempty"`
}

type ServiceSpec struct {
	Type     string            `json:"type,omitempty"`
	Selector map[string]string `json:"selector"`
	Ports    []*ServicePort    `json:"ports"`

	ClusterIP string `json:"clusterIP,omitempty"`
}

type Service struct {
	*ResourceDefinition
	Metadata *Metadata    `json:"metadata"`
	Spec     *ServiceSpec `json:"spec"`
	// Status   *ServiceStatus `json:"status"`
}

type ServiceList struct {
	Items []*Service `json:"items"`
}

// Secret
//==============================================================================
type Secret struct {
	*ResourceDefinition
	Metadata *Metadata         `json:"metadata"`
	Type     string            `json:"type"`
	Data     map[string]string `json:"data"`
}

type SecretList struct {
	Items []*Secret `json:"items"`
}

// Event
//==============================================================================
type Source struct {
	Host string `json:"host"`
}

type Event struct {
	*ResourceDefinition
	Message string  `json:"message"`
	Count   int     `json:"count"`
	Source  *Source `json:"source"`
}

type EventList struct {
	Items []*Event `json:"items"`
}

// TODO not sure if this should be in the types file.. related to queries, but is a Kube-specific thing
type QueryParams struct {
	LabelSelector string
	FieldSelector string
}
<|endoftext|>


<commit_before>// Copyright 2015 Jean Niklas L'orange.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package edn

import (
	"bufio"
	"bytes"
	"fmt"
	"strconv"
	"strings"
)

type Keyword string

func (k Keyword) String() string {
	return fmt.Sprintf(":%s", string(k))
}

func (k Keyword) MarshalEDN() ([]byte, error) {
	return []byte(k.String()), nil
}

type Symbol string

func (s Symbol) String() string {
	return string(s)
}

func (s Symbol) MarshalEDN() ([]byte, error) {
	return []byte(s), nil
}

type Vector []interface{}

func (v Vector) String() string {
	vals := []string{}
	for _, val := range v {
		vals = append(vals, fmt.Sprint(val))
	}
	return "[" + strings.Join(vals, " ") + "]"
}

type List []interface{}

func (l List) String() string {
	vals := []string{}
	for _, val := range l {
		vals = append(vals, fmt.Sprint(val))
	}
	return "(" + strings.Join(vals, " ") + ")"
}

type Tag struct {
	Tagname string
	Value   interface{}
}

func (t Tag) String() string {
	return fmt.Sprintf("#%s %s", t.Tagname, t.Value)
}

func (t Tag) MarshalEDN() ([]byte, error) {
	str := []byte(fmt.Sprintf(`#%s `, t.Tagname))
	b, err := Marshal(t.Value)
	if err != nil {
		return nil, err
	}
	return append(str, b...), nil
}

func (t *Tag) UnmarshalEDN(bs []byte) error {
	// read actual tag, using the lexer.
	var lex lexer
	lex.reset()
	buf := bufio.NewReader(bytes.NewBuffer(bs))
	start := 0
	endTag := 0
tag:
	for {
		r, rlen, err := buf.ReadRune()
		if err != nil {
			return err
		}

		ls := lex.state(r)
		switch ls {
		case lexIgnore:
			start += rlen
			endTag += rlen
		case lexError:
			return lex.err
		case lexEndPrev:
			break tag
		case lexEnd: // unexpected, assuming tag which is not ending with lexEnd
			return errUnexpected
		case lexCont:
			endTag += rlen
		}
	}
	t.Tagname = string(bs[start+1 : endTag])
	return Unmarshal(bs[endTag:], &t.Value)
}

type Rune rune

func (r Rune) MarshalEDN() ([]byte, error) {
	buf := bytes.NewBuffer(make([]byte, 0, 10))
	encodeRune(buf, rune(r))
	return buf.Bytes(), nil
}

func encodeRune(buf *bytes.Buffer, r rune) {
	if !isWhitespace(r) {
		buf.WriteByte('\\')
		buf.WriteRune(r)
	} else {
		switch r {
		case '\b':
			buf.WriteString(`\backspace`)
		case '\f':
			buf.WriteString(`\formfeed`)
		case '\n':
			buf.WriteString(`\newline`)
		case '\r':
			buf.WriteString(`\return`)
		case '\t':
			buf.WriteString(`\tab`)
		case ' ':
			buf.WriteString(`\space`)
		default:
			val := strconv.QuoteRuneToASCII(r)
			buf.WriteString(val[1 : len(val)-1])
		}
	}
}

type Set map[interface{}]bool

func (s Set) String() string {
	keys := []string{}
	for k, _ := range s {
		keys = append(keys, fmt.Sprint(k))
	}
	return "#{" + strings.Join(keys, " ") + "}"
}

type Map map[interface{}]interface{}

func (m Map) String() string {
	kvs := []string{}
	for k, v := range m {
		kvs = append(kvs, fmt.Sprintf("%s %s", k, v))
	}
	return "{" + strings.Join(kvs, ", ") + "}"
}
<commit_msg>Fix bug where char comma was wrongly printed<commit_after>// Copyright 2015 Jean Niklas L'orange.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package edn

import (
	"bufio"
	"bytes"
	"fmt"
	"strings"
)

type Keyword string

func (k Keyword) String() string {
	return fmt.Sprintf(":%s", string(k))
}

func (k Keyword) MarshalEDN() ([]byte, error) {
	return []byte(k.String()), nil
}

type Symbol string

func (s Symbol) String() string {
	return string(s)
}

func (s Symbol) MarshalEDN() ([]byte, error) {
	return []byte(s), nil
}

type Vector []interface{}

func (v Vector) String() string {
	vals := []string{}
	for _, val := range v {
		vals = append(vals, fmt.Sprint(val))
	}
	return "[" + strings.Join(vals, " ") + "]"
}

type List []interface{}

func (l List) String() string {
	vals := []string{}
	for _, val := range l {
		vals = append(vals, fmt.Sprint(val))
	}
	return "(" + strings.Join(vals, " ") + ")"
}

type Tag struct {
	Tagname string
	Value   interface{}
}

func (t Tag) String() string {
	return fmt.Sprintf("#%s %s", t.Tagname, t.Value)
}

func (t Tag) MarshalEDN() ([]byte, error) {
	str := []byte(fmt.Sprintf(`#%s `, t.Tagname))
	b, err := Marshal(t.Value)
	if err != nil {
		return nil, err
	}
	return append(str, b...), nil
}

func (t *Tag) UnmarshalEDN(bs []byte) error {
	// read actual tag, using the lexer.
	var lex lexer
	lex.reset()
	buf := bufio.NewReader(bytes.NewBuffer(bs))
	start := 0
	endTag := 0
tag:
	for {
		r, rlen, err := buf.ReadRune()
		if err != nil {
			return err
		}

		ls := lex.state(r)
		switch ls {
		case lexIgnore:
			start += rlen
			endTag += rlen
		case lexError:
			return lex.err
		case lexEndPrev:
			break tag
		case lexEnd: // unexpected, assuming tag which is not ending with lexEnd
			return errUnexpected
		case lexCont:
			endTag += rlen
		}
	}
	t.Tagname = string(bs[start+1 : endTag])
	return Unmarshal(bs[endTag:], &t.Value)
}

type Rune rune

func (r Rune) MarshalEDN() ([]byte, error) {
	buf := bytes.NewBuffer(make([]byte, 0, 10))
	encodeRune(buf, rune(r))
	return buf.Bytes(), nil
}

func encodeRune(buf *bytes.Buffer, r rune) {
	const hex = "0123456789abcdef"
	if !isWhitespace(r) {
		buf.WriteByte('\\')
		buf.WriteRune(r)
	} else {
		switch r {
		case '\b':
			buf.WriteString(`\backspace`)
		case '\f':
			buf.WriteString(`\formfeed`)
		case '\n':
			buf.WriteString(`\newline`)
		case '\r':
			buf.WriteString(`\return`)
		case '\t':
			buf.WriteString(`\tab`)
		case ' ':
			buf.WriteString(`\space`)
		default:
			buf.WriteByte('\\')
			buf.WriteByte('u')
			buf.WriteByte(hex[r>>12&0xF])
			buf.WriteByte(hex[r>>8&0xF])
			buf.WriteByte(hex[r>>4&0xF])
			buf.WriteByte(hex[r&0xF])
		}
	}
}

type Set map[interface{}]bool

func (s Set) String() string {
	keys := []string{}
	for k, _ := range s {
		keys = append(keys, fmt.Sprint(k))
	}
	return "#{" + strings.Join(keys, " ") + "}"
}

type Map map[interface{}]interface{}

func (m Map) String() string {
	kvs := []string{}
	for k, v := range m {
		kvs = append(kvs, fmt.Sprintf("%s %s", k, v))
	}
	return "{" + strings.Join(kvs, ", ") + "}"
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"text/template"

	"path/filepath"
)

func CheckCommand(cmd string, fname string) (m *Config) {
	if isV {
		fmt.Println("search:", fname)
	}
	if _, err := os.Stat(fname); err != nil {
		return nil
	}
	if isV {
		fmt.Println("  Found the config file.:", fname)
	}
	m = LoadConfig(fname)
	if m == nil {
		return
	}
	cval, ok := m.Commands[cmd]
	if !ok {
		if isV {
			fmt.Println("  Command not found.:", fname)
		}
		return nil
	}
	if isV {
		fmt.Println("found.:", cval)
	}
	return
}

func CheckImage(cmd string, fname string) (m *Config) {
	if isV {
		fmt.Println("search:", fname)
	}
	if _, err := os.Stat(fname); err != nil {
		return nil
	}
	if isV {
		fmt.Println("  Found the config file.:", fname)
	}
	m = LoadConfig(fname)
	if m == nil {
		return
	}

	cval, ok := m.Images[cmd]
	if !ok {
		if isV {
			fmt.Println("  Image not found.:", fname)
		}
		return nil
	}
	if isV {
		fmt.Println("found.:", cval)
	}
	return
}

var CheckCmdsMap = make(map[string][]string)

func InitCheckCmds() {
	CheckCmdsMap = make(map[string][]string)
}

func CheckCmds(cmd string, fname string) (m *Config) {
	if isV {
		fmt.Println("search:", fname)
	}
	if _, err := os.Stat(fname); err != nil {
		return nil
	}
	if isV {
		fmt.Println("  Found the config file.:", fname)
	}
	m = LoadConfig(fname)
	if m == nil {
		return nil
	}

	for key, val := range m.Commands {
		_, ok := CheckCmdsMap[key]
		if !ok {
			CheckCmdsMap[key] = []string{val, fname}
		}
	}

	return nil
}

func SearchConfig(cmd string, f func(string, string) *Config) (*Config, string) {
	fname := `.dcenv_` + envShell

	p := os.Getenv("DCENV_DIR")
	if p != "" {
		t := strings.Split(p, string(os.PathSeparator))
		for i := len(t); i > 0; i-- {
			s := strings.Join(t[0:i], string(os.PathSeparator)) + string(os.PathSeparator) + fname
			tc := f(cmd, s)
			if tc != nil {
				return tc, s
			}
		}
	}

	p, err := os.Getwd()
	if err != nil {
		fmt.Println(err)
		exit(1)
		return nil, p
	}
	t := strings.Split(p, string(os.PathSeparator))
	for i := len(t); i > 0; i-- {
		s := strings.Join(t[0:i], string(os.PathSeparator)) + string(os.PathSeparator) + fname
		tc := f(cmd, s)
		if tc != nil {
			return tc, s
		}
	}

	s := filepath.Join(envHome, "files", fname)
	tc := f(cmd, s)
	if tc != nil {
		return tc, s
	}

	return nil, s
}

type scriptArg struct {
	Tag    string
	Image  string
	Cmd    string
	CfgDir string
	Envs   map[string]string
}

func MakeArgsFile(sft int) {
	s := ""
	for _, t := range os.Args[sft:] {
		if strings.Contains(t, " ") {
			s += ` "` + t + `"`
		} else {
			s += " " + t
		}
	}
	//  if len(os.Args[sft:]) > 0 {
	//    s = `"`+strings.Join(os.Args[sft:],`" "`)+`"`
	//  }
	fname := filepath.Join(envHome, "files", "__args__")
	err := ioutil.WriteFile(fname, []byte(s), 0666)
	if err != nil {
		fmt.Println(err)
		exit(1)
		return
	}
}

func templateFile(fname string, addBuf string, sarg interface{}, oname string) error {
	buf, e := ioutil.ReadFile(fname)
	if e != nil {
		return e
	}
	fp, e := os.OpenFile(oname, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
	if e != nil {
		return e
	}
	if e := template.Must(template.New("script").Parse(string(buf)+addBuf)).Execute(fp, sarg); e != nil {
		return e
	}
	if e := fp.Close(); e != nil {
		return e
	}
	if isV {
		fmt.Println("write file:", oname)
	}
	return nil
}

func MakeExecFile(m *Config, cmd string, fname string) {
	//make script
	cval := m.Commands[cmd]
	cnt := m.Images[cval]

	sarg := scriptArg{
		Tag:    cnt.Tag,
		Image:  cval,
		Cmd:    cmd,
		CfgDir: filepath.Dir(fname),
		Envs:   cnt.Commands[cmd],
	}

	err := templateFile(
		filepath.Join(envHome, "files", `header_`+envShell),
		cnt.Script,
		sarg,
		filepath.Join(envHome, "tmp", envCommand))
	if err != nil {
		fmt.Println(err)
		exit(1)
		return
	}
	return
}

func MakeExecFileSystem(cmd string) {
	s := filepath.Join(envHome, "shims")
	plist := filepath.SplitList(os.Getenv("PATH"))
	for i, p := range plist {
		if p == s {
			plist = append(plist[0:i], plist[i+1:]...)
			break
		}
	}
	ret := strings.Join(plist, string(os.PathListSeparator))

	sarg := scriptArg{
		Cmd:    cmd,
		CfgDir: ret,
	}

	err := templateFile(
		filepath.Join(envHome, "files", `system_`+envShell),
		"",
		sarg,
		filepath.Join(envHome, "tmp", envCommand))
	if err != nil {
		fmt.Println(err)
		exit(1)
		return
	}
	return
}

func MakeShimsFile(cmd string) {
	//make script
	sarg := scriptArg{
		Cmd: cmd,
	}
	err := templateFile(
		filepath.Join(envHome, "files", `shims_`+envShell),
		"",
		sarg,
		filepath.Join(envHome, "shims", cmd+envExt))
	if err != nil {
		fmt.Println(err)
		exit(1)
		return
	}
	return
}

func ShowExecFile(m *Config, cmd string, fname string) bool {
	//make script
	cval := (*m).Commands[cmd]
	cnt := (*m).Images[cval]

	sarg := scriptArg{
		Tag:    cnt.Tag,
		Image:  cval,
		Cmd:    cmd,
		CfgDir: filepath.Dir(fname),
		Envs:   cnt.Commands[cmd],
	}

	tf := template.FuncMap{
	//    "tocmd": func(s string) string { sl:=strings.Split(s,"/");return sl[len(sl)-1] },
	//    "todir": func(s string) string { sl:=strings.Split(s,"/");return strings.Join(sl[:len(sl)-1],"/") },
	}
	hname := `header_` + envShell
	buf, err := ioutil.ReadFile(filepath.Join(envHome, "files", hname))
	if err != nil {
		fmt.Println(err)
		exit(1)
		return false
	}
	tpl := template.Must(template.New("script").Funcs(tf).Parse(string(buf) + cnt.Script))
	if err := tpl.Execute(os.Stdout, sarg); err != nil {
		fmt.Println(err, hname)
		exit(1)
		return false
	}
	return true
}
<commit_msg>Remove unused code.<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"text/template"

	"path/filepath"
)

func CheckCommand(cmd string, fname string) (m *Config) {
	if isV {
		fmt.Println("search:", fname)
	}
	if _, err := os.Stat(fname); err != nil {
		return nil
	}
	if isV {
		fmt.Println("  Found the config file.:", fname)
	}
	m = LoadConfig(fname)
	if m == nil {
		return
	}
	cval, ok := m.Commands[cmd]
	if !ok {
		if isV {
			fmt.Println("  Command not found.:", fname)
		}
		return nil
	}
	if isV {
		fmt.Println("found.:", cval)
	}
	return
}

func CheckImage(cmd string, fname string) (m *Config) {
	if isV {
		fmt.Println("search:", fname)
	}
	if _, err := os.Stat(fname); err != nil {
		return nil
	}
	if isV {
		fmt.Println("  Found the config file.:", fname)
	}
	m = LoadConfig(fname)
	if m == nil {
		return
	}

	cval, ok := m.Images[cmd]
	if !ok {
		if isV {
			fmt.Println("  Image not found.:", fname)
		}
		return nil
	}
	if isV {
		fmt.Println("found.:", cval)
	}
	return
}

var CheckCmdsMap = make(map[string][]string)

func InitCheckCmds() {
	CheckCmdsMap = make(map[string][]string)
}

func CheckCmds(cmd string, fname string) (m *Config) {
	if isV {
		fmt.Println("search:", fname)
	}
	if _, err := os.Stat(fname); err != nil {
		return nil
	}
	if isV {
		fmt.Println("  Found the config file.:", fname)
	}
	m = LoadConfig(fname)
	if m == nil {
		return nil
	}

	for key, val := range m.Commands {
		_, ok := CheckCmdsMap[key]
		if !ok {
			CheckCmdsMap[key] = []string{val, fname}
		}
	}

	return nil
}

func SearchConfig(cmd string, f func(string, string) *Config) (*Config, string) {
	fname := `.dcenv_` + envShell

	p := os.Getenv("DCENV_DIR")
	if p != "" {
		t := strings.Split(p, string(os.PathSeparator))
		for i := len(t); i > 0; i-- {
			s := strings.Join(t[0:i], string(os.PathSeparator)) + string(os.PathSeparator) + fname
			tc := f(cmd, s)
			if tc != nil {
				return tc, s
			}
		}
	}

	p, err := os.Getwd()
	if err != nil {
		fmt.Println(err)
		exit(1)
		return nil, p
	}
	t := strings.Split(p, string(os.PathSeparator))
	for i := len(t); i > 0; i-- {
		s := strings.Join(t[0:i], string(os.PathSeparator)) + string(os.PathSeparator) + fname
		tc := f(cmd, s)
		if tc != nil {
			return tc, s
		}
	}

	s := filepath.Join(envHome, "files", fname)
	tc := f(cmd, s)
	if tc != nil {
		return tc, s
	}

	return nil, s
}

type scriptArg struct {
	Tag    string
	Image  string
	Cmd    string
	CfgDir string
	Envs   map[string]string
}

func MakeArgsFile(sft int) {
	s := ""
	for _, t := range os.Args[sft:] {
		if strings.Contains(t, " ") {
			s += ` "` + t + `"`
		} else {
			s += " " + t
		}
	}
	//  if len(os.Args[sft:]) > 0 {
	//    s = `"`+strings.Join(os.Args[sft:],`" "`)+`"`
	//  }
	fname := filepath.Join(envHome, "files", "__args__")
	err := ioutil.WriteFile(fname, []byte(s), 0666)
	if err != nil {
		fmt.Println(err)
		exit(1)
		return
	}
}

func templateFile(fname string, addBuf string, sarg interface{}, oname string) error {
	buf, e := ioutil.ReadFile(fname)
	if e != nil {
		return e
	}
	fp, e := os.OpenFile(oname, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
	if e != nil {
		return e
	}
	if e := template.Must(template.New("script").Parse(string(buf)+addBuf)).Execute(fp, sarg); e != nil {
		return e
	}
	if e := fp.Close(); e != nil {
		return e
	}
	if isV {
		fmt.Println("write file:", oname)
	}
	return nil
}

func MakeExecFile(m *Config, cmd string, fname string) {
	//make script
	cval := m.Commands[cmd]
	cnt := m.Images[cval]

	sarg := scriptArg{
		Tag:    cnt.Tag,
		Image:  cval,
		Cmd:    cmd,
		CfgDir: filepath.Dir(fname),
		Envs:   cnt.Commands[cmd],
	}

	err := templateFile(
		filepath.Join(envHome, "files", `header_`+envShell),
		cnt.Script,
		sarg,
		filepath.Join(envHome, "tmp", envCommand))
	if err != nil {
		fmt.Println(err)
		exit(1)
		return
	}
	return
}

func MakeExecFileSystem(cmd string) {
	s := filepath.Join(envHome, "shims")
	plist := filepath.SplitList(os.Getenv("PATH"))
	for i, p := range plist {
		if p == s {
			plist = append(plist[0:i], plist[i+1:]...)
			break
		}
	}
	ret := strings.Join(plist, string(os.PathListSeparator))

	sarg := scriptArg{
		Cmd:    cmd,
		CfgDir: ret,
	}

	err := templateFile(
		filepath.Join(envHome, "files", `system_`+envShell),
		"",
		sarg,
		filepath.Join(envHome, "tmp", envCommand))
	if err != nil {
		fmt.Println(err)
		exit(1)
		return
	}
	return
}

func MakeShimsFile(cmd string) {
	//make script
	sarg := scriptArg{
		Cmd: cmd,
	}
	err := templateFile(
		filepath.Join(envHome, "files", `shims_`+envShell),
		"",
		sarg,
		filepath.Join(envHome, "shims", cmd+envExt))
	if err != nil {
		fmt.Println(err)
		exit(1)
		return
	}
	return
}

func ShowExecFile(m *Config, cmd string, fname string) bool {
	//make script
	cval := (*m).Commands[cmd]
	cnt := (*m).Images[cval]

	sarg := scriptArg{
		Tag:    cnt.Tag,
		Image:  cval,
		Cmd:    cmd,
		CfgDir: filepath.Dir(fname),
		Envs:   cnt.Commands[cmd],
	}

	hname := `header_` + envShell
	buf, err := ioutil.ReadFile(filepath.Join(envHome, "files", hname))
	if err != nil {
		fmt.Println(err)
		exit(1)
		return false
	}
	tpl := template.Must(template.New("script").Parse(string(buf) + cnt.Script))
	if err := tpl.Execute(os.Stdout, sarg); err != nil {
		fmt.Println(err, hname)
		exit(1)
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package main

// TODO: don't Atoi? gorilla already matches regex, and Atoi returns id 0 if error

import (
	"encoding/json"
	"fmt"
	"log"
	"mime"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/gorilla/mux"

	"example.com/internal/taskstore"
)

type taskServer struct {
	// Mutex protects access to the 'store' field. We don't make assumptions
	// about the safety of any method of store, so all accesses are protected.
	sync.Mutex
	store *taskstore.TaskStore
}

func NewTaskServer() *taskServer {
	store := taskstore.New()
	return &taskServer{store: store}
}

// renderJSON renders 'v' as JSON and writes it as a response into w.
func renderJSON(w http.ResponseWriter, v interface{}) {
	js, err := json.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(js)
}

func (ts *taskServer) createTaskHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling task create at %s\n", req.URL.Path)

	// Types used internally in this handler to (de-)serialize the request and
	// response from/to JSON.
	type RequestTask struct {
		Text string    `json:"text"`
		Tags []string  `json:"tags"`
		Due  time.Time `json:"due"`
	}

	type ResponseId struct {
		Id int `json:"id"`
	}

	// Enforce a JSON Content-Type.
	contentType := req.Header.Get("Content-Type")
	mediatype, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if mediatype != "application/json" {
		http.Error(w, "expect application/json Content-Type", http.StatusUnsupportedMediaType)
		return
	}

	dec := json.NewDecoder(req.Body)
	dec.DisallowUnknownFields()
	var rt RequestTask
	if err := dec.Decode(&rt); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	ts.Lock()
	id := ts.store.CreateTask(rt.Text, rt.Tags, rt.Due)
	defer ts.Unlock()

	renderJSON(w, ResponseId{Id: id})
}

func (ts *taskServer) getAllTasksHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling get all tasks at %s\n", req.URL.Path)

	ts.Lock()
	allTasks := ts.store.GetAllTasks()
	ts.Unlock()

	renderJSON(w, allTasks)
}

func (ts *taskServer) getTaskHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling get task at %s\n", req.URL.Path)

	// Here and elsewhere, not checking error of Atoi because the router only
	// matches the [0-9]+ regex.
	id, _ := strconv.Atoi(mux.Vars(req)["id"])
	ts.Lock()
	task, err := ts.store.GetTask(id)
	ts.Unlock()

	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}

	renderJSON(w, task)
}

func (ts *taskServer) deleteTaskHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling delete task at %s\n", req.URL.Path)

	id, _ := strconv.Atoi(mux.Vars(req)["id"])
	ts.Lock()
	err := ts.store.DeleteTask(id)
	ts.Unlock()

	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
	}
}

func (ts *taskServer) deleteAllTasksHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling delete all tasks at %s\n", req.URL.Path)
	ts.Lock()
	ts.store.DeleteAllTasks()
	ts.Unlock()
}

func (ts *taskServer) tagHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling tasks by tag at %s\n", req.URL.Path)

	tag := mux.Vars(req)["tag"]
	ts.Lock()
	tasks := ts.store.GetTasksByTag(tag)
	ts.Unlock()

	renderJSON(w, tasks)
}

func (ts *taskServer) dueHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling tasks by tag at %s\n", req.URL.Path)

	vars := mux.Vars(req)
	badRequestError := func() {
		http.Error(w, fmt.Sprintf("expect /due/<year>/<month>/<day>, got %v", req.URL.Path), http.StatusBadRequest)
	}

	year, _ := strconv.Atoi(vars["year"])
	month, _ := strconv.Atoi(vars["month"])
	if month < int(time.January) || month > int(time.December) {
		badRequestError()
		return
	}
	day, _ := strconv.Atoi(vars["day"])

	ts.Lock()
	tasks := ts.store.GetTasksByDueDate(year, time.Month(month), day)
	ts.Unlock()

	renderJSON(w, tasks)
}

func main() {
	router := mux.NewRouter()
	router.StrictSlash(true)
	server := NewTaskServer()

	router.HandleFunc("/task/", server.createTaskHandler).Methods("POST")
	router.HandleFunc("/task/", server.getAllTasksHandler).Methods("GET")
	router.HandleFunc("/task/", server.deleteAllTasksHandler).Methods("DELETE")
	router.HandleFunc("/task/{id:[0-9]+}/", server.getTaskHandler).Methods("GET")
	router.HandleFunc("/task/{id:[0-9]+}/", server.deleteTaskHandler).Methods("DELETE")
	router.HandleFunc("/tag/{tag}/", server.tagHandler).Methods("GET")
	router.HandleFunc("/due/{year:[0-9]+}/{month:[0-9]+}/{day:[0-9]+}/", server.dueHandler).Methods("GET")

	log.Fatal(http.ListenAndServe("localhost:"+os.Getenv("SERVERPORT"), router))
}
<commit_msg>Clean up TODO<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"mime"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/gorilla/mux"

	"example.com/internal/taskstore"
)

type taskServer struct {
	// Mutex protects access to the 'store' field. We don't make assumptions
	// about the safety of any method of store, so all accesses are protected.
	sync.Mutex
	store *taskstore.TaskStore
}

func NewTaskServer() *taskServer {
	store := taskstore.New()
	return &taskServer{store: store}
}

// renderJSON renders 'v' as JSON and writes it as a response into w.
func renderJSON(w http.ResponseWriter, v interface{}) {
	js, err := json.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(js)
}

func (ts *taskServer) createTaskHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling task create at %s\n", req.URL.Path)

	// Types used internally in this handler to (de-)serialize the request and
	// response from/to JSON.
	type RequestTask struct {
		Text string    `json:"text"`
		Tags []string  `json:"tags"`
		Due  time.Time `json:"due"`
	}

	type ResponseId struct {
		Id int `json:"id"`
	}

	// Enforce a JSON Content-Type.
	contentType := req.Header.Get("Content-Type")
	mediatype, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if mediatype != "application/json" {
		http.Error(w, "expect application/json Content-Type", http.StatusUnsupportedMediaType)
		return
	}

	dec := json.NewDecoder(req.Body)
	dec.DisallowUnknownFields()
	var rt RequestTask
	if err := dec.Decode(&rt); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	ts.Lock()
	id := ts.store.CreateTask(rt.Text, rt.Tags, rt.Due)
	defer ts.Unlock()

	renderJSON(w, ResponseId{Id: id})
}

func (ts *taskServer) getAllTasksHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling get all tasks at %s\n", req.URL.Path)

	ts.Lock()
	allTasks := ts.store.GetAllTasks()
	ts.Unlock()

	renderJSON(w, allTasks)
}

func (ts *taskServer) getTaskHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling get task at %s\n", req.URL.Path)

	// Here and elsewhere, not checking error of Atoi because the router only
	// matches the [0-9]+ regex.
	id, _ := strconv.Atoi(mux.Vars(req)["id"])
	ts.Lock()
	task, err := ts.store.GetTask(id)
	ts.Unlock()

	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}

	renderJSON(w, task)
}

func (ts *taskServer) deleteTaskHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling delete task at %s\n", req.URL.Path)

	id, _ := strconv.Atoi(mux.Vars(req)["id"])
	ts.Lock()
	err := ts.store.DeleteTask(id)
	ts.Unlock()

	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
	}
}

func (ts *taskServer) deleteAllTasksHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling delete all tasks at %s\n", req.URL.Path)
	ts.Lock()
	ts.store.DeleteAllTasks()
	ts.Unlock()
}

func (ts *taskServer) tagHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling tasks by tag at %s\n", req.URL.Path)

	tag := mux.Vars(req)["tag"]
	ts.Lock()
	tasks := ts.store.GetTasksByTag(tag)
	ts.Unlock()

	renderJSON(w, tasks)
}

func (ts *taskServer) dueHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("handling tasks by tag at %s\n", req.URL.Path)

	vars := mux.Vars(req)
	badRequestError := func() {
		http.Error(w, fmt.Sprintf("expect /due/<year>/<month>/<day>, got %v", req.URL.Path), http.StatusBadRequest)
	}

	year, _ := strconv.Atoi(vars["year"])
	month, _ := strconv.Atoi(vars["month"])
	if month < int(time.January) || month > int(time.December) {
		badRequestError()
		return
	}
	day, _ := strconv.Atoi(vars["day"])

	ts.Lock()
	tasks := ts.store.GetTasksByDueDate(year, time.Month(month), day)
	ts.Unlock()

	renderJSON(w, tasks)
}

func main() {
	router := mux.NewRouter()
	router.StrictSlash(true)
	server := NewTaskServer()

	router.HandleFunc("/task/", server.createTaskHandler).Methods("POST")
	router.HandleFunc("/task/", server.getAllTasksHandler).Methods("GET")
	router.HandleFunc("/task/", server.deleteAllTasksHandler).Methods("DELETE")
	router.HandleFunc("/task/{id:[0-9]+}/", server.getTaskHandler).Methods("GET")
	router.HandleFunc("/task/{id:[0-9]+}/", server.deleteTaskHandler).Methods("DELETE")
	router.HandleFunc("/tag/{tag}/", server.tagHandler).Methods("GET")
	router.HandleFunc("/due/{year:[0-9]+}/{month:[0-9]+}/{day:[0-9]+}/", server.dueHandler).Methods("GET")

	log.Fatal(http.ListenAndServe("localhost:"+os.Getenv("SERVERPORT"), router))
}
<|endoftext|>


<commit_before>package transcode

import (
	"errors"
)

var (
	// ErrInvalidCodec is returned when an invalid transcoder codec is selected
	ErrInvalidCodec = errors.New("transcode: no such transcoder codec")
	// ErrInvalidQuality is returned when an invalid quality is selected for a given codec
	ErrInvalidQuality = errors.New("transcode: invalid quality for transcoder codec")
)

// Enabled determines whether transcoding is available and enabled for wavepipe
var Enabled bool

// FFmpegPath is the path to the ffmpeg binary detected by the transcode manager
var FFmpegPath string

// Transcoder represents a transcoding operation, and the methods which must be defined
// for a transcoder
type Transcoder interface {
	Codec() string
	FFmpegCodec() string
	Quality() string
}

// Factory generates a new Transcoder depending on the input parameters
func Factory(codec string, quality string) (Transcoder, error) {
	// Check for a valid codec
	switch codec {
	// MP3
	case "mp3", "MP3":
		return NewMP3Transcoder(quality)
	// Invalid choice
	default:
		return nil, ErrInvalidCodec
	}
}
<commit_msg>Remove unneded method from Transcoder interface<commit_after>package transcode

import (
	"errors"
)

var (
	// ErrInvalidCodec is returned when an invalid transcoder codec is selected
	ErrInvalidCodec = errors.New("transcode: no such transcoder codec")
	// ErrInvalidQuality is returned when an invalid quality is selected for a given codec
	ErrInvalidQuality = errors.New("transcode: invalid quality for transcoder codec")
)

// Enabled determines whether transcoding is available and enabled for wavepipe
var Enabled bool

// FFmpegPath is the path to the ffmpeg binary detected by the transcode manager
var FFmpegPath string

// Transcoder represents a transcoding operation, and the methods which must be defined
// for a transcoder
type Transcoder interface {
	Codec() string
	Quality() string
}

// Factory generates a new Transcoder depending on the input parameters
func Factory(codec string, quality string) (Transcoder, error) {
	// Check for a valid codec
	switch codec {
	// MP3
	case "mp3", "MP3":
		return NewMP3Transcoder(quality)
	// Invalid choice
	default:
		return nil, ErrInvalidCodec
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/shiena/ansicolor"

	"../alias"
	"../commands"
	"../completion"
	"../conio"
	"../dos"
	"../history"
	"../interpreter"
	"../lua"
)

var rxAnsiEscCode = regexp.MustCompile("\x1b[^a-zA-Z]*[a-zA-Z]")

var stamp string
var commit string
var version string
var ansiOut io.Writer

func nyagosPrompt(L lua.Lua) int {
	template, err := L.ToString(-1)
	if err != nil {
		template = "[" + err.Error() + "]"
	}
	text := Format2Prompt(template)
	fmt.Fprint(ansiOut, text)
	text = rxAnsiEscCode.ReplaceAllString(text, "")
	lfPos := strings.LastIndex(text, "\n")
	if lfPos >= 0 {
		text = text[lfPos+1:]
	}
	L.PushInteger(lua.Integer(conio.GetStringWidth(text)))
	return 1
}

func printPrompt(this *conio.LineEditor) int {
	L, ok := this.Tag.(lua.Lua)
	if !ok {
		panic("printPrompt conio.LineEditor.Tag is not lua.Lua")
	}
	L.GetGlobal("nyagos")
	L.GetField(-1, "prompt")
	L.Remove(-2)

	if !L.IsFunction(-1) {
		L.Pop(1)
		return 0
	}

	L.PushString(os.Getenv("PROMPT"))
	L.Call(1, 1)
	length, lengthErr := L.ToInteger(-1)
	L.Pop(1)
	if lengthErr == nil {
		return length
	} else {
		fmt.Fprintf(os.Stderr,
			"nyagos.prompt: Length invalid: %s\n",
			lengthErr.Error())
		return 0
	}
}

func when_panic() {
	err := recover()
	if err == nil {
		return
	}
	fmt.Fprintln(os.Stderr, "************ Panic Occured. ***********")
	fmt.Fprintln(os.Stderr, err)
	fmt.Fprintln(os.Stderr, "*** Please copy these error message ***")
	fmt.Fprintln(os.Stderr, "*** And hit ENTER key to quit.      ***")
	var dummy [1]byte
	os.Stdin.Read(dummy[:])
}

func main() {
	defer when_panic()

	dos.CoInitializeEx(0, dos.COINIT_MULTITHREADED)
	defer dos.CoUninitialize()

	conio.DisableCtrlC()

	completion := conio.KeyGoFuncT{F: completion.KeyFuncCompletion}

	if err := conio.BindKeySymbolFunc(conio.K_CTRL_I, "COMPLETE", &completion); err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
	}

	// ANSI Escape Sequence Support
	ansiOut = ansicolor.NewAnsiColorWriter(os.Stdout)

	commands.Init()
	alias.Init()

	// Lua extension
	L := lua.New()
	L.OpenLibs()
	SetLuaFunctions(L)
	L.GetGlobal("nyagos")
	if stamp != "" {
		L.PushString(stamp)
		L.SetField(-2, "stamp")
	}
	if commit != "" {
		L.PushString(commit)
		L.SetField(-2, "commit")
	}
	if version != "" {
		L.PushString(version)
		L.SetField(-2, "version")
	}
	L.PushGoFunction(nyagosPrompt)
	L.SetField(-2, "prompt")
	L.Pop(1)
	defer L.Close()

	if !optionParse(L) {
		return
	}

	appData := filepath.Join(os.Getenv("APPDATA"), "NYAOS_ORG")
	os.Mkdir(appData, 0777)
	histPath := filepath.Join(appData, "nyagos.history")
	history.Load(histPath)
	history.Save(histPath) // cut over max-line

	exeName, exeNameErr := dos.GetModuleFileName()
	if exeNameErr != nil {
		fmt.Fprintln(os.Stderr, exeNameErr)
	}
	exeFolder := filepath.Dir(exeName)
	nyagos_lua := filepath.Join(exeFolder, "nyagos.lua")
	if _, err := os.Stat(nyagos_lua); err == nil {
		err := L.Source(nyagos_lua)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
	}

	conio.DefaultEditor.Tag = L
	conio.DefaultEditor.Prompt = printPrompt

	for {
		wd, wdErr := os.Getwd()
		if wdErr == nil {
			conio.SetTitle("NYAGOS - " + wd)
		} else {
			conio.SetTitle("NYAGOS - " + wdErr.Error())
		}
		history_count := conio.DefaultEditor.HistoryLen()
		line, cont := conio.DefaultEditor.ReadLine()
		if !cont {
			break
		}

		var isReplaced bool
		line, isReplaced = history.Replace(line)
		if isReplaced {
			fmt.Fprintln(os.Stdout, line)
		}
		if line == "" {
			continue
		}
		if conio.DefaultEditor.HistoryLen() > history_count {
			fd, err := os.OpenFile(histPath, os.O_APPEND, 0600)
			if err == nil {
				fmt.Fprintln(fd, line)
				fd.Close()
			} else {
				fmt.Fprintln(os.Stderr, err.Error())
			}
		} else {
			conio.DefaultEditor.HistoryResetPointer()
		}

		stackPos := L.GetTop()
		L.GetGlobal("nyagos")
		L.GetField(-1, "filter")
		if L.IsFunction(-1) {
			L.PushString(line)
			err := L.Call(1, 1)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
			} else {
				if L.IsString(-1) {
					line, err = L.ToString(-1)
					if err != nil {
						fmt.Fprintln(os.Stderr, err)
					}
				}
			}
		}
		L.SetTop(stackPos)

		it := interpreter.New()
		it.Tag = L
		whatToDo, err := it.Interpret(line)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
		if whatToDo == interpreter.SHUTDOWN {
			break
		}
	}
}
<commit_msg>Print Stacktrace not scrolled out at panic<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"runtime/debug"
	"strings"

	"github.com/shiena/ansicolor"

	"../alias"
	"../commands"
	"../completion"
	"../conio"
	"../dos"
	"../history"
	"../interpreter"
	"../lua"
)

var rxAnsiEscCode = regexp.MustCompile("\x1b[^a-zA-Z]*[a-zA-Z]")

var stamp string
var commit string
var version string
var ansiOut io.Writer

func nyagosPrompt(L lua.Lua) int {
	template, err := L.ToString(-1)
	if err != nil {
		template = "[" + err.Error() + "]"
	}
	text := Format2Prompt(template)
	fmt.Fprint(ansiOut, text)
	text = rxAnsiEscCode.ReplaceAllString(text, "")
	lfPos := strings.LastIndex(text, "\n")
	if lfPos >= 0 {
		text = text[lfPos+1:]
	}
	L.PushInteger(lua.Integer(conio.GetStringWidth(text)))
	return 1
}

func printPrompt(this *conio.LineEditor) int {
	L, ok := this.Tag.(lua.Lua)
	if !ok {
		panic("printPrompt conio.LineEditor.Tag is not lua.Lua")
	}
	L.GetGlobal("nyagos")
	L.GetField(-1, "prompt")
	L.Remove(-2)

	if !L.IsFunction(-1) {
		L.Pop(1)
		return 0
	}

	L.PushString(os.Getenv("PROMPT"))
	L.Call(1, 1)
	length, lengthErr := L.ToInteger(-1)
	L.Pop(1)
	if lengthErr == nil {
		return length
	} else {
		fmt.Fprintf(os.Stderr,
			"nyagos.prompt: Length invalid: %s\n",
			lengthErr.Error())
		return 0
	}
}

func when_panic() {
	err := recover()
	if err == nil {
		return
	}
	fmt.Fprintln(os.Stderr, "************ Panic Occured. ***********")
	fmt.Fprintln(os.Stderr, err)
	debug.PrintStack()
	fmt.Fprintln(os.Stderr, "*** Please copy these error message ***")
	fmt.Fprintln(os.Stderr, "*** And hit ENTER key to quit.      ***")
	var dummy [1]byte
	os.Stdin.Read(dummy[:])
}

func main() {
	defer when_panic()

	dos.CoInitializeEx(0, dos.COINIT_MULTITHREADED)
	defer dos.CoUninitialize()

	conio.DisableCtrlC()

	completion := conio.KeyGoFuncT{F: completion.KeyFuncCompletion}

	if err := conio.BindKeySymbolFunc(conio.K_CTRL_I, "COMPLETE", &completion); err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
	}

	// ANSI Escape Sequence Support
	ansiOut = ansicolor.NewAnsiColorWriter(os.Stdout)

	commands.Init()
	alias.Init()

	// Lua extension
	L := lua.New()
	L.OpenLibs()
	SetLuaFunctions(L)
	L.GetGlobal("nyagos")
	if stamp != "" {
		L.PushString(stamp)
		L.SetField(-2, "stamp")
	}
	if commit != "" {
		L.PushString(commit)
		L.SetField(-2, "commit")
	}
	if version != "" {
		L.PushString(version)
		L.SetField(-2, "version")
	}
	L.PushGoFunction(nyagosPrompt)
	L.SetField(-2, "prompt")
	L.Pop(1)
	defer L.Close()

	if !optionParse(L) {
		return
	}

	appData := filepath.Join(os.Getenv("APPDATA"), "NYAOS_ORG")
	os.Mkdir(appData, 0777)
	histPath := filepath.Join(appData, "nyagos.history")
	history.Load(histPath)
	history.Save(histPath) // cut over max-line

	exeName, exeNameErr := dos.GetModuleFileName()
	if exeNameErr != nil {
		fmt.Fprintln(os.Stderr, exeNameErr)
	}
	exeFolder := filepath.Dir(exeName)
	nyagos_lua := filepath.Join(exeFolder, "nyagos.lua")
	if _, err := os.Stat(nyagos_lua); err == nil {
		err := L.Source(nyagos_lua)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
	}

	conio.DefaultEditor.Tag = L
	conio.DefaultEditor.Prompt = printPrompt

	for {
		wd, wdErr := os.Getwd()
		if wdErr == nil {
			conio.SetTitle("NYAGOS - " + wd)
		} else {
			conio.SetTitle("NYAGOS - " + wdErr.Error())
		}
		history_count := conio.DefaultEditor.HistoryLen()
		line, cont := conio.DefaultEditor.ReadLine()
		if !cont {
			break
		}

		var isReplaced bool
		line, isReplaced = history.Replace(line)
		if isReplaced {
			fmt.Fprintln(os.Stdout, line)
		}
		if line == "" {
			continue
		}
		if conio.DefaultEditor.HistoryLen() > history_count {
			fd, err := os.OpenFile(histPath, os.O_APPEND, 0600)
			if err == nil {
				fmt.Fprintln(fd, line)
				fd.Close()
			} else {
				fmt.Fprintln(os.Stderr, err.Error())
			}
		} else {
			conio.DefaultEditor.HistoryResetPointer()
		}

		stackPos := L.GetTop()
		L.GetGlobal("nyagos")
		L.GetField(-1, "filter")
		if L.IsFunction(-1) {
			L.PushString(line)
			err := L.Call(1, 1)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
			} else {
				if L.IsString(-1) {
					line, err = L.ToString(-1)
					if err != nil {
						fmt.Fprintln(os.Stderr, err)
					}
				}
			}
		}
		L.SetTop(stackPos)

		it := interpreter.New()
		it.Tag = L
		whatToDo, err := it.Interpret(line)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}
		if whatToDo == interpreter.SHUTDOWN {
			break
		}
	}
}
<|endoftext|>


<commit_before>package proxy

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof" // Expose pprof if configured
	"os"
	"sort"
	"strconv"

	proxyAgent "github.com/hashicorp/consul/agent/proxy"
	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/command/flags"
	proxyImpl "github.com/hashicorp/consul/connect/proxy"

	"github.com/hashicorp/consul/logger"
	"github.com/hashicorp/logutils"
	"github.com/mitchellh/cli"
)

func New(ui cli.Ui, shutdownCh <-chan struct{}) *cmd {
	c := &cmd{UI: ui, shutdownCh: shutdownCh}
	c.init()
	return c
}

type cmd struct {
	UI    cli.Ui
	flags *flag.FlagSet
	http  *flags.HTTPFlags
	help  string

	shutdownCh <-chan struct{}

	logFilter *logutils.LevelFilter
	logOutput io.Writer
	logger    *log.Logger

	// flags
	logLevel    string
	cfgFile     string
	proxyID     string
	pprofAddr   string
	service     string
	serviceAddr string
	upstreams   map[string]proxyImpl.UpstreamConfig
	listen      string

	// test flags
	testNoStart bool // don't start the proxy, just exit 0
}

func (c *cmd) init() {
	c.flags = flag.NewFlagSet("", flag.ContinueOnError)

	c.flags.StringVar(&c.cfgFile, "dev-config", "",
		"If set, proxy config is read on startup from this file (in HCL or JSON"+
			"format). If a config file is given, the proxy will use that instead of "+
			"querying the local agent for it's configuration. It will not reload it "+
			"except on startup. In this mode the proxy WILL NOT authorize incoming "+
			"connections with the local agent which is totally insecure. This is "+
			"ONLY for internal development and testing and will probably be removed "+
			"once proxy implementation is more complete..")

	c.flags.StringVar(&c.proxyID, "proxy-id", "",
		"The proxy's ID on the local agent.")

	c.flags.StringVar(&c.logLevel, "log-level", "INFO",
		"Specifies the log level.")

	c.flags.StringVar(&c.pprofAddr, "pprof-addr", "",
		"Enable debugging via pprof. Providing a host:port (or just ':port') "+
			"enables profiling HTTP endpoints on that address.")

	c.flags.StringVar(&c.service, "service", "",
		"Name of the service this proxy is representing.")

	c.flags.Var((*FlagUpstreams)(&c.upstreams), "upstream",
		"Upstream service to support connecting to. The format should be "+
			"'name:addr', such as 'db:8181'. This will make 'db' available "+
			"on port 8181. This can be repeated multiple times.")

	c.flags.StringVar(&c.serviceAddr, "service-addr", "",
		"Address of the local service to proxy. Only useful if -listen "+
			"and -service are both set.")

	c.flags.StringVar(&c.listen, "listen", "",
		"Address to listen for inbound connections to the proxied service. "+
			"Must be specified with -service and -service-addr.")

	c.http = &flags.HTTPFlags{}
	flags.Merge(c.flags, c.http.ClientFlags())
	flags.Merge(c.flags, c.http.ServerFlags())
	c.help = flags.Usage(help, c.flags)
}

func (c *cmd) Run(args []string) int {
	if err := c.flags.Parse(args); err != nil {
		return 1
	}
	if len(c.flags.Args()) > 0 {
		c.UI.Error(fmt.Sprintf("Should have no non-flag arguments."))
		return 1
	}

	// Load the proxy ID and token from env vars if they're set
	if c.proxyID == "" {
		c.proxyID = os.Getenv(proxyAgent.EnvProxyId)
	}
	if c.http.Token() == "" {
		c.http.SetToken(os.Getenv(proxyAgent.EnvProxyToken))
	}

	// Setup the log outputs
	logConfig := &logger.Config{
		LogLevel: c.logLevel,
	}
	logFilter, logGate, _, logOutput, ok := logger.Setup(logConfig, c.UI)
	if !ok {
		return 1
	}
	c.logFilter = logFilter
	c.logOutput = logOutput
	c.logger = log.New(logOutput, "", log.LstdFlags)

	// Enable Pprof if needed
	if c.pprofAddr != "" {
		go func() {
			c.UI.Output(fmt.Sprintf("Starting pprof HTTP endpoints on "+
				"http://%s/debug/pprof", c.pprofAddr))
			log.Fatal(http.ListenAndServe(c.pprofAddr, nil))
		}()
	}

	// Setup Consul client
	client, err := c.http.APIClient()
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
		return 1
	}

	// Get the proper configuration watcher
	cfgWatcher, err := c.configWatcher(client)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error preparing configuration: %s", err))
		return 1
	}

	p, err := proxyImpl.New(client, cfgWatcher, c.logger)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Failed initializing proxy: %s", err))
		return 1
	}

	// Hook the shutdownCh up to close the proxy
	go func() {
		<-c.shutdownCh
		p.Close()
	}()

	c.UI.Output("Consul Connect proxy starting")

	c.UI.Output("Log data will now stream in as it occurs:\n")
	logGate.Flush()

	// Run the proxy unless our tests require we don't
	if !c.testNoStart {
		if err := p.Serve(); err != nil {
			c.UI.Error(fmt.Sprintf("Failed running proxy: %s", err))
		}
	}

	c.UI.Output("Consul Connect proxy shutdown")
	return 0
}

func (c *cmd) configWatcher(client *api.Client) (proxyImpl.ConfigWatcher, error) {
	// Manual configuration file is specified.
	if c.cfgFile != "" {
		cfg, err := proxyImpl.ParseConfigFile(c.cfgFile)
		if err != nil {
			return nil, err
		}
		return proxyImpl.NewStaticConfigWatcher(cfg), nil
	}

	// Use the configured proxy ID
	if c.proxyID != "" {
		return proxyImpl.NewAgentConfigWatcher(client, c.proxyID, c.logger)
	}

	// Otherwise, we're representing a manually specified service.
	if c.service == "" {
		return nil, fmt.Errorf(
			"-service or -proxy-id must be specified so that proxy can " +
				"configure itself.")
	}

	// Convert our upstreams to a slice of configurations. We do this
	// deterministically by alphabetizing the upstream keys. We do this so
	// that tests can compare the upstream values.
	upstreamKeys := make([]string, 0, len(c.upstreams))
	for k := range c.upstreams {
		upstreamKeys = append(upstreamKeys, k)
	}
	sort.Strings(upstreamKeys)
	upstreams := make([]proxyImpl.UpstreamConfig, 0, len(c.upstreams))
	for _, k := range upstreamKeys {
		upstreams = append(upstreams, c.upstreams[k])
	}

	// Parse out our listener if we have one
	var listener proxyImpl.PublicListenerConfig
	if c.listen != "" {
		host, portRaw, err := net.SplitHostPort(c.listen)
		if err != nil {
			return nil, err
		}

		port, err := strconv.ParseInt(portRaw, 0, 0)
		if err != nil {
			return nil, err
		}

		if c.serviceAddr == "" {
			return nil, fmt.Errorf(
				"-service-addr must be specified with -listen so the proxy " +
					"knows the backend service address.")
		}

		listener.BindAddress = host
		listener.BindPort = int(port)
		listener.LocalServiceAddress = c.serviceAddr
	}

	return proxyImpl.NewStaticConfigWatcher(&proxyImpl.Config{
		ProxiedServiceName: c.service,
		PublicListener:     listener,
		Upstreams:          upstreams,
	}), nil
}

func (c *cmd) Synopsis() string {
	return synopsis
}

func (c *cmd) Help() string {
	return c.help
}

const synopsis = "Runs a Consul Connect proxy"
const help = `
Usage: consul proxy [options]

  Starts a Consul Connect proxy and runs until an interrupt is received.
`
<commit_msg>command/connect/proxy: detailed help<commit_after>package proxy

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof" // Expose pprof if configured
	"os"
	"sort"
	"strconv"

	proxyAgent "github.com/hashicorp/consul/agent/proxy"
	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/command/flags"
	proxyImpl "github.com/hashicorp/consul/connect/proxy"

	"github.com/hashicorp/consul/logger"
	"github.com/hashicorp/logutils"
	"github.com/mitchellh/cli"
)

func New(ui cli.Ui, shutdownCh <-chan struct{}) *cmd {
	c := &cmd{UI: ui, shutdownCh: shutdownCh}
	c.init()
	return c
}

type cmd struct {
	UI    cli.Ui
	flags *flag.FlagSet
	http  *flags.HTTPFlags
	help  string

	shutdownCh <-chan struct{}

	logFilter *logutils.LevelFilter
	logOutput io.Writer
	logger    *log.Logger

	// flags
	logLevel    string
	cfgFile     string
	proxyID     string
	pprofAddr   string
	service     string
	serviceAddr string
	upstreams   map[string]proxyImpl.UpstreamConfig
	listen      string

	// test flags
	testNoStart bool // don't start the proxy, just exit 0
}

func (c *cmd) init() {
	c.flags = flag.NewFlagSet("", flag.ContinueOnError)

	c.flags.StringVar(&c.cfgFile, "dev-config", "",
		"If set, proxy config is read on startup from this file (in HCL or JSON"+
			"format). If a config file is given, the proxy will use that instead of "+
			"querying the local agent for it's configuration. It will not reload it "+
			"except on startup. In this mode the proxy WILL NOT authorize incoming "+
			"connections with the local agent which is totally insecure. This is "+
			"ONLY for internal development and testing and will probably be removed "+
			"once proxy implementation is more complete..")

	c.flags.StringVar(&c.proxyID, "proxy-id", "",
		"The proxy's ID on the local agent.")

	c.flags.StringVar(&c.logLevel, "log-level", "INFO",
		"Specifies the log level.")

	c.flags.StringVar(&c.pprofAddr, "pprof-addr", "",
		"Enable debugging via pprof. Providing a host:port (or just ':port') "+
			"enables profiling HTTP endpoints on that address.")

	c.flags.StringVar(&c.service, "service", "",
		"Name of the service this proxy is representing.")

	c.flags.Var((*FlagUpstreams)(&c.upstreams), "upstream",
		"Upstream service to support connecting to. The format should be "+
			"'name:addr', such as 'db:8181'. This will make 'db' available "+
			"on port 8181. This can be repeated multiple times.")

	c.flags.StringVar(&c.serviceAddr, "service-addr", "",
		"Address of the local service to proxy. Only useful if -listen "+
			"and -service are both set.")

	c.flags.StringVar(&c.listen, "listen", "",
		"Address to listen for inbound connections to the proxied service. "+
			"Must be specified with -service and -service-addr.")

	c.http = &flags.HTTPFlags{}
	flags.Merge(c.flags, c.http.ClientFlags())
	flags.Merge(c.flags, c.http.ServerFlags())
	c.help = flags.Usage(help, c.flags)
}

func (c *cmd) Run(args []string) int {
	if err := c.flags.Parse(args); err != nil {
		return 1
	}
	if len(c.flags.Args()) > 0 {
		c.UI.Error(fmt.Sprintf("Should have no non-flag arguments."))
		return 1
	}

	// Load the proxy ID and token from env vars if they're set
	if c.proxyID == "" {
		c.proxyID = os.Getenv(proxyAgent.EnvProxyId)
	}
	if c.http.Token() == "" {
		c.http.SetToken(os.Getenv(proxyAgent.EnvProxyToken))
	}

	// Setup the log outputs
	logConfig := &logger.Config{
		LogLevel: c.logLevel,
	}
	logFilter, logGate, _, logOutput, ok := logger.Setup(logConfig, c.UI)
	if !ok {
		return 1
	}
	c.logFilter = logFilter
	c.logOutput = logOutput
	c.logger = log.New(logOutput, "", log.LstdFlags)

	// Enable Pprof if needed
	if c.pprofAddr != "" {
		go func() {
			c.UI.Output(fmt.Sprintf("Starting pprof HTTP endpoints on "+
				"http://%s/debug/pprof", c.pprofAddr))
			log.Fatal(http.ListenAndServe(c.pprofAddr, nil))
		}()
	}

	// Setup Consul client
	client, err := c.http.APIClient()
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
		return 1
	}

	// Get the proper configuration watcher
	cfgWatcher, err := c.configWatcher(client)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error preparing configuration: %s", err))
		return 1
	}

	p, err := proxyImpl.New(client, cfgWatcher, c.logger)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Failed initializing proxy: %s", err))
		return 1
	}

	// Hook the shutdownCh up to close the proxy
	go func() {
		<-c.shutdownCh
		p.Close()
	}()

	c.UI.Output("Consul Connect proxy starting")

	c.UI.Output("Log data will now stream in as it occurs:\n")
	logGate.Flush()

	// Run the proxy unless our tests require we don't
	if !c.testNoStart {
		if err := p.Serve(); err != nil {
			c.UI.Error(fmt.Sprintf("Failed running proxy: %s", err))
		}
	}

	c.UI.Output("Consul Connect proxy shutdown")
	return 0
}

func (c *cmd) configWatcher(client *api.Client) (proxyImpl.ConfigWatcher, error) {
	// Manual configuration file is specified.
	if c.cfgFile != "" {
		cfg, err := proxyImpl.ParseConfigFile(c.cfgFile)
		if err != nil {
			return nil, err
		}
		return proxyImpl.NewStaticConfigWatcher(cfg), nil
	}

	// Use the configured proxy ID
	if c.proxyID != "" {
		return proxyImpl.NewAgentConfigWatcher(client, c.proxyID, c.logger)
	}

	// Otherwise, we're representing a manually specified service.
	if c.service == "" {
		return nil, fmt.Errorf(
			"-service or -proxy-id must be specified so that proxy can " +
				"configure itself.")
	}

	// Convert our upstreams to a slice of configurations. We do this
	// deterministically by alphabetizing the upstream keys. We do this so
	// that tests can compare the upstream values.
	upstreamKeys := make([]string, 0, len(c.upstreams))
	for k := range c.upstreams {
		upstreamKeys = append(upstreamKeys, k)
	}
	sort.Strings(upstreamKeys)
	upstreams := make([]proxyImpl.UpstreamConfig, 0, len(c.upstreams))
	for _, k := range upstreamKeys {
		upstreams = append(upstreams, c.upstreams[k])
	}

	// Parse out our listener if we have one
	var listener proxyImpl.PublicListenerConfig
	if c.listen != "" {
		host, portRaw, err := net.SplitHostPort(c.listen)
		if err != nil {
			return nil, err
		}

		port, err := strconv.ParseInt(portRaw, 0, 0)
		if err != nil {
			return nil, err
		}

		if c.serviceAddr == "" {
			return nil, fmt.Errorf(
				"-service-addr must be specified with -listen so the proxy " +
					"knows the backend service address.")
		}

		listener.BindAddress = host
		listener.BindPort = int(port)
		listener.LocalServiceAddress = c.serviceAddr
	}

	return proxyImpl.NewStaticConfigWatcher(&proxyImpl.Config{
		ProxiedServiceName: c.service,
		PublicListener:     listener,
		Upstreams:          upstreams,
	}), nil
}

func (c *cmd) Synopsis() string {
	return synopsis
}

func (c *cmd) Help() string {
	return c.help
}

const synopsis = "Runs a Consul Connect proxy"
const help = `
Usage: consul connect proxy [options]

  Starts a Consul Connect proxy and runs until an interrupt is received.
  The proxy can be used to accept inbound connections for a service,
  wrap outbound connections to upstream services, or both. This enables
  a non-Connect-aware application to use Connect.

  The proxy requires service:write permissions for the service it represents.

  Consul can automatically start and manage this proxy by specifying the
  "proxy" configuration within your service definition.

  The example below shows how to start a local proxy for establishing outbound
  connections to "db" representing the frontend service. Once running, any
  process that creates a TCP connection to the specified port (8181) will
  establish a mutual TLS connection to "db" identified as "frontend".

    $ consul connect proxy -service frontend -upstream db:8181

  The next example starts a local proxy that also accepts inbound connections
  on port 8443, authorizes the connection, then proxies it to port 8080:

    $ consul connect proxy \
        -service frontend \
        -service-addr 127.0.0.1:8080 \
        -listen ':8443'

  A proxy can accept both inbound connections as well as proxy to upstream
  services by specifying both the "-listen" and "-upstream" flags.

`
<|endoftext|>


<commit_before>package pir

import (
	"errors"
	"strings"

	"github.com/privacylab/talek/pir/common"
)

// DB is a memory area for PIR computations shared with a PIR daemon.
type DB struct {
	DB    []byte
	shard common.Shard
}

type pirReq struct {
	reqtype  int
	response chan []byte
}

// Server is a connection and state for a running PIR Server.
type Server struct {
	newshard   func(int, []byte, string) common.Shard
	backing    string
	CellLength int
	CellCount  int
	BatchSize  int
	DB         *DB
}

// Connect opens a Server for communication with a unix-socket representating a
// running PIR Daemon.
func NewServer(backing string) (*Server, error) {
	server := new(Server)
	server.backing = backing

	for k, v := range common.PIRBackings {
		if strings.HasPrefix(backing, k) {
			server.newshard = v
			return server, nil
		}
	}

	return nil, errors.New("Backing " + backing + " is not known")
}

// Disconnect closes a Server connection
func (s *Server) Disconnect() error {
	if s.DB != nil {
		s.DB.Free()
	}
	return nil
}

// Configure sets the size of the DB and operational parameters.
func (s *Server) Configure(celllength int, cellcount int, batchsize int) error {
	s.BatchSize = batchsize
	s.CellCount = cellcount
	s.CellLength = celllength

	if s.CellCount%8 != 0 || s.CellLength%8 != 0 {
		return errors.New("invalid sizing of database; everything needs to be multiples of 8 bytes")
	}

	return nil
}

// GetDB provides direct access to the DB of the Server.
func (s *Server) GetDB() (*DB, error) {
	if s.CellCount == 0 || s.CellLength == 0 {
		return nil, errors.New("pir server unconfigured")
	}
	db := new(DB)

	db.DB = make([]byte, s.CellCount*s.CellLength)

	return db, nil
}

// SetDB updates the PIR Server Database
func (s *Server) SetDB(db *DB) error {
	if s.DB != nil {
		s.DB.Free()
	}

	db.shard = s.newshard(s.CellLength, db.DB, s.backing)
	if db.shard == nil {
		return errors.New("Couldn't set DB")
	}
	s.DB = db
	return nil
}

// Free releases memory for a DB instance
func (db *DB) Free() error {
	if db.shard != nil {
		db.shard.Free()
	}
	return nil
}

// Read makes a PIR request against the server.
func (s *Server) Read(masks []byte, responseChan chan []byte) error {
	if s.DB == nil || s.CellCount == 0 {
		return errors.New("db not configured")
	}

	if len(masks) != (s.CellCount*s.BatchSize)/8 {
		return errors.New("wrong mask length")
	}

	responses, err := s.DB.shard.Read(masks, len(masks))
	if err != nil {
		return err
	}
	responseChan <- responses

	return nil
}
<commit_msg>libpir needs to pass length of individual request to Read call<commit_after>package pir

import (
	"errors"
	"strings"

	"github.com/privacylab/talek/pir/common"
)

// DB is a memory area for PIR computations shared with a PIR daemon.
type DB struct {
	DB    []byte
	shard common.Shard
}

type pirReq struct {
	reqtype  int
	response chan []byte
}

// Server is a connection and state for a running PIR Server.
type Server struct {
	newshard   func(int, []byte, string) common.Shard
	backing    string
	CellLength int
	CellCount  int
	BatchSize  int
	DB         *DB
}

// Connect opens a Server for communication with a unix-socket representating a
// running PIR Daemon.
func NewServer(backing string) (*Server, error) {
	server := new(Server)
	server.backing = backing

	for k, v := range common.PIRBackings {
		if strings.HasPrefix(backing, k) {
			server.newshard = v
			return server, nil
		}
	}

	return nil, errors.New("Backing " + backing + " is not known")
}

// Disconnect closes a Server connection
func (s *Server) Disconnect() error {
	if s.DB != nil {
		s.DB.Free()
	}
	return nil
}

// Configure sets the size of the DB and operational parameters.
func (s *Server) Configure(celllength int, cellcount int, batchsize int) error {
	s.BatchSize = batchsize
	s.CellCount = cellcount
	s.CellLength = celllength

	if s.CellCount%8 != 0 || s.CellLength%8 != 0 {
		return errors.New("invalid sizing of database; everything needs to be multiples of 8 bytes")
	}

	return nil
}

// GetDB provides direct access to the DB of the Server.
func (s *Server) GetDB() (*DB, error) {
	if s.CellCount == 0 || s.CellLength == 0 {
		return nil, errors.New("pir server unconfigured")
	}
	db := new(DB)

	db.DB = make([]byte, s.CellCount*s.CellLength)

	return db, nil
}

// SetDB updates the PIR Server Database
func (s *Server) SetDB(db *DB) error {
	if s.DB != nil {
		s.DB.Free()
	}

	db.shard = s.newshard(s.CellLength, db.DB, s.backing)
	if db.shard == nil {
		return errors.New("Couldn't set DB")
	}
	s.DB = db
	return nil
}

// Free releases memory for a DB instance
func (db *DB) Free() error {
	if db.shard != nil {
		db.shard.Free()
	}
	return nil
}

// Read makes a PIR request against the server.
func (s *Server) Read(masks []byte, responseChan chan []byte) error {
	if s.DB == nil || s.CellCount == 0 {
		return errors.New("db not configured")
	}

	if len(masks) != (s.CellCount*s.BatchSize)/8 {
		return errors.New("wrong mask length")
	}

	responses, err := s.DB.shard.Read(masks, s.CellCount/8)
	if err != nil {
		return err
	}
	responseChan <- responses

	return nil
}
<|endoftext|>


<commit_before>package cruncy

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/segmentio/ksuid"
	log "github.com/sirupsen/logrus"
)

// DataFields is map to interface def

// AtomicIntCounter uses an int64 internally.
type AtomicIntCounter int64

// Get returns current counter
func (c *AtomicIntCounter) Get() int64 {
	return atomic.LoadInt64((*int64)(c))
}

func (c *AtomicIntCounter) inc() int64 {
	return atomic.AddInt64((*int64)(c), 1)
}

// TimerData containes run time data
type TimerData struct {
	Title          string
	Uuid           string
	StartTimeRun   time.Time
	StartTimeBatch time.Time
	EndTimeRun     time.Time

	BatchSize  int64
	PrevRows   int64
	Index      AtomicIntCounter
	ErrorCount AtomicIntCounter
	mu         sync.RWMutex
}

// NewTimer creates a new timer struct
func NewTimer(title string) *TimerData {
	timer := &TimerData{}
	timer.Title = title
	timer.Uuid = ksuid.New().String()
	timer.StartTimeRun = time.Now()
	timer.StartTimeBatch = timer.StartTimeRun
	timer.PrevRows = 0
	timer.ErrorCount = 0
	return timer
}

// BatchDuractionSeconds returns durection in seconds
func (timer *TimerData) BatchDuractionSeconds() int64 {
	t1 := time.Now()
	duration := t1.Sub(timer.StartTimeBatch)
	return int64(duration.Seconds())
}

// TotalDuractionSeconds returns total duration in seconds
func (timer *TimerData) TotalDuractionSeconds() int64 {
	t1 := time.Now()
	duration := t1.Sub(timer.StartTimeRun)
	return int64(duration.Seconds())
}

// TotalDuration returns duration as a time.Duration
func (timer *TimerData) TotalDuration() time.Duration {
	t1 := time.Now()
	return t1.Sub(timer.StartTimeRun)
}

// ShowTotalDuration outputs duration to log with fields
func (timer *TimerData) ShowTotalDuration() {

	cnt := timer.Index.Get()
	timer.mu.RLock()
	uuid := timer.Uuid
	title := timer.Title
	startTime := timer.StartTimeRun
	timer.mu.RUnlock()

	t1 := time.Now()
	duration := t1.Sub(startTime)
	ds := int64(duration.Seconds())
	if ds > 0 {
		msg := fmt.Sprintf("Total duration:, %v rows =%d rate = %d rows/sec ", duration, cnt, cnt/ds)
		log.WithFields(log.Fields{
			"uuid":       uuid,
			"title":      title,
			"total_rows": cnt,
			"avg_flow":   cnt / ds,
			"State":      "stopped",
		}).Info(msg)
	} else {
		log.WithFields(log.Fields{
			"uuid":       uuid,
			"title":      title,
			"total_rows": cnt,
			"avg_flow":   cnt,
			"State":      "stopped",
		}).Infof("Total duration:, %v rows =%d  SUPER FAST", duration, cnt)

	}
}

// LogFields returns summary data as fieldmap
func (timer *TimerData) LogFields() log.Fields {
	cnt := timer.Index.Get()
	timer.mu.RLock()
	uuid := timer.Uuid
	title := timer.Title
	startTime := timer.StartTimeRun
	timer.mu.RUnlock()

	t1 := time.Now()
	duration := t1.Sub(startTime)
	ds := int64(duration.Seconds())
	avgFlow := cnt
	if ds > 0 {
		avgFlow = cnt / ds
	}

	return log.Fields{
		"uuid":       uuid,
		"title":      title,
		"total_rows": cnt,
		"avg_flow/s": avgFlow,
		"start_time": startTime.UTC().Format("2006-01-02T15:04:05-0700"),
		"end_time":   t1.UTC().Format("2006-01-02T15:04:05-0700"),
	}

}

// ShowBatchTime show averages to now
func (timer *TimerData) ShowBatchTime() {

	cnt := timer.Index.Get()
	timer.mu.RLock()
	uuid := timer.Uuid
	title := timer.Title
	prevRows := timer.PrevRows
	startTime := timer.StartTimeBatch
	timer.mu.RUnlock()

	diff := cnt - prevRows

	t1 := time.Now()
	duration := t1.Sub(startTime)
	d2 := timer.TotalDuration()

	ds := int64(d2.Seconds())
	dsBatch := int64(duration.Seconds())

	if ds > 0 && dsBatch > 0 {
		msg := fmt.Sprintf("%d rows avg flow %d/s - batch time %v batch size %d batch_flow %d \n", cnt, cnt/ds, duration, diff, diff/dsBatch)
		log.WithFields(log.Fields{
			"uuid":       uuid,
			"title":      title,
			"index":      cnt,
			"total_flow": cnt / ds,
			"batch_time": duration,
			"batch_size": diff,
			"batch_flow": diff / dsBatch,
			"State":      "in_batch",
		}).Info(msg)
	} else {
		log.Printf("%d rows - batch time %v \n", cnt, duration)
	}

	timer.mu.Lock()
	timer.PrevRows = cnt
	timer.StartTimeBatch = time.Now()
	timer.mu.Unlock()
}

// Tick increases tick with one
func (timer *TimerData) Tick() {
	cnt := timer.Index.inc()

	if cnt%100000 == 0 {
		timer.ShowBatchTime()
	}
}

// Stop stops the timer
func (timer *TimerData) Stop() time.Time {
	timer.mu.Lock()
	timer.EndTimeRun = time.Now()
	timer.mu.Unlock()
	return timer.EndTimeRun
}

// IncError adds one to number of errors
func (timer *TimerData) IncError() int64 {
	timer.ErrorCount.inc()
	return timer.ErrorCount.Get()
}
<commit_msg>Added sep logger<commit_after>package cruncy

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/segmentio/ksuid"
	log "github.com/sirupsen/logrus"
)

// DataFields is map to interface def

// AtomicIntCounter uses an int64 internally.
type AtomicIntCounter int64

// Get returns current counter
func (c *AtomicIntCounter) Get() int64 {
	return atomic.LoadInt64((*int64)(c))
}

func (c *AtomicIntCounter) inc() int64 {
	return atomic.AddInt64((*int64)(c), 1)
}

// TimerData containes run time data
type TimerData struct {
	Title          string
	Uuid           string
	StartTimeRun   time.Time
	StartTimeBatch time.Time
	EndTimeRun     time.Time

	BatchSize  int64
	PrevRows   int64
	Index      AtomicIntCounter
	ErrorCount AtomicIntCounter
	mu         sync.RWMutex
	Logger     *log.Entry
}

// NewTimer creates a new timer struct
func NewTimer(title string) *TimerData {
	timer := &TimerData{}
	timer.Title = title
	timer.Uuid = ksuid.New().String()
	timer.StartTimeRun = time.Now()
	timer.StartTimeBatch = timer.StartTimeRun
	timer.PrevRows = 0
	timer.ErrorCount = 0
	timer.Logger = log.WithFields(log.Fields{"uuid": timer.Uuid, "title": title})
	return timer
}

// BatchDuractionSeconds returns durection in seconds
func (timer *TimerData) BatchDuractionSeconds() int64 {
	t1 := time.Now()
	duration := t1.Sub(timer.StartTimeBatch)
	return int64(duration.Seconds())
}

// TotalDuractionSeconds returns total duration in seconds
func (timer *TimerData) TotalDuractionSeconds() int64 {
	t1 := time.Now()
	duration := t1.Sub(timer.StartTimeRun)
	return int64(duration.Seconds())
}

// TotalDuration returns duration as a time.Duration
func (timer *TimerData) TotalDuration() time.Duration {
	t1 := time.Now()
	return t1.Sub(timer.StartTimeRun)
}

// ShowTotalDuration outputs duration to log with fields
func (timer *TimerData) ShowTotalDuration() {

	cnt := timer.Index.Get()
	timer.mu.RLock()
	startTime := timer.StartTimeRun
	timer.mu.RUnlock()

	t1 := time.Now()
	duration := t1.Sub(startTime)
	ds := int64(duration.Seconds())
	if ds > 0 {
		msg := fmt.Sprintf("Total duration:, %v rows =%d rate = %d rows/sec ", duration, cnt, cnt/ds)
		timer.Logger.WithFields(log.Fields{
			"total_rows": cnt,
			"avg_flow":   cnt / ds,
			"State":      "stopped",
		}).Info(msg)
	} else {
		timer.Logger.WithFields(log.Fields{
			"total_rows": cnt,
			"avg_flow":   cnt,
			"State":      "stopped",
		}).Infof("Total duration:, %v rows =%d  SUPER FAST", duration, cnt)

	}
}

// LogFields returns summary data as fieldmap
func (timer *TimerData) LogFields() log.Fields {
	cnt := timer.Index.Get()
	timer.mu.RLock()
	uuid := timer.Uuid
	title := timer.Title
	startTime := timer.StartTimeRun
	timer.mu.RUnlock()

	t1 := time.Now()
	duration := t1.Sub(startTime)
	ds := int64(duration.Seconds())
	avgFlow := cnt
	if ds > 0 {
		avgFlow = cnt / ds
	}

	return log.Fields{
		"uuid":       uuid,
		"title":      title,
		"total_rows": cnt,
		"duration_s": ds,
		"avg_flow_s": avgFlow,
		"start_time": startTime.UTC().Format("2006-01-02T15:04:05-0700"),
		"end_time":   t1.UTC().Format("2006-01-02T15:04:05-0700"),
	}

}

// ShowBatchTime show averages to now
func (timer *TimerData) ShowBatchTime() {

	cnt := timer.Index.Get()
	timer.mu.RLock()
	prevRows := timer.PrevRows
	startTime := timer.StartTimeBatch
	timer.mu.RUnlock()

	diff := cnt - prevRows

	t1 := time.Now()
	duration := t1.Sub(startTime)
	d2 := timer.TotalDuration()

	ds := int64(d2.Seconds())
	dsBatch := int64(duration.Seconds())

	if ds > 0 && dsBatch > 0 {
		msg := fmt.Sprintf("%d rows avg flow %d/s - batch time %v batch size %d batch_flow %d \n", cnt, cnt/ds, duration, diff, diff/dsBatch)
		timer.Logger.WithFields(log.Fields{
			"index":      cnt,
			"total_flow": cnt / ds,
			"batch_time": duration,
			"batch_size": diff,
			"batch_flow": diff / dsBatch,
			"State":      "in_batch",
		}).Info(msg)
	} else {
		timer.Logger.Printf("%d rows - batch time %v \n", cnt, duration)
	}

	timer.mu.Lock()
	timer.PrevRows = cnt
	timer.StartTimeBatch = time.Now()
	timer.mu.Unlock()
}

// Tick increases tick with one
func (timer *TimerData) Tick() {
	cnt := timer.Index.inc()

	if cnt%100000 == 0 {
		timer.ShowBatchTime()
	}
}

// Stop stops the timer
func (timer *TimerData) Stop() time.Time {
	timer.mu.Lock()
	timer.EndTimeRun = time.Now()
	timer.mu.Unlock()
	return timer.EndTimeRun
}

// IncError adds one to number of errors
func (timer *TimerData) IncError() int64 {
	timer.ErrorCount.inc()
	return timer.ErrorCount.Get()
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/gogo/protobuf/types"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"

	"go.pedge.io/proto/rpclog"
	"golang.org/x/net/context"
)

type localBlockAPIServer struct {
	protorpclog.Logger
	dir string
}

func newLocalBlockAPIServer(dir string) (*localBlockAPIServer, error) {
	server := &localBlockAPIServer{
		Logger: protorpclog.NewLogger("pfs.BlockAPIServer.Local"),
		dir:    dir,
	}
	if err := os.MkdirAll(server.blockDir(), 0777); err != nil {
		return nil, err
	}
	if err := os.MkdirAll(server.objectDir(), 0777); err != nil {
		return nil, err
	}
	if err := os.MkdirAll(server.tagDir(), 0777); err != nil {
		return nil, err
	}
	return server, nil
}

func (s *localBlockAPIServer) PutObject(server pfsclient.ObjectAPI_PutObjectServer) (retErr error) {
	func() { s.Log(nil, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(nil, nil, retErr, time.Since(start)) }(time.Now())
	defer drainObjectServer(server)
	hash := newHash()
	tmpPath := filepath.Join(s.objectDir(), uuid.NewWithoutDashes())
	putObjectReader := &putObjectReader{
		server: server,
	}
	r := io.TeeReader(putObjectReader, hash)
	if err := func() error {
		w, err := os.Create(tmpPath)
		if err != nil {
			return err
		}
		defer func() {
			if err := w.Close(); err != nil && retErr == nil {
				retErr = err
			}
		}()
		if _, err := io.Copy(w, r); err != nil {
			return err
		}
		return nil
	}(); err != nil {
		return err
	}
	object := &pfsclient.Object{Hash: hex.EncodeToString(hash.Sum(nil))}
	if err := server.SendAndClose(object); err != nil {
		return err
	}
	objectPath := s.objectPath(object)
	if err := os.Rename(tmpPath, objectPath); err != nil && retErr == nil {
		retErr = err
	}
	if _, err := s.TagObject(server.Context(), &pfsclient.TagObjectRequest{
		Object: object,
		Tags:   putObjectReader.tags,
	}); err != nil {
		return err
	}
	return nil
}

func (s *localBlockAPIServer) GetObject(request *pfsclient.Object, getObjectServer pfsclient.ObjectAPI_GetObjectServer) (retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, nil, retErr, time.Since(start)) }(time.Now())
	file, err := os.Open(s.objectPath(request))
	if err != nil {
		return err
	}
	defer func() {
		if err := file.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	return grpcutil.WriteToStreamingBytesServer(file, getObjectServer)
}

func (s *localBlockAPIServer) GetObjects(request *pfsclient.GetObjectsRequest, getObjectsServer pfsclient.ObjectAPI_GetObjectsServer) (retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, nil, retErr, time.Since(start)) }(time.Now())
	offsetBytes := int64(request.OffsetBytes)
	sizeBytes := int64(request.SizeBytes)
	for _, object := range request.Objects {
		fileInfo, err := os.Stat(s.objectPath(object))
		if err != nil {
			return err
		}
		if fileInfo.Size() < offsetBytes {
			offsetBytes -= fileInfo.Size()
			continue
		}
		if err := func() error {
			file, err := os.Open(s.objectPath(object))
			if err != nil {
				return err
			}
			defer func() {
				if err := file.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()
			if request.SizeBytes == 0 {
				sizeBytes = fileInfo.Size() - offsetBytes
			}
			return grpcutil.WriteToStreamingBytesServer(io.NewSectionReader(file, offsetBytes, sizeBytes), getObjectsServer)
		}(); err != nil {
			return err
		}
		if request.SizeBytes != 0 {
			sizeBytes -= (fileInfo.Size() - offsetBytes)
			if sizeBytes <= 0 {
				break
			}
		}
		offsetBytes = 0
	}
	return nil
}

func (s *localBlockAPIServer) TagObject(ctx context.Context, request *pfsclient.TagObjectRequest) (response *types.Empty, retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, response, retErr, time.Since(start)) }(time.Now())
	objectPath := s.objectPath(request.Object)
	for _, tag := range request.Tags {
		if err := os.Symlink(objectPath, s.tagPath(tag)); err != nil {
			return nil, err
		}
	}
	return &types.Empty{}, nil
}

func (s *localBlockAPIServer) InspectObject(ctx context.Context, request *pfsclient.Object) (response *pfsclient.ObjectInfo, retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, response, retErr, time.Since(start)) }(time.Now())
	fileInfo, err := os.Stat(s.objectPath(request))
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("object: %s not found", request.Hash)
		}
		return nil, err
	}
	return &pfsclient.ObjectInfo{
		Object: request,
		BlockRef: &pfsclient.BlockRef{
			Range: &pfsclient.ByteRange{
				Upper: uint64(fileInfo.Size()),
			},
		},
	}, nil
}

func (s *localBlockAPIServer) CheckObject(ctx context.Context, request *pfsclient.CheckObjectRequest) (response *pfsclient.CheckObjectResponse, retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, response, retErr, time.Since(start)) }(time.Now())
	_, err := os.Stat(s.objectPath(request.Object))
	if err != nil {
		if os.IsNotExist(err) {
			return &pfsclient.CheckObjectResponse{
				Exists: false,
			}, nil
		}
		return nil, err
	}
	return &pfsclient.CheckObjectResponse{
		Exists: true,
	}, nil
}

func (s *localBlockAPIServer) ListObjects(request *pfsclient.ListObjectsRequest, listObjectsServer pfsclient.ObjectAPI_ListObjectsServer) (retErr error) {
	return errors.New("TODO")
}

func (s *localBlockAPIServer) ListTags(request *pfsclient.ListTagsRequest, server pfsclient.ObjectAPI_ListTagsServer) (retErr error) {
	return errors.New("TODO")
}

func (s *localBlockAPIServer) DeleteTags(ctx context.Context, request *pfsclient.DeleteTagsRequest) (response *pfsclient.DeleteTagsResponse, retErr error) {
	return nil, errors.New("TODO")
}

func (s *localBlockAPIServer) DeleteObjects(ctx context.Context, request *pfsclient.DeleteObjectsRequest) (response *pfsclient.DeleteObjectsResponse, retErr error) {
	return nil, errors.New("TODO")
}

func (s *localBlockAPIServer) GetTag(request *pfsclient.Tag, getTagServer pfsclient.ObjectAPI_GetTagServer) (retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, nil, retErr, time.Since(start)) }(time.Now())
	file, err := os.Open(s.tagPath(request))
	if err != nil {
		return err
	}
	defer func() {
		if err := file.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	return grpcutil.WriteToStreamingBytesServer(file, getTagServer)
}

func (s *localBlockAPIServer) InspectTag(ctx context.Context, request *pfsclient.Tag) (response *pfsclient.ObjectInfo, retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, response, retErr, time.Since(start)) }(time.Now())
	objectPath, err := os.Readlink(s.tagPath(request))
	if err != nil {
		return nil, err
	}
	return s.InspectObject(ctx, &pfsclient.Object{Hash: filepath.Base(objectPath)})
}

func (s *localBlockAPIServer) Compact(ctx context.Context, request *types.Empty) (response *types.Empty, retErr error) {
	return &types.Empty{}, nil
}

func (s *localBlockAPIServer) blockDir() string {
	return filepath.Join(s.dir, "block")
}

func (s *localBlockAPIServer) blockPath(block *pfsclient.Block) string {
	return filepath.Join(s.blockDir(), block.Hash)
}

func (s *localBlockAPIServer) objectDir() string {
	return filepath.Join(s.dir, "object")
}

func (s *localBlockAPIServer) objectPath(object *pfsclient.Object) string {
	return filepath.Join(s.objectDir(), object.Hash)
}

func (s *localBlockAPIServer) tagDir() string {
	return filepath.Join(s.dir, "tag")
}

func (s *localBlockAPIServer) tagPath(tag *pfsclient.Tag) string {
	return filepath.Join(s.tagDir(), tag.Name)
}

func (s *localBlockAPIServer) indexDir() string {
	return filepath.Join(s.dir, "index")
}

func (s *localBlockAPIServer) indexPath(prefix string) string {
	return filepath.Join(s.indexDir(), prefix)
}

type putObjectReader struct {
	server pfsclient.ObjectAPI_PutObjectServer
	buffer bytes.Buffer
	tags   []*pfsclient.Tag
}

func (r *putObjectReader) Read(p []byte) (int, error) {
	if r.buffer.Len() == 0 {
		request, err := r.server.Recv()
		if err != nil {
			return 0, err
		}
		// buffer.Write cannot error
		r.buffer.Write(request.Value)
		r.tags = append(r.tags, request.Tags...)
	}
	return r.buffer.Read(p)
}

func drainObjectServer(putObjectServer pfsclient.ObjectAPI_PutObjectServer) {
	for {
		if _, err := putObjectServer.Recv(); err != nil {
			break
		}
	}
}
<commit_msg>Make local block semantics the same as obj's.<commit_after>package server

import (
	"bytes"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/gogo/protobuf/types"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"

	"go.pedge.io/proto/rpclog"
	"golang.org/x/net/context"
)

type localBlockAPIServer struct {
	protorpclog.Logger
	dir string
}

func newLocalBlockAPIServer(dir string) (*localBlockAPIServer, error) {
	server := &localBlockAPIServer{
		Logger: protorpclog.NewLogger("pfs.BlockAPIServer.Local"),
		dir:    dir,
	}
	if err := os.MkdirAll(server.blockDir(), 0777); err != nil {
		return nil, err
	}
	if err := os.MkdirAll(server.objectDir(), 0777); err != nil {
		return nil, err
	}
	if err := os.MkdirAll(server.tagDir(), 0777); err != nil {
		return nil, err
	}
	return server, nil
}

func (s *localBlockAPIServer) PutObject(server pfsclient.ObjectAPI_PutObjectServer) (retErr error) {
	func() { s.Log(nil, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(nil, nil, retErr, time.Since(start)) }(time.Now())
	defer drainObjectServer(server)
	hash := newHash()
	tmpPath := filepath.Join(s.objectDir(), uuid.NewWithoutDashes())
	putObjectReader := &putObjectReader{
		server: server,
	}
	r := io.TeeReader(putObjectReader, hash)
	if err := func() error {
		w, err := os.Create(tmpPath)
		if err != nil {
			return err
		}
		defer func() {
			if err := w.Close(); err != nil && retErr == nil {
				retErr = err
			}
		}()
		if _, err := io.Copy(w, r); err != nil {
			return err
		}
		return nil
	}(); err != nil {
		return err
	}
	object := &pfsclient.Object{Hash: hex.EncodeToString(hash.Sum(nil))}
	if err := server.SendAndClose(object); err != nil {
		return err
	}
	objectPath := s.objectPath(object)
	if err := os.Rename(tmpPath, objectPath); err != nil && retErr == nil {
		retErr = err
	}
	if _, err := s.TagObject(server.Context(), &pfsclient.TagObjectRequest{
		Object: object,
		Tags:   putObjectReader.tags,
	}); err != nil {
		return err
	}
	return nil
}

func (s *localBlockAPIServer) GetObject(request *pfsclient.Object, getObjectServer pfsclient.ObjectAPI_GetObjectServer) (retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, nil, retErr, time.Since(start)) }(time.Now())
	file, err := os.Open(s.objectPath(request))
	if err != nil {
		return err
	}
	defer func() {
		if err := file.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	return grpcutil.WriteToStreamingBytesServer(file, getObjectServer)
}

func (s *localBlockAPIServer) GetObjects(request *pfsclient.GetObjectsRequest, getObjectsServer pfsclient.ObjectAPI_GetObjectsServer) (retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, nil, retErr, time.Since(start)) }(time.Now())
	offsetBytes := int64(request.OffsetBytes)
	sizeBytes := int64(request.SizeBytes)
	for _, object := range request.Objects {
		fileInfo, err := os.Stat(s.objectPath(object))
		if err != nil {
			return err
		}
		if fileInfo.Size() < offsetBytes {
			offsetBytes -= fileInfo.Size()
			continue
		}
		if err := func() error {
			file, err := os.Open(s.objectPath(object))
			if err != nil {
				return err
			}
			defer func() {
				if err := file.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()
			if request.SizeBytes == 0 {
				sizeBytes = fileInfo.Size() - offsetBytes
			}
			return grpcutil.WriteToStreamingBytesServer(io.NewSectionReader(file, offsetBytes, sizeBytes), getObjectsServer)
		}(); err != nil {
			return err
		}
		if request.SizeBytes != 0 {
			sizeBytes -= (fileInfo.Size() - offsetBytes)
			if sizeBytes <= 0 {
				break
			}
		}
		offsetBytes = 0
	}
	return nil
}

func (s *localBlockAPIServer) TagObject(ctx context.Context, request *pfsclient.TagObjectRequest) (response *types.Empty, retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, response, retErr, time.Since(start)) }(time.Now())
	objectPath := s.objectPath(request.Object)
	for _, tag := range request.Tags {
		if err := os.RemoveAll(s.tagPath(tag)); err != nil {
			return nil, err
		}
		if err := os.Symlink(objectPath, s.tagPath(tag)); err != nil {
			return nil, err
		}
	}
	return &types.Empty{}, nil
}

func (s *localBlockAPIServer) InspectObject(ctx context.Context, request *pfsclient.Object) (response *pfsclient.ObjectInfo, retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, response, retErr, time.Since(start)) }(time.Now())
	fileInfo, err := os.Stat(s.objectPath(request))
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("object: %s not found", request.Hash)
		}
		return nil, err
	}
	return &pfsclient.ObjectInfo{
		Object: request,
		BlockRef: &pfsclient.BlockRef{
			Range: &pfsclient.ByteRange{
				Upper: uint64(fileInfo.Size()),
			},
		},
	}, nil
}

func (s *localBlockAPIServer) CheckObject(ctx context.Context, request *pfsclient.CheckObjectRequest) (response *pfsclient.CheckObjectResponse, retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, response, retErr, time.Since(start)) }(time.Now())
	_, err := os.Stat(s.objectPath(request.Object))
	if err != nil {
		if os.IsNotExist(err) {
			return &pfsclient.CheckObjectResponse{
				Exists: false,
			}, nil
		}
		return nil, err
	}
	return &pfsclient.CheckObjectResponse{
		Exists: true,
	}, nil
}

func (s *localBlockAPIServer) ListObjects(request *pfsclient.ListObjectsRequest, listObjectsServer pfsclient.ObjectAPI_ListObjectsServer) (retErr error) {
	return errors.New("TODO")
}

func (s *localBlockAPIServer) ListTags(request *pfsclient.ListTagsRequest, server pfsclient.ObjectAPI_ListTagsServer) (retErr error) {
	return errors.New("TODO")
}

func (s *localBlockAPIServer) DeleteTags(ctx context.Context, request *pfsclient.DeleteTagsRequest) (response *pfsclient.DeleteTagsResponse, retErr error) {
	return nil, errors.New("TODO")
}

func (s *localBlockAPIServer) DeleteObjects(ctx context.Context, request *pfsclient.DeleteObjectsRequest) (response *pfsclient.DeleteObjectsResponse, retErr error) {
	return nil, errors.New("TODO")
}

func (s *localBlockAPIServer) GetTag(request *pfsclient.Tag, getTagServer pfsclient.ObjectAPI_GetTagServer) (retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, nil, retErr, time.Since(start)) }(time.Now())
	file, err := os.Open(s.tagPath(request))
	if err != nil {
		return err
	}
	defer func() {
		if err := file.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	return grpcutil.WriteToStreamingBytesServer(file, getTagServer)
}

func (s *localBlockAPIServer) InspectTag(ctx context.Context, request *pfsclient.Tag) (response *pfsclient.ObjectInfo, retErr error) {
	func() { s.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { s.Log(request, response, retErr, time.Since(start)) }(time.Now())
	objectPath, err := os.Readlink(s.tagPath(request))
	if err != nil {
		return nil, err
	}
	return s.InspectObject(ctx, &pfsclient.Object{Hash: filepath.Base(objectPath)})
}

func (s *localBlockAPIServer) Compact(ctx context.Context, request *types.Empty) (response *types.Empty, retErr error) {
	return &types.Empty{}, nil
}

func (s *localBlockAPIServer) blockDir() string {
	return filepath.Join(s.dir, "block")
}

func (s *localBlockAPIServer) blockPath(block *pfsclient.Block) string {
	return filepath.Join(s.blockDir(), block.Hash)
}

func (s *localBlockAPIServer) objectDir() string {
	return filepath.Join(s.dir, "object")
}

func (s *localBlockAPIServer) objectPath(object *pfsclient.Object) string {
	return filepath.Join(s.objectDir(), object.Hash)
}

func (s *localBlockAPIServer) tagDir() string {
	return filepath.Join(s.dir, "tag")
}

func (s *localBlockAPIServer) tagPath(tag *pfsclient.Tag) string {
	return filepath.Join(s.tagDir(), tag.Name)
}

func (s *localBlockAPIServer) indexDir() string {
	return filepath.Join(s.dir, "index")
}

func (s *localBlockAPIServer) indexPath(prefix string) string {
	return filepath.Join(s.indexDir(), prefix)
}

type putObjectReader struct {
	server pfsclient.ObjectAPI_PutObjectServer
	buffer bytes.Buffer
	tags   []*pfsclient.Tag
}

func (r *putObjectReader) Read(p []byte) (int, error) {
	if r.buffer.Len() == 0 {
		request, err := r.server.Recv()
		if err != nil {
			return 0, err
		}
		// buffer.Write cannot error
		r.buffer.Write(request.Value)
		r.tags = append(r.tags, request.Tags...)
	}
	return r.buffer.Read(p)
}

func drainObjectServer(putObjectServer pfsclient.ObjectAPI_PutObjectServer) {
	for {
		if _, err := putObjectServer.Recv(); err != nil {
			break
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"code.cloudfoundry.org/buildpackapplifecycle"
	"code.cloudfoundry.org/buildpackapplifecycle/buildpackrunner"
	"code.cloudfoundry.org/buildpackapplifecycle/credhub"
	"code.cloudfoundry.org/buildpackapplifecycle/databaseuri"
	"code.cloudfoundry.org/buildpackapplifecycle/platformoptions"
	"code.cloudfoundry.org/goshims/osshim"
)

func main() {
	config := buildpackapplifecycle.NewLifecycleBuilderConfig([]string{}, false, false)

	if err := config.Parse(os.Args[1:len(os.Args)]); err != nil {
		println(err.Error())
		os.Exit(1)
	}

	if err := config.Validate(); err != nil {
		println(err.Error())
		usage()
	}

	if platformOptions, err := platformoptions.Get(os.Getenv("VCAP_PLATFORM_OPTIONS")); err != nil {
		fmt.Fprintf(os.Stderr, "Invalid platform options: %v", err)
		os.Exit(3)
	} else if platformOptions != nil && platformOptions.CredhubURI != "" {
		err = credhub.New(&osshim.OsShim{}).InterpolateServiceRefs(platformOptions.CredhubURI)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to interpolate credhub refs: %v", err)
			os.Exit(4)
		}
	}

	if os.Getenv("VCAP_SERVICES") != "" {
		dbUri := databaseuri.New()
		if creds, err := dbUri.Credentials([]byte(os.Getenv("VCAP_SERVICES"))); err == nil {
			databaseUrl := dbUri.Uri(creds)
			if databaseUrl != "" {
				os.Setenv("DATABASE_URL", databaseUrl)
			}
		}
	}

	runner := buildpackrunner.New(&config)
	defer runner.CleanUp()

	_, err := runner.Run()
	if err != nil {
		println(err.Error())
		os.Exit(buildpackapplifecycle.ExitCodeFromError(err))
	}

	os.Exit(0)
}

func usage() {
	flag.PrintDefaults()
	os.Exit(1)
}
<commit_msg>Builder: Allow clean up to run on successful completion<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"code.cloudfoundry.org/buildpackapplifecycle"
	"code.cloudfoundry.org/buildpackapplifecycle/buildpackrunner"
	"code.cloudfoundry.org/buildpackapplifecycle/credhub"
	"code.cloudfoundry.org/buildpackapplifecycle/databaseuri"
	"code.cloudfoundry.org/buildpackapplifecycle/platformoptions"
	"code.cloudfoundry.org/goshims/osshim"
)

func main() {
	config := buildpackapplifecycle.NewLifecycleBuilderConfig([]string{}, false, false)

	if err := config.Parse(os.Args[1:len(os.Args)]); err != nil {
		println(err.Error())
		os.Exit(1)
	}

	if err := config.Validate(); err != nil {
		println(err.Error())
		usage()
	}

	if platformOptions, err := platformoptions.Get(os.Getenv("VCAP_PLATFORM_OPTIONS")); err != nil {
		fmt.Fprintf(os.Stderr, "Invalid platform options: %v", err)
		os.Exit(3)
	} else if platformOptions != nil && platformOptions.CredhubURI != "" {
		err = credhub.New(&osshim.OsShim{}).InterpolateServiceRefs(platformOptions.CredhubURI)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to interpolate credhub refs: %v", err)
			os.Exit(4)
		}
	}

	if os.Getenv("VCAP_SERVICES") != "" {
		dbUri := databaseuri.New()
		if creds, err := dbUri.Credentials([]byte(os.Getenv("VCAP_SERVICES"))); err == nil {
			databaseUrl := dbUri.Uri(creds)
			if databaseUrl != "" {
				os.Setenv("DATABASE_URL", databaseUrl)
			}
		}
	}

	runner := buildpackrunner.New(&config)
	defer runner.CleanUp()

	_, err := runner.Run()
	if err != nil {
		println(err.Error())
		os.Exit(buildpackapplifecycle.ExitCodeFromError(err))
	}
}

func usage() {
	flag.PrintDefaults()
	os.Exit(1)
}
<|endoftext|>