<commit_before>package fst

import (
	"fmt"
	"github.com/balzaczyy/golucene/core/util"
)

/* Expert: invoked by Builder whenever a suffix is serialized. */
type FreezeTail func([]*UnCompiledNode, int, []int) error

/*
Builds a minimal FST (maps an []int term to an arbitrary output) from
pre-sorted terms with outputs. The FST becomes an FSA if you use
NoOutputs. The FST is written on-the-fly into a compact serialized
format byte array, which can be saved to / loaded from a Directory or
used directly for traversal. The FST is always finite (no cycles).

NOTE: the algorithm is described at
http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.24.3698

FSTs larger than 2.1GB are now possible (as of Lucene 4.2). FSTs
containing more than 2.1B nodes are also now possible, however they
cannot be packed.
*/
type Builder struct {
	dedupHash *NodeHash
	fst       *FST
	NO_OUTPUT interface{}

	// simplistic pruning: we prune node (and all following nodes) if
	// less than this number of terms go through it:
	minSuffixCount1 int

	// better pruning: we prune node (and all following nodes) if the
	// prior node has less than this number of terms go through it:
	minSuffixCount2 int

	doShareNonSingletonNodes bool
	shareMaxTailLength       int

	lastInput *util.IntsRef

	// for packing
	doPackFST               bool
	acceptableOverheadRatio float32

	// current frontier
	frontier []*UnCompiledNode

	_freezeTail FreezeTail
}

/*
NOTE: not many instances of Node or CompiledNode are in memory while
the FST is being built; it's only the current "frontier":
*/
type Node interface {
	isCompiled() bool
}

/* Expert: holds a pending (seen but not yet serialized) Node. */
type UnCompiledNode struct {
	owner      *Builder
	arcs       []*Arc
	output     interface{}
	isFinal    bool
	inputCount int64

	// This node's depth, starting from the automaton root.
	depth int
}

func newUnCompiledNode(owner *Builder, depth int) *UnCompiledNode {
	return &UnCompiledNode{
		owner:  owner,
		arcs:   []*Arc{new(Arc)},
		output: owner.NO_OUTPUT,
		depth:  depth,
	}
}

func (n *UnCompiledNode) isCompiled() bool { return false }

func (n *UnCompiledNode) lastOutput(labelToMatch int) interface{} {
	panic("not implementd yet")
}

func (n *UnCompiledNode) addArc(label int, target Node) {
	panic("not implemented yet")
}

/*
Instantiates an FST/FSA builder with all the possible tuning and
construction tweaks. Read parameter documentation carefully.

...
*/
func NewBuilder(inputType InputType, minSuffixCount1, minSuffixCount2 int,
	doShareSuffix, doShareNonSingletonNodes bool, shareMaxTailLength int,
	outputs Outputs, freezeTail FreezeTail, doPackFST bool,
	acceptableOverheadRatio float32, allowArrayArcs bool, bytesPageBits int) *Builder {

	fst := newFST(inputType, outputs, doPackFST, acceptableOverheadRatio, allowArrayArcs, bytesPageBits)
	f := make([]*UnCompiledNode, 10)
	ans := &Builder{
		minSuffixCount1:          minSuffixCount1,
		minSuffixCount2:          minSuffixCount2,
		_freezeTail:              freezeTail,
		doShareNonSingletonNodes: doShareNonSingletonNodes,
		shareMaxTailLength:       shareMaxTailLength,
		doPackFST:                doPackFST,
		acceptableOverheadRatio:  acceptableOverheadRatio,
		fst:       fst,
		NO_OUTPUT: outputs.NoOutput(),
		frontier:  f,
		lastInput: util.NewEmptyIntsRef(),
	}
	if doShareSuffix {
		ans.dedupHash = newNodeHash(fst, fst.bytes.reverseReaderAllowSingle(false))
	}
	for i, _ := range f {
		f[i] = newUnCompiledNode(ans, i)
	}
	return ans
}

func (b *Builder) freezeTail(prefixLenPlus1 int) error {
	if b._freezeTail != nil {
		panic("not implemented yet")
	}
	panic("not implemented yet")
}

/*
It's OK to add the same input twice in a row with different outputs,
as long as outputs impls the merge method. Note that input is fully
consumed after this method is returned (so caller is free to reuse),
but output is not. So if your outputs are changeable (eg
ByteSequenceOutputs or IntSequenceOutputs) then you cannot reuse
across calls.
*/
func (b *Builder) Add(input *util.IntsRef, output interface{}) error {
	assert2(b.lastInput.Length == 0 || !input.CompareTo(b.lastInput),
		"inputs are added out of order, lastInput=%v vs input=%v",
		b.lastInput, input)

	if input.Length == 0 {
		// empty input: only allowed as first input. We have to special
		// case this becaues the packed FST format cannot represent the
		// empty input since 'finalness' is stored on the incoming arc,
		// not on the node
		b.frontier[0].inputCount++
		b.frontier[0].isFinal = true
		b.fst.setEmptyOutput(output)
		return nil
	}

	// compare shared prefix length
	pos1 := 0
	pos2 := input.Offset
	pos1Stop := b.lastInput.Length
	if input.Length < pos1Stop {
		pos1Stop = input.Length
	}
	for {
		b.frontier[pos1].inputCount++
		if pos1 >= pos1Stop || b.lastInput.Ints[pos1] != input.Ints[pos2] {
			break
		}
		pos1++
		pos2++
	}
	prefixLenPlus1 := pos1 + 1

	if len(b.frontier) < input.Length+1 {
		next := make([]*UnCompiledNode, util.Oversize(input.Length+1, util.NUM_BYTES_OBJECT_REF))
		copy(next, b.frontier)
		for idx := len(b.frontier); idx < len(next); idx++ {
			next[idx] = newUnCompiledNode(b, idx)
		}
		b.frontier = next
	}

	// minimize/compile states from previous input's orphan'd suffix
	err := b.freezeTail(prefixLenPlus1)
	if err != nil {
		return err
	}

	// init tail states for current input
	for idx := prefixLenPlus1; idx <= input.Length; idx++ {
		b.frontier[idx-1].addArc(input.Ints[input.Offset+idx-1], b.frontier[idx])
		b.frontier[idx].inputCount++
	}

	lastNode := b.frontier[input.Length]
	if b.lastInput.Length != input.Length || prefixLenPlus1 != input.Length+1 {
		lastNode.isFinal = true
		lastNode.output = b.NO_OUTPUT
	}

	// push conflicting outputs forward, only as far as needed
	for idx := 1; idx < prefixLenPlus1; idx++ {
		// node := b.frontier[idx]
		parentNode := b.frontier[idx-1]

		lastOutput := parentNode.lastOutput(input.Ints[input.Offset+idx-1])

		var commonOutputPrefix interface{}
		// var wordSuffix interface{}

		if lastOutput != b.NO_OUTPUT {
			panic("not implemented yet")
		} else {
			panic("not implemented yet")
		}

		output = b.fst.outputs.Subtract(output, commonOutputPrefix)
	}

	if b.lastInput.Length == input.Length && prefixLenPlus1 == 1+input.Length {
		// same input more than 1 time in a row, mapping to multiple outputs
		panic("not implemented yet")
	} else {
		panic("not implemented yet")
	}

	// save last input
	b.lastInput.CopyInts(input)
	return nil
}

func assert(ok bool) {
	assert2(ok, "assert fail")
}

func assert2(ok bool, msg string, args ...interface{}) {
	if !ok {
		panic(fmt.Sprintf(msg, args...))
	}
}
<commit_msg>implement Builder.freezeTail()<commit_after>package fst

import (
	"fmt"
	"github.com/balzaczyy/golucene/core/util"
)

/* Expert: invoked by Builder whenever a suffix is serialized. */
type FreezeTail func([]*UnCompiledNode, int, []int) error

/*
Builds a minimal FST (maps an []int term to an arbitrary output) from
pre-sorted terms with outputs. The FST becomes an FSA if you use
NoOutputs. The FST is written on-the-fly into a compact serialized
format byte array, which can be saved to / loaded from a Directory or
used directly for traversal. The FST is always finite (no cycles).

NOTE: the algorithm is described at
http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.24.3698

FSTs larger than 2.1GB are now possible (as of Lucene 4.2). FSTs
containing more than 2.1B nodes are also now possible, however they
cannot be packed.
*/
type Builder struct {
	dedupHash *NodeHash
	fst       *FST
	NO_OUTPUT interface{}

	// simplistic pruning: we prune node (and all following nodes) if
	// less than this number of terms go through it:
	minSuffixCount1 int

	// better pruning: we prune node (and all following nodes) if the
	// prior node has less than this number of terms go through it:
	minSuffixCount2 int

	doShareNonSingletonNodes bool
	shareMaxTailLength       int

	lastInput *util.IntsRef

	// for packing
	doPackFST               bool
	acceptableOverheadRatio float32

	// current frontier
	frontier []*UnCompiledNode

	_freezeTail FreezeTail
}

/*
NOTE: not many instances of Node or CompiledNode are in memory while
the FST is being built; it's only the current "frontier":
*/
type Node interface {
	isCompiled() bool
}

/* Expert: holds a pending (seen but not yet serialized) Node. */
type UnCompiledNode struct {
	owner      *Builder
	arcs       []*Arc
	output     interface{}
	isFinal    bool
	inputCount int64

	// This node's depth, starting from the automaton root.
	depth int
}

func newUnCompiledNode(owner *Builder, depth int) *UnCompiledNode {
	return &UnCompiledNode{
		owner:  owner,
		arcs:   []*Arc{new(Arc)},
		output: owner.NO_OUTPUT,
		depth:  depth,
	}
}

func (n *UnCompiledNode) isCompiled() bool { return false }

func (n *UnCompiledNode) lastOutput(labelToMatch int) interface{} {
	panic("not implementd yet")
}

func (n *UnCompiledNode) addArc(label int, target Node) {
	panic("not implemented yet")
}

/*
Instantiates an FST/FSA builder with all the possible tuning and
construction tweaks. Read parameter documentation carefully.

...
*/
func NewBuilder(inputType InputType, minSuffixCount1, minSuffixCount2 int,
	doShareSuffix, doShareNonSingletonNodes bool, shareMaxTailLength int,
	outputs Outputs, freezeTail FreezeTail, doPackFST bool,
	acceptableOverheadRatio float32, allowArrayArcs bool, bytesPageBits int) *Builder {

	fst := newFST(inputType, outputs, doPackFST, acceptableOverheadRatio, allowArrayArcs, bytesPageBits)
	f := make([]*UnCompiledNode, 10)
	ans := &Builder{
		minSuffixCount1:          minSuffixCount1,
		minSuffixCount2:          minSuffixCount2,
		_freezeTail:              freezeTail,
		doShareNonSingletonNodes: doShareNonSingletonNodes,
		shareMaxTailLength:       shareMaxTailLength,
		doPackFST:                doPackFST,
		acceptableOverheadRatio:  acceptableOverheadRatio,
		fst:       fst,
		NO_OUTPUT: outputs.NoOutput(),
		frontier:  f,
		lastInput: util.NewEmptyIntsRef(),
	}
	if doShareSuffix {
		ans.dedupHash = newNodeHash(fst, fst.bytes.reverseReaderAllowSingle(false))
	}
	for i, _ := range f {
		f[i] = newUnCompiledNode(ans, i)
	}
	return ans
}

func (b *Builder) freezeTail(prefixLenPlus1 int) error {
	if b._freezeTail != nil {
		// Custom plugin:
		return b._freezeTail(b.frontier, prefixLenPlus1, b.lastInput.Value())
	}
	panic("not implemented yet")
}

/*
It's OK to add the same input twice in a row with different outputs,
as long as outputs impls the merge method. Note that input is fully
consumed after this method is returned (so caller is free to reuse),
but output is not. So if your outputs are changeable (eg
ByteSequenceOutputs or IntSequenceOutputs) then you cannot reuse
across calls.
*/
func (b *Builder) Add(input *util.IntsRef, output interface{}) error {
	assert2(b.lastInput.Length == 0 || !input.CompareTo(b.lastInput),
		"inputs are added out of order, lastInput=%v vs input=%v",
		b.lastInput, input)

	if input.Length == 0 {
		// empty input: only allowed as first input. We have to special
		// case this becaues the packed FST format cannot represent the
		// empty input since 'finalness' is stored on the incoming arc,
		// not on the node
		b.frontier[0].inputCount++
		b.frontier[0].isFinal = true
		b.fst.setEmptyOutput(output)
		return nil
	}

	// compare shared prefix length
	pos1 := 0
	pos2 := input.Offset
	pos1Stop := b.lastInput.Length
	if input.Length < pos1Stop {
		pos1Stop = input.Length
	}
	for {
		b.frontier[pos1].inputCount++
		if pos1 >= pos1Stop || b.lastInput.Ints[pos1] != input.Ints[pos2] {
			break
		}
		pos1++
		pos2++
	}
	prefixLenPlus1 := pos1 + 1

	if len(b.frontier) < input.Length+1 {
		next := make([]*UnCompiledNode, util.Oversize(input.Length+1, util.NUM_BYTES_OBJECT_REF))
		copy(next, b.frontier)
		for idx := len(b.frontier); idx < len(next); idx++ {
			next[idx] = newUnCompiledNode(b, idx)
		}
		b.frontier = next
	}

	// minimize/compile states from previous input's orphan'd suffix
	err := b.freezeTail(prefixLenPlus1)
	if err != nil {
		return err
	}

	// init tail states for current input
	for idx := prefixLenPlus1; idx <= input.Length; idx++ {
		b.frontier[idx-1].addArc(input.Ints[input.Offset+idx-1], b.frontier[idx])
		b.frontier[idx].inputCount++
	}

	lastNode := b.frontier[input.Length]
	if b.lastInput.Length != input.Length || prefixLenPlus1 != input.Length+1 {
		lastNode.isFinal = true
		lastNode.output = b.NO_OUTPUT
	}

	// push conflicting outputs forward, only as far as needed
	for idx := 1; idx < prefixLenPlus1; idx++ {
		// node := b.frontier[idx]
		parentNode := b.frontier[idx-1]

		lastOutput := parentNode.lastOutput(input.Ints[input.Offset+idx-1])

		var commonOutputPrefix interface{}
		// var wordSuffix interface{}

		if lastOutput != b.NO_OUTPUT {
			panic("not implemented yet")
		} else {
			panic("not implemented yet")
		}

		output = b.fst.outputs.Subtract(output, commonOutputPrefix)
	}

	if b.lastInput.Length == input.Length && prefixLenPlus1 == 1+input.Length {
		// same input more than 1 time in a row, mapping to multiple outputs
		panic("not implemented yet")
	} else {
		panic("not implemented yet")
	}

	// save last input
	b.lastInput.CopyInts(input)
	return nil
}

func assert(ok bool) {
	assert2(ok, "assert fail")
}

func assert2(ok bool, msg string, args ...interface{}) {
	if !ok {
		panic(fmt.Sprintf(msg, args...))
	}
}
<|endoftext|>


<commit_before>package httpfunction

import (
	"net/http"
	"log"
	"dao"
	"reflect"
)
//默认的上传方法
func Upload_default(w http.ResponseWriter, r *http.Request, resp *Datatablesdata)(err error) {
	tableName := r.URL.Query().Get(URLTableName)
	resp.Upload.Id, err = Getpostfile(r,tableName)
	if err != nil {
		log.Println("failed to action upload in func ViewHandle, err: ", err.Error())
		return
	}
	err = Common_HandleFile(tableName, nil, FILES_NEEDED_ALL, resp)
	if err != nil {
		log.Println("failed to handle file in upload, err: ", err.Error())
		return
	}
	return
}

//默认的创建方法
func Create_default(w http.ResponseWriter, r *http.Request, resp *Datatablesdata) (err error) {
	tableName, id, err := Get_tableName_id(r)
	res, err := Createdatatablesline(r, tableName, id)
	if err != nil {
		log.Println("failed to create a new line, err: ", err.Error())
		return
	}
	err = Common_create_edit(tableName, res, resp)
	if err != nil {
		log.Println("failed to add data to Datatablesdata, err: ", err.Error())
		return
	}
	err = Common_HandleFile(tableName, res, FILES_NEEDED_ONE, resp)
	if err != nil {
		log.Println("failed to handle file in create, err: ", err.Error())
		return
	}
	return
}

//默认的编辑方法
func Edit_default(w http.ResponseWriter, r *http.Request, resp *Datatablesdata) (err error)  {
	tableName, id ,err := Get_tableName_id(r)
	res, err := Editdatatablesline(r, tableName, id)
	if err != nil {
		log.Println("failed to edit line, err: ", err.Error())
		return
	}
	err = Common_create_edit(tableName, res, resp)
	if err != nil {
		log.Println("failed to add data to Datatablesdata, err: ", err.Error())
		return
	}
	err = Common_HandleFile(tableName, res, FILES_NEEDED_ONE, resp)
	if err != nil {
		log.Println("failed to handle file in edit, err: ", err.Error())
		return
	}
	return
}

//默认的删除方法
func Remove_default(w http.ResponseWriter, r *http.Request, resp *Datatablesdata) (err error) {
	tableName, id ,err := Get_tableName_id(r)
	resp.Data, err = dao.GetDataStructSilce(tableName)
	if err != nil {
		log.Println("falied to get datastruct silce, err: ", err.Error())
		return
	}
	var IdSlice []interface{}
	for i := range id {
		IdSlice = append(IdSlice, id[i])
	}
	err = dao.GetDataByIdSlice(tableName, IdSlice, resp.Data)
	if err != nil{
		log.Println("failed to get data by id array, err: ", err.Error())
		return
	}
	err = Deldatatablesline(r, tableName, id)
	if err != nil {
		log.Println("failed to remove line, err: ",err.Error())
		return
	}

	return
}

//默认的GET方法
func GET_default(w http.ResponseWriter, r *http.Request, resp *Datatablesdata) (err error){
	tableName := r.URL.Query().Get(URLTableName)
	resp.Data, err = dao.GetDataStructSilce(tableName)
	if err != nil {
		log.Println("falied to get datastruct silce, err: ", err.Error())
		return
	}
	err = dao.GetAll(tableName, resp.Data)
	if err != nil {
		log.Println("failed to get all data, err: ",err.Error())
		return
	}
	if reflect.ValueOf(resp.Data).Elem().Len() == 0 {
		log.Println("response is empty")
		resp.Data = make([]interface{}, 0)
	}
	err = Common_HandleFile(tableName, nil, FILES_NEEDED_ALL, resp)
	if err != nil {
		log.Println("failed to handle file in GET, err: ", err.Error())
		return
	}

	return
}

//用于获取请求的表名和主键
func Get_tableName_id(r *http.Request)(tableName string, id []string, err error){
	tableName = r.URL.Query().Get(URLTableName)
	id, err = GetDataTableId(r, tableName)
	if err != nil {
		log.Println("failed to get post data Id, err: ", err.Error())
		return
	}
	return
}

//用于创建和编辑之后，加载要回应的数据
func Common_create_edit(tableName string, res []dao.DataTablesDao, resp *Datatablesdata)(err error) {

	resp.Data, err = dao.GetDataStructSilce(tableName)
	if err != nil{
		log.Println("failed to get data struct slice, err: ", err.Error())
		return
	}
	var ids []interface{}
	for i := range res {
		ids = append(ids, res[i].GetId())
	}
	err = dao.GetDataByIdSlice(tableName, ids, resp.Data)
	if err != nil{
		log.Println("failed to get data by id array, err: ", err.Error())
		return
	}
	return
}

//用于除删除外的所有请求之后， 判断是否要加载文件信息，是则在函数内加载
func Common_HandleFile(tableName string, res []dao.DataTablesDao, flag int, resp *Datatablesdata)(err error){
	useToJudge, err := dao.GetDataStruct(tableName)
	if err != nil {
		log.Println("failed to get data struct, err: ", err.Error())
		return
	}
	if JudgeDataStructFileId(useToJudge) {
		err = HandleFilesData(tableName, resp, res, flag)
		if err != nil {
			log.Println("failed to HandleFilesData, err: ", err.Error())
			return
		}
	}
	return
}
<commit_msg>little change<commit_after>package httpfunction

import (
	"net/http"
	"log"
	"dao"
	"reflect"
)
//默认的上传方法
func Upload_default(w http.ResponseWriter, r *http.Request, resp *Datatablesdata)(err error) {
	tableName := r.URL.Query().Get(URLTableName)
	resp.Upload.Id, err = Getpostfile(r,tableName)
	if err != nil {
		log.Println("failed to action upload in func ViewHandle, err: ", err.Error())
		return
	}
	err = Common_HandleFile(tableName, nil, FILES_NEEDED_ALL, resp)
	if err != nil {
		log.Println("failed to handle file in upload, err: ", err.Error())
		return
	}
	return
}

//默认的创建方法
func Create_default(w http.ResponseWriter, r *http.Request, resp *Datatablesdata) (err error) {
	tableName, id, err := Get_tableName_id(r)
	res, err := Createdatatablesline(r, tableName, id)
	if err != nil {
		log.Println("failed to create a new line, err: ", err.Error())
		return
	}
	err = Common_create_edit(tableName, res, resp)
	if err != nil {
		log.Println("failed to add data to Datatablesdata, err: ", err.Error())
		return
	}
	err = Common_HandleFile(tableName, res, FILES_NEEDED_ONE, resp)
	if err != nil {
		log.Println("failed to handle file in create, err: ", err.Error())
		return
	}
	return
}

//默认的编辑方法
func Edit_default(w http.ResponseWriter, r *http.Request, resp *Datatablesdata) (err error)  {
	tableName, id ,err := Get_tableName_id(r)
	res, err := Editdatatablesline(r, tableName, id)
	if err != nil {
		log.Println("failed to edit line, err: ", err.Error())
		return
	}
	err = Common_create_edit(tableName, res, resp)
	if err != nil {
		log.Println("failed to add data to Datatablesdata, err: ", err.Error())
		return
	}
	err = Common_HandleFile(tableName, res, FILES_NEEDED_ONE, resp)
	if err != nil {
		log.Println("failed to handle file in edit, err: ", err.Error())
		return
	}
	return
}

//默认的删除方法
func Remove_default(w http.ResponseWriter, r *http.Request, resp *Datatablesdata) (err error) {
	tableName, id ,err := Get_tableName_id(r)
	if err != nil {
		log.Println("failed to get excute Get_tableName_id, err: ", err.Error())
		return
	}
	resp.Data, err = dao.GetDataStructSilce(tableName)
	if err != nil {
		log.Println("falied to get datastruct silce, err: ", err.Error())
		return
	}
	var IdSlice []interface{}
	for i := range id {
		IdSlice = append(IdSlice, id[i])
	}
	err = dao.GetDataByIdSlice(tableName, IdSlice, resp.Data)
	if err != nil{
		log.Println("failed to get data by id array, err: ", err.Error())
		return
	}
	err = Deldatatablesline(r, tableName, id)
	if err != nil {
		log.Println("failed to remove line, err: ",err.Error())
		return
	}
	return
}

//默认的GET方法
func GET_default(w http.ResponseWriter, r *http.Request, resp *Datatablesdata) (err error){
	tableName := r.URL.Query().Get(URLTableName)
	resp.Data, err = dao.GetDataStructSilce(tableName)
	if err != nil {
		log.Println("falied to get datastruct silce, err: ", err.Error())
		return
	}
	err = dao.GetAll(tableName, resp.Data)
	if err != nil {
		log.Println("failed to get all data, err: ",err.Error())
		return
	}
	if reflect.ValueOf(resp.Data).Elem().Len() == 0 {
		log.Println("response is empty")
		resp.Data = make([]interface{}, 0)
	}
	err = Common_HandleFile(tableName, nil, FILES_NEEDED_ALL, resp)
	if err != nil {
		log.Println("failed to handle file in GET, err: ", err.Error())
		return
	}

	return
}

//用于获取请求的表名和主键
func Get_tableName_id(r *http.Request)(tableName string, id []string, err error){
	tableName = r.URL.Query().Get(URLTableName)
	id, err = GetDataTableId(r, tableName)
	if err != nil {
		log.Println("failed to get post data Id, err: ", err.Error())
		return
	}
	return
}

//用于创建和编辑之后，加载要回应的数据
func Common_create_edit(tableName string, res []dao.DataTablesDao, resp *Datatablesdata)(err error) {

	resp.Data, err = dao.GetDataStructSilce(tableName)
	if err != nil{
		log.Println("failed to get data struct slice, err: ", err.Error())
		return
	}
	var ids []interface{}
	for i := range res {
		ids = append(ids, res[i].GetId())
	}
	err = dao.GetDataByIdSlice(tableName, ids, resp.Data)
	if err != nil{
		log.Println("failed to get data by id array, err: ", err.Error())
		return
	}
	return
}

//用于除删除外的所有请求之后， 判断是否要加载文件信息，是则在函数内加载
func Common_HandleFile(tableName string, res []dao.DataTablesDao, flag int, resp *Datatablesdata)(err error){
	useToJudge, err := dao.GetDataStruct(tableName)
	if err != nil {
		log.Println("failed to get data struct, err: ", err.Error())
		return
	}
	if JudgeDataStructFileId(useToJudge) {
		err = HandleFilesData(tableName, resp, res, flag)
		if err != nil {
			log.Println("failed to HandleFilesData, err: ", err.Error())
			return
		}
	}
	return
}
<|endoftext|>


<commit_before>package smtpd

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net"
	"testing"
	"time"
)

// Create test server to run commands against.
// Sleep to give ListenAndServe time to finishing listening before creating clients.
// This seems to only be necessary since Go 1.5.
func init() {
	server := &Server{Addr: "127.0.0.1:52525", Handler: nil}
	go server.ListenAndServe()
	time.Sleep(1 * time.Millisecond)
}

// Create a client to run commands with. Parse the banner for 220 response.
func newConn(t *testing.T) net.Conn {
	conn, err := net.Dial("tcp", "127.0.0.1:52525")
	if err != nil {
		t.Fatalf("Failed to connect to test server: %v", err)
	}
	banner, readErr := bufio.NewReader(conn).ReadString('\n')
	if readErr != nil {
		t.Fatalf("Failed to read banner from test server: %v", readErr)
	}
	if banner[0:3] != "220" {
		t.Fatalf("Read incorrect banner from test server: %v", banner)
	}
	return conn
}

// Send a command and verify the 3 digit code from the response.
func cmdCode(t *testing.T, conn net.Conn, cmd string, code string) {
	fmt.Fprintf(conn, "%s\r\n", cmd)
	resp, err := bufio.NewReader(conn).ReadString('\n')
	if err != nil {
		t.Fatalf("Failed to read response from test server: %v", err)
	}
	if resp[0:3] != code {
		t.Errorf("Command \"%s\" response code is %s, want %s", cmd, resp[0:3], code)
	}
}

// Simple tests: connect, send command, then send QUIT.
// RFC 2821 section 4.1.4 specifies that these commands do not require a prior EHLO,
// only that clients should send one, so test without EHLO.
func TestSimpleCommands(t *testing.T) {
	tests := []struct {
		cmd  string
		code string
	}{
		{"NOOP", "250"},
		{"RSET", "250"},
		{"HELP", "502"},
		{"VRFY", "502"},
		{"EXPN", "502"},
		{"TEST", "500"}, // Unsupported command
		{"", "500"},     // Blank command
	}

	for _, tt := range tests {
		conn := newConn(t)
		cmdCode(t, conn, tt.cmd, tt.code)
		cmdCode(t, conn, "QUIT", "221")
		conn.Close()
	}
}

func TestCmdHELO(t *testing.T) {
	// Send HELO, expect greeting.
	cmdCode(t, newConn(t), "HELO host.example.com", "250")
}

func TestCmdEHLO(t *testing.T) {
	conn := newConn(t)

	// Send EHLO, expect greeting.
	cmdCode(t, conn, "EHLO host.example.com", "250")

	// Verify that EHLO resets the current transaction state like RSET.
	// See RFC 2821 section 4.1.4 for more detail.
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")
	cmdCode(t, conn, "RCPT TO:<recipient@example.com>", "250")
	cmdCode(t, conn, "EHLO host.example.com", "250")
	cmdCode(t, conn, "DATA", "503")

	cmdCode(t, conn, "QUIT", "221")
	conn.Close()
}

func TestCmdRSET(t *testing.T) {
	conn := newConn(t)
	cmdCode(t, conn, "EHLO host.example.com", "250")

	// Verify that RSET clears the current transaction state.
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")
	cmdCode(t, conn, "RCPT TO:<recipient@example.com>", "250")
	cmdCode(t, conn, "RSET", "250")
	cmdCode(t, conn, "DATA", "503")

	cmdCode(t, conn, "QUIT", "221")
	conn.Close()
}

func TestCmdMAIL(t *testing.T) {
	conn := newConn(t)
	cmdCode(t, conn, "EHLO host.example.com", "250")

	// MAIL with no FROM arg should return 501 syntax error
	cmdCode(t, conn, "MAIL", "501")
	// MAIL with empty FROM arg should return 501 syntax error
	cmdCode(t, conn, "MAIL FROM:", "501")
	// MAIL with DSN-style FROM arg should return 250 Ok
	cmdCode(t, conn, "MAIL FROM:<>", "250")
	// MAIL with valid FROM arg should return 250 Ok
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")

	cmdCode(t, conn, "QUIT", "221")
	conn.Close()
}

func TestCmdRCPT(t *testing.T) {
	conn := newConn(t)
	cmdCode(t, conn, "EHLO host.example.com", "250")

	// RCPT without prior MAIL should return 503 bad sequence
	cmdCode(t, conn, "RCPT", "503")

	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")

	// RCPT with no TO arg should return 501 syntax error
	cmdCode(t, conn, "RCPT", "501")

	// RCPT with empty TO arg should return 501 syntax error
	cmdCode(t, conn, "RCPT TO:", "501")

	// RCPT with valid TO arg should return 250 Ok
	cmdCode(t, conn, "RCPT TO:<recipient@example.com>", "250")

	// Up to 100 valid recipients should return 250 Ok
	for i := 2; i < 101; i++ {
		cmdCode(t, conn, fmt.Sprintf("RCPT TO:<recipient%v@example.com>", i), "250")
	}

	// 101st valid recipient with valid TO arg should return 452 too many recipients
	cmdCode(t, conn, "RCPT TO:<recipient101@example.com>", "452")

	cmdCode(t, conn, "QUIT", "221")
	conn.Close()
}

func TestCmdDATA(t *testing.T) {
	conn := newConn(t)
	cmdCode(t, conn, "EHLO host.example.com", "250")

	// DATA without prior MAIL & RCPT should return 503 bad sequence
	cmdCode(t, conn, "DATA", "503")
	cmdCode(t, conn, "RSET", "250")

	// DATA without prior RCPT should return 503 bad sequence
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")
	cmdCode(t, conn, "DATA", "503")
	cmdCode(t, conn, "RSET", "250")

	// Test a full mail transaction.
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")
	cmdCode(t, conn, "RCPT TO:<recipient@example.com>", "250")
	cmdCode(t, conn, "DATA", "354")
	cmdCode(t, conn, "Test message.\r\n.", "250")

	// Test a full mail transaction with a bad last recipient.
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")
	cmdCode(t, conn, "RCPT TO:<recipient@example.com>", "250")
	cmdCode(t, conn, "RCPT TO:", "501")
	cmdCode(t, conn, "DATA", "354")
	cmdCode(t, conn, "Test message.\r\n.", "250")

	cmdCode(t, conn, "QUIT", "221")
	conn.Close()
}

func TestMakeHeaders(t *testing.T) {
	now := time.Now().Format("Mon, _2 Jan 2006 15:04:05 -0700 (MST)")
	valid := "Received: from clientName (clientHost [clientIP])\r\n" +
		"        by serverName (smtpd) with SMTP\r\n" +
		"        for <recipient@example.com>; " +
		fmt.Sprintf("%s\r\n", now)

	srv := &Server{"", nil, "smtpd", "serverName"}
	s := &session{srv, nil, nil, nil, "clientIP", "clientHost", "clientName"}
	headers := s.makeHeaders([]string{"recipient@example.com"})
	if string(headers) != valid {
		t.Errorf("makeHeaders() returned\n%v, want\n%v", string(headers), valid)
	}
}

// Test parsing of commands into verbs and arguments.
func TestParseLine(t *testing.T) {
	tests := []struct {
		line string
		verb string
		args string
	}{
		{"EHLO host.example.com", "EHLO", "host.example.com"},
		{"MAIL FROM:<sender@example.com>", "MAIL", "FROM:<sender@example.com>"},
		{"RCPT TO:<recipient@example.com>", "RCPT", "TO:<recipient@example.com>"},
		{"QUIT", "QUIT", ""},
	}
	s := &session{}
	for _, tt := range tests {
		verb, args := s.parseLine(tt.line)
		if verb != tt.verb || args != tt.args {
			t.Errorf("ParseLine(%v) returned %v, %v, want %v, %v", tt.line, verb, args, tt.verb, tt.args)
		}
	}
}

// Test reading of complete lines from the socket.
func TestReadLine(t *testing.T) {
	var buf bytes.Buffer
	s := &session{}
	s.br = bufio.NewReader(&buf)

	// Ensure readLine() returns an EOF error on an empty buffer.
	_, err := s.readLine()
	if err != io.EOF {
		t.Errorf("readLine() on empty buffer returned err: %v, want EOF", err)
	}

	// Ensure trailing <CRLF> is stripped.
	line := "FOO BAR BAZ\r\n"
	cmd := "FOO BAR BAZ"
	buf.Write([]byte(line))
	output, err := s.readLine()
	if err != nil {
		t.Errorf("readLine(%v) returned err: %v", line, err)
	} else if output != cmd {
		t.Errorf("readLine(%v) returned %v, want %v", line, output, cmd)
	}
}

// Test reading of message data, including dot stuffing (see RFC 5321 section 4.5.2).
func TestReadData(t *testing.T) {
	tests := []struct {
		lines string
		data  string
	}{
		// Single line message.
		{"Test message.\r\n.\r\n", "Test message.\r\n"},

		// Single line message with leading period removed.
		{".Test message.\r\n.\r\n", "Test message.\r\n"},

		// Multiple line message.
		{"Line 1.\r\nLine 2.\r\nLine 3.\r\n.\r\n", "Line 1.\r\nLine 2.\r\nLine 3.\r\n"},

		// Multiple line message with leading period removed.
		{"Line 1.\r\n.Line 2.\r\nLine 3.\r\n.\r\n", "Line 1.\r\nLine 2.\r\nLine 3.\r\n"},

		// Multiple line message with one leading period removed.
		{"Line 1.\r\n..Line 2.\r\nLine 3.\r\n.\r\n", "Line 1.\r\n.Line 2.\r\nLine 3.\r\n"},
	}
	var buf bytes.Buffer
	s := &session{}
	s.br = bufio.NewReader(&buf)

	// Ensure readData() returns an EOF error on an empty buffer.
	_, err := s.readData()
	if err != io.EOF {
		t.Errorf("readData() on empty buffer returned err: %v, want EOF", err)
	}

	for _, tt := range tests {
		buf.Write([]byte(tt.lines))
		data, err := s.readData()
		if err != nil {
			t.Errorf("readData(%v) returned err: %v", tt.lines, err)
		} else if string(data) != tt.data {
			t.Errorf("readData(%v) returned %v, want %v", tt.lines, string(data), tt.data)
		}
	}
}
<commit_msg>Add a benchmark for simple mail processing.<commit_after>package smtpd

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net"
	"testing"
	"time"
)

// Create test server to run commands against.
// Sleep to give ListenAndServe time to finishing listening before creating clients.
// This seems to only be necessary since Go 1.5.
func init() {
	server := &Server{Addr: "127.0.0.1:52525", Handler: nil}
	go server.ListenAndServe()
	time.Sleep(1 * time.Millisecond)
}

// Create a client to run commands with. Parse the banner for 220 response.
func newConn(t *testing.T) net.Conn {
	conn, err := net.Dial("tcp", "127.0.0.1:52525")
	if err != nil {
		t.Fatalf("Failed to connect to test server: %v", err)
	}
	banner, readErr := bufio.NewReader(conn).ReadString('\n')
	if readErr != nil {
		t.Fatalf("Failed to read banner from test server: %v", readErr)
	}
	if banner[0:3] != "220" {
		t.Fatalf("Read incorrect banner from test server: %v", banner)
	}
	return conn
}

// Send a command and verify the 3 digit code from the response.
func cmdCode(t *testing.T, conn net.Conn, cmd string, code string) {
	fmt.Fprintf(conn, "%s\r\n", cmd)
	resp, err := bufio.NewReader(conn).ReadString('\n')
	if err != nil {
		t.Fatalf("Failed to read response from test server: %v", err)
	}
	if resp[0:3] != code {
		t.Errorf("Command \"%s\" response code is %s, want %s", cmd, resp[0:3], code)
	}
}

// Simple tests: connect, send command, then send QUIT.
// RFC 2821 section 4.1.4 specifies that these commands do not require a prior EHLO,
// only that clients should send one, so test without EHLO.
func TestSimpleCommands(t *testing.T) {
	tests := []struct {
		cmd  string
		code string
	}{
		{"NOOP", "250"},
		{"RSET", "250"},
		{"HELP", "502"},
		{"VRFY", "502"},
		{"EXPN", "502"},
		{"TEST", "500"}, // Unsupported command
		{"", "500"},     // Blank command
	}

	for _, tt := range tests {
		conn := newConn(t)
		cmdCode(t, conn, tt.cmd, tt.code)
		cmdCode(t, conn, "QUIT", "221")
		conn.Close()
	}
}

func TestCmdHELO(t *testing.T) {
	// Send HELO, expect greeting.
	cmdCode(t, newConn(t), "HELO host.example.com", "250")
}

func TestCmdEHLO(t *testing.T) {
	conn := newConn(t)

	// Send EHLO, expect greeting.
	cmdCode(t, conn, "EHLO host.example.com", "250")

	// Verify that EHLO resets the current transaction state like RSET.
	// See RFC 2821 section 4.1.4 for more detail.
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")
	cmdCode(t, conn, "RCPT TO:<recipient@example.com>", "250")
	cmdCode(t, conn, "EHLO host.example.com", "250")
	cmdCode(t, conn, "DATA", "503")

	cmdCode(t, conn, "QUIT", "221")
	conn.Close()
}

func TestCmdRSET(t *testing.T) {
	conn := newConn(t)
	cmdCode(t, conn, "EHLO host.example.com", "250")

	// Verify that RSET clears the current transaction state.
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")
	cmdCode(t, conn, "RCPT TO:<recipient@example.com>", "250")
	cmdCode(t, conn, "RSET", "250")
	cmdCode(t, conn, "DATA", "503")

	cmdCode(t, conn, "QUIT", "221")
	conn.Close()
}

func TestCmdMAIL(t *testing.T) {
	conn := newConn(t)
	cmdCode(t, conn, "EHLO host.example.com", "250")

	// MAIL with no FROM arg should return 501 syntax error
	cmdCode(t, conn, "MAIL", "501")
	// MAIL with empty FROM arg should return 501 syntax error
	cmdCode(t, conn, "MAIL FROM:", "501")
	// MAIL with DSN-style FROM arg should return 250 Ok
	cmdCode(t, conn, "MAIL FROM:<>", "250")
	// MAIL with valid FROM arg should return 250 Ok
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")

	cmdCode(t, conn, "QUIT", "221")
	conn.Close()
}

func TestCmdRCPT(t *testing.T) {
	conn := newConn(t)
	cmdCode(t, conn, "EHLO host.example.com", "250")

	// RCPT without prior MAIL should return 503 bad sequence
	cmdCode(t, conn, "RCPT", "503")

	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")

	// RCPT with no TO arg should return 501 syntax error
	cmdCode(t, conn, "RCPT", "501")

	// RCPT with empty TO arg should return 501 syntax error
	cmdCode(t, conn, "RCPT TO:", "501")

	// RCPT with valid TO arg should return 250 Ok
	cmdCode(t, conn, "RCPT TO:<recipient@example.com>", "250")

	// Up to 100 valid recipients should return 250 Ok
	for i := 2; i < 101; i++ {
		cmdCode(t, conn, fmt.Sprintf("RCPT TO:<recipient%v@example.com>", i), "250")
	}

	// 101st valid recipient with valid TO arg should return 452 too many recipients
	cmdCode(t, conn, "RCPT TO:<recipient101@example.com>", "452")

	cmdCode(t, conn, "QUIT", "221")
	conn.Close()
}

func TestCmdDATA(t *testing.T) {
	conn := newConn(t)
	cmdCode(t, conn, "EHLO host.example.com", "250")

	// DATA without prior MAIL & RCPT should return 503 bad sequence
	cmdCode(t, conn, "DATA", "503")
	cmdCode(t, conn, "RSET", "250")

	// DATA without prior RCPT should return 503 bad sequence
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")
	cmdCode(t, conn, "DATA", "503")
	cmdCode(t, conn, "RSET", "250")

	// Test a full mail transaction.
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")
	cmdCode(t, conn, "RCPT TO:<recipient@example.com>", "250")
	cmdCode(t, conn, "DATA", "354")
	cmdCode(t, conn, "Test message.\r\n.", "250")

	// Test a full mail transaction with a bad last recipient.
	cmdCode(t, conn, "MAIL FROM:<sender@example.com>", "250")
	cmdCode(t, conn, "RCPT TO:<recipient@example.com>", "250")
	cmdCode(t, conn, "RCPT TO:", "501")
	cmdCode(t, conn, "DATA", "354")
	cmdCode(t, conn, "Test message.\r\n.", "250")

	cmdCode(t, conn, "QUIT", "221")
	conn.Close()
}

func TestMakeHeaders(t *testing.T) {
	now := time.Now().Format("Mon, _2 Jan 2006 15:04:05 -0700 (MST)")
	valid := "Received: from clientName (clientHost [clientIP])\r\n" +
		"        by serverName (smtpd) with SMTP\r\n" +
		"        for <recipient@example.com>; " +
		fmt.Sprintf("%s\r\n", now)

	srv := &Server{"", nil, "smtpd", "serverName"}
	s := &session{srv, nil, nil, nil, "clientIP", "clientHost", "clientName"}
	headers := s.makeHeaders([]string{"recipient@example.com"})
	if string(headers) != valid {
		t.Errorf("makeHeaders() returned\n%v, want\n%v", string(headers), valid)
	}
}

// Test parsing of commands into verbs and arguments.
func TestParseLine(t *testing.T) {
	tests := []struct {
		line string
		verb string
		args string
	}{
		{"EHLO host.example.com", "EHLO", "host.example.com"},
		{"MAIL FROM:<sender@example.com>", "MAIL", "FROM:<sender@example.com>"},
		{"RCPT TO:<recipient@example.com>", "RCPT", "TO:<recipient@example.com>"},
		{"QUIT", "QUIT", ""},
	}
	s := &session{}
	for _, tt := range tests {
		verb, args := s.parseLine(tt.line)
		if verb != tt.verb || args != tt.args {
			t.Errorf("ParseLine(%v) returned %v, %v, want %v, %v", tt.line, verb, args, tt.verb, tt.args)
		}
	}
}

// Test reading of complete lines from the socket.
func TestReadLine(t *testing.T) {
	var buf bytes.Buffer
	s := &session{}
	s.br = bufio.NewReader(&buf)

	// Ensure readLine() returns an EOF error on an empty buffer.
	_, err := s.readLine()
	if err != io.EOF {
		t.Errorf("readLine() on empty buffer returned err: %v, want EOF", err)
	}

	// Ensure trailing <CRLF> is stripped.
	line := "FOO BAR BAZ\r\n"
	cmd := "FOO BAR BAZ"
	buf.Write([]byte(line))
	output, err := s.readLine()
	if err != nil {
		t.Errorf("readLine(%v) returned err: %v", line, err)
	} else if output != cmd {
		t.Errorf("readLine(%v) returned %v, want %v", line, output, cmd)
	}
}

// Test reading of message data, including dot stuffing (see RFC 5321 section 4.5.2).
func TestReadData(t *testing.T) {
	tests := []struct {
		lines string
		data  string
	}{
		// Single line message.
		{"Test message.\r\n.\r\n", "Test message.\r\n"},

		// Single line message with leading period removed.
		{".Test message.\r\n.\r\n", "Test message.\r\n"},

		// Multiple line message.
		{"Line 1.\r\nLine 2.\r\nLine 3.\r\n.\r\n", "Line 1.\r\nLine 2.\r\nLine 3.\r\n"},

		// Multiple line message with leading period removed.
		{"Line 1.\r\n.Line 2.\r\nLine 3.\r\n.\r\n", "Line 1.\r\nLine 2.\r\nLine 3.\r\n"},

		// Multiple line message with one leading period removed.
		{"Line 1.\r\n..Line 2.\r\nLine 3.\r\n.\r\n", "Line 1.\r\n.Line 2.\r\nLine 3.\r\n"},
	}
	var buf bytes.Buffer
	s := &session{}
	s.br = bufio.NewReader(&buf)

	// Ensure readData() returns an EOF error on an empty buffer.
	_, err := s.readData()
	if err != io.EOF {
		t.Errorf("readData() on empty buffer returned err: %v, want EOF", err)
	}

	for _, tt := range tests {
		buf.Write([]byte(tt.lines))
		data, err := s.readData()
		if err != nil {
			t.Errorf("readData(%v) returned err: %v", tt.lines, err)
		} else if string(data) != tt.data {
			t.Errorf("readData(%v) returned %v, want %v", tt.lines, string(data), tt.data)
		}
	}
}

// Benchmark the mail handling without the network stack introducing latency.
func BenchmarkReceive(b *testing.B) {
	clientConn, serverConn := net.Pipe()

	server := &Server{}
	session, _ := server.newSession(serverConn)
	go session.serve()

	reader := bufio.NewReader(clientConn)
	_, _ = reader.ReadString('\n') // Read greeting message first.

	b.ResetTimer()

	// Benchmark a full mail transaction.
	for i := 0; i < b.N; i++ {
		fmt.Fprintf(clientConn, "%s\r\n", "EHLO host.example.com")
		_, _ = reader.ReadString('\n')
		fmt.Fprintf(clientConn, "%s\r\n", "MAIL FROM:<sender@example.com>")
		_, _ = reader.ReadString('\n')
		fmt.Fprintf(clientConn, "%s\r\n", "RCPT TO:<recipient@example.com>")
		_, _ = reader.ReadString('\n')
		fmt.Fprintf(clientConn, "%s\r\n", "DATA")
		_, _ = reader.ReadString('\n')
		fmt.Fprintf(clientConn, "%s\r\n", "Test message.\r\n.")
		_, _ = reader.ReadString('\n')
		fmt.Fprintf(clientConn, "%s\r\n", "QUIT")
		_, _ = reader.ReadString('\n')
	}
}
<|endoftext|>


<commit_before>package main

import (
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/cmd"
	"os"
)

// When we import an environment provider implementation
// here, it will register itself with environs, and hence
// be available to the juju command.
import (
	_ "launchpad.net/juju-core/environs/ec2"
	_ "launchpad.net/juju-core/environs/openstack"
)

var jujuDoc = `
juju provides easy, intelligent service orchestration on top of environments
such as OpenStack, Amazon AWS, or bare metal.

https://juju.ubuntu.com/
`

// Main registers subcommands for the juju executable, and hands over control
// to the cmd package. This function is not redundant with main, because it
// provides an entry point for testing with arbitrary command line arguments.
func Main(args []string) {
	juju := &cmd.SuperCommand{Name: "juju", Doc: jujuDoc, Log: &cmd.Log{}}

	// Register creation commands.
	juju.Register(&BootstrapCommand{})
	juju.Register(&DeployCommand{})
	juju.Register(&AddRelationCommand{})
	juju.Register(&AddUnitCommand{})

	// Register destruction commands.
	juju.Register(&DestroyMachineCommand{})
	juju.Register(&DestroyRelationCommand{})
	juju.Register(&DestroyServiceCommand{})
	juju.Register(&DestroyUnitCommand{})
	juju.Register(&DestroyEnvironmentCommand{})

	// Register error resolution commands.
	juju.Register(&StatusCommand{})
	juju.Register(&SCPCommand{})
	juju.Register(&SSHCommand{})
	juju.Register(&ResolvedCommand{})

	// Register configuration commands.
	juju.Register(&InitCommand{})
	juju.Register(&GetCommand{})
	juju.Register(&SetCommand{})
	juju.Register(&GetConstraintsCommand{})
	juju.Register(&SetConstraintsCommand{})
	juju.Register(&ExposeCommand{})
	juju.Register(&UnexposeCommand{})
	juju.Register(&UpgradeJujuCommand{})

	os.Exit(cmd.Main(juju, cmd.DefaultContext(), args[1:]))
}

func main() {
	Main(os.Args)
}

func addEnvironFlags(name *string, f *gnuflag.FlagSet) {
	f.StringVar(name, "e", "", "juju environment to operate in")
	f.StringVar(name, "environment", "", "")
}
<commit_msg>[author=rvb][r=jtv] Import maas provider.<commit_after>package main

import (
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/cmd"
	"os"
)

// When we import an environment provider implementation
// here, it will register itself with environs, and hence
// be available to the juju command.
import (
	_ "launchpad.net/juju-core/environs/ec2"
	_ "launchpad.net/juju-core/environs/maas"
	_ "launchpad.net/juju-core/environs/openstack"
)

var jujuDoc = `
juju provides easy, intelligent service orchestration on top of environments
such as OpenStack, Amazon AWS, or bare metal.

https://juju.ubuntu.com/
`

// Main registers subcommands for the juju executable, and hands over control
// to the cmd package. This function is not redundant with main, because it
// provides an entry point for testing with arbitrary command line arguments.
func Main(args []string) {
	juju := &cmd.SuperCommand{Name: "juju", Doc: jujuDoc, Log: &cmd.Log{}}

	// Register creation commands.
	juju.Register(&BootstrapCommand{})
	juju.Register(&DeployCommand{})
	juju.Register(&AddRelationCommand{})
	juju.Register(&AddUnitCommand{})

	// Register destruction commands.
	juju.Register(&DestroyMachineCommand{})
	juju.Register(&DestroyRelationCommand{})
	juju.Register(&DestroyServiceCommand{})
	juju.Register(&DestroyUnitCommand{})
	juju.Register(&DestroyEnvironmentCommand{})

	// Register error resolution commands.
	juju.Register(&StatusCommand{})
	juju.Register(&SCPCommand{})
	juju.Register(&SSHCommand{})
	juju.Register(&ResolvedCommand{})

	// Register configuration commands.
	juju.Register(&InitCommand{})
	juju.Register(&GetCommand{})
	juju.Register(&SetCommand{})
	juju.Register(&GetConstraintsCommand{})
	juju.Register(&SetConstraintsCommand{})
	juju.Register(&ExposeCommand{})
	juju.Register(&UnexposeCommand{})
	juju.Register(&UpgradeJujuCommand{})

	os.Exit(cmd.Main(juju, cmd.DefaultContext(), args[1:]))
}

func main() {
	Main(os.Args)
}

func addEnvironFlags(name *string, f *gnuflag.FlagSet) {
	f.StringVar(name, "e", "", "juju environment to operate in")
	f.StringVar(name, "environment", "", "")
}
<|endoftext|>


<commit_before><commit_msg>Remove logging statement from cmd.writeOutput<commit_after><|endoftext|>


<commit_before><commit_msg>Get rid of Compiler.gen() method<commit_after><|endoftext|>


<commit_before>package pongo2_test

import (
	"testing"

	"github.com/flosch/pongo2/v6"
	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type TestSuite struct{}

var (
	_          = Suite(&TestSuite{})
	testSuite2 = pongo2.NewSet("test suite 2", pongo2.MustNewLocalFileSystemLoader(""))
)

func parseTemplate(s string, c pongo2.Context) string {
	t, err := testSuite2.FromString(s)
	if err != nil {
		panic(err)
	}
	out, err := t.Execute(c)
	if err != nil {
		panic(err)
	}
	return out
}

func parseTemplateFn(s string, c pongo2.Context) func() {
	return func() {
		parseTemplate(s, c)
	}
}

func (s *TestSuite) TestMisc(c *C) {
	// Must
	// TODO: Add better error message (see issue #18)
	c.Check(
		func() { pongo2.Must(testSuite2.FromFile("template_tests/inheritance/base2.tpl")) },
		PanicMatches,
		`\[Error \(where: fromfile\) in .*template_tests[/\\]inheritance[/\\]doesnotexist.tpl | Line 1 Col 12 near 'doesnotexist.tpl'\] open .*template_tests[/\\]inheritance[/\\]doesnotexist.tpl: no such file or directory`,
	)

	// Context
	c.Check(parseTemplateFn("", pongo2.Context{"'illegal": nil}), PanicMatches, ".*not a valid identifier.*")

	// Registers
	c.Check(pongo2.RegisterFilter("escape", nil).Error(), Matches, ".*is already registered")
	c.Check(pongo2.RegisterTag("for", nil).Error(), Matches, ".*is already registered")

	// ApplyFilter
	v, err := pongo2.ApplyFilter("title", pongo2.AsValue("this is a title"), nil)
	if err != nil {
		c.Fatal(err)
	}
	c.Check(v.String(), Equals, "This Is A Title")
	c.Check(func() {
		_, err := pongo2.ApplyFilter("doesnotexist", nil, nil)
		if err != nil {
			panic(err)
		}
	}, PanicMatches, `\[Error \(where: applyfilter\)\] filter with name 'doesnotexist' not found`)
}

func (s *TestSuite) TestImplicitExecCtx(c *C) {
	tpl, err := pongo2.FromString("{{ ImplicitExec }}")
	if err != nil {
		c.Fatalf("Error in FromString: %v", err)
	}

	val := "a stringy thing"

	res, err := tpl.Execute(pongo2.Context{
		"Value": val,
		"ImplicitExec": func(ctx *pongo2.ExecutionContext) string {
			return ctx.Public["Value"].(string)
		},
	})
	if err != nil {
		c.Fatalf("Error executing template: %v", err)
	}

	c.Check(res, Equals, val)

	// The implicit ctx should not be persisted from call-to-call
	res, err = tpl.Execute(pongo2.Context{
		"ImplicitExec": func() string {
			return val
		},
	})

	if err != nil {
		c.Fatalf("Error executing template: %v", err)
	}

	c.Check(res, Equals, val)
}
<commit_msg>tests: add simple fuzzing<commit_after>package pongo2_test

import (
	"testing"

	"github.com/flosch/pongo2/v6"
	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type TestSuite struct{}

var (
	_          = Suite(&TestSuite{})
	testSuite2 = pongo2.NewSet("test suite 2", pongo2.MustNewLocalFileSystemLoader(""))
)

func parseTemplate(s string, c pongo2.Context) string {
	t, err := testSuite2.FromString(s)
	if err != nil {
		panic(err)
	}
	out, err := t.Execute(c)
	if err != nil {
		panic(err)
	}
	return out
}

func parseTemplateFn(s string, c pongo2.Context) func() {
	return func() {
		parseTemplate(s, c)
	}
}

func (s *TestSuite) TestMisc(c *C) {
	// Must
	// TODO: Add better error message (see issue #18)
	c.Check(
		func() { pongo2.Must(testSuite2.FromFile("template_tests/inheritance/base2.tpl")) },
		PanicMatches,
		`\[Error \(where: fromfile\) in .*template_tests[/\\]inheritance[/\\]doesnotexist.tpl | Line 1 Col 12 near 'doesnotexist.tpl'\] open .*template_tests[/\\]inheritance[/\\]doesnotexist.tpl: no such file or directory`,
	)

	// Context
	c.Check(parseTemplateFn("", pongo2.Context{"'illegal": nil}), PanicMatches, ".*not a valid identifier.*")

	// Registers
	c.Check(pongo2.RegisterFilter("escape", nil).Error(), Matches, ".*is already registered")
	c.Check(pongo2.RegisterTag("for", nil).Error(), Matches, ".*is already registered")

	// ApplyFilter
	v, err := pongo2.ApplyFilter("title", pongo2.AsValue("this is a title"), nil)
	if err != nil {
		c.Fatal(err)
	}
	c.Check(v.String(), Equals, "This Is A Title")
	c.Check(func() {
		_, err := pongo2.ApplyFilter("doesnotexist", nil, nil)
		if err != nil {
			panic(err)
		}
	}, PanicMatches, `\[Error \(where: applyfilter\)\] filter with name 'doesnotexist' not found`)
}

func (s *TestSuite) TestImplicitExecCtx(c *C) {
	tpl, err := pongo2.FromString("{{ ImplicitExec }}")
	if err != nil {
		c.Fatalf("Error in FromString: %v", err)
	}

	val := "a stringy thing"

	res, err := tpl.Execute(pongo2.Context{
		"Value": val,
		"ImplicitExec": func(ctx *pongo2.ExecutionContext) string {
			return ctx.Public["Value"].(string)
		},
	})
	if err != nil {
		c.Fatalf("Error executing template: %v", err)
	}

	c.Check(res, Equals, val)

	// The implicit ctx should not be persisted from call-to-call
	res, err = tpl.Execute(pongo2.Context{
		"ImplicitExec": func() string {
			return val
		},
	})

	if err != nil {
		c.Fatalf("Error executing template: %v", err)
	}

	c.Check(res, Equals, val)
}

func FuzzSimpleExecution(f *testing.F) {
	f.Add("{{ foobar }}", "test123")
	f.Fuzz(func(t *testing.T, tpl, contextValue string) {
		out, err := pongo2.FromString(tpl)
		if err != nil && out != nil {
			t.Errorf("%v", err)
		}
		if err == nil {
			out.Execute(pongo2.Context{
				"foobar": contextValue,
			})
		}
	})
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package uniter

import (
	"bufio"
	"fmt"
	"io"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/worker/uniter/jujuc"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"sync"
	"time"
)

// HookContext is the implementation of jujuc.Context.
type HookContext struct {
	unit *state.Unit

	// config holds the service configuration.
	config map[string]interface{}

	// id identifies the context.
	id string

	// uuid is the universally unique identifier of the environment.
	uuid string

	// relationId identifies the relation for which a relation hook is
	// executing. If it is -1, the context is not running a relation hook;
	// otherwise, its value must be a valid key into the relations map.
	relationId int

	// remoteUnitName identifies the changing unit of the executing relation
	// hook. It will be empty if the context is not running a relation hook,
	// or if it is running a relation-broken hook.
	remoteUnitName string

	// relations contains the context for every relation the unit is a member
	// of, keyed on relation id.
	relations map[int]*ContextRelation
}

func NewHookContext(unit *state.Unit, id, uuid string, relationId int,
	remoteUnitName string, relations map[int]*ContextRelation) *HookContext {
	return &HookContext{
		unit:           unit,
		id:             id,
		uuid:           uuid,
		relationId:     relationId,
		remoteUnitName: remoteUnitName,
		relations:      relations,
	}
}

func (ctx *HookContext) UnitName() string {
	return ctx.unit.Name()
}

func (ctx *HookContext) PublicAddress() (string, bool) {
	return ctx.unit.PublicAddress()
}

func (ctx *HookContext) PrivateAddress() (string, bool) {
	return ctx.unit.PrivateAddress()
}

func (ctx *HookContext) OpenPort(protocol string, port int) error {
	return ctx.unit.OpenPort(protocol, port)
}

func (ctx *HookContext) ClosePort(protocol string, port int) error {
	return ctx.unit.ClosePort(protocol, port)
}

func (ctx *HookContext) Config() (map[string]interface{}, error) {
	if ctx.config == nil {
		var err error
		ctx.config, err = ctx.unit.ServiceConfig()
		if err != nil {
			return nil, err
		}
	}
	return ctx.config, nil
}

func (ctx *HookContext) HookRelation() (jujuc.ContextRelation, bool) {
	return ctx.Relation(ctx.relationId)
}

func (ctx *HookContext) RemoteUnitName() (string, bool) {
	return ctx.remoteUnitName, ctx.remoteUnitName != ""
}

func (ctx *HookContext) Relation(id int) (jujuc.ContextRelation, bool) {
	r, found := ctx.relations[id]
	return r, found
}

func (ctx *HookContext) RelationIds() []int {
	ids := []int{}
	for id := range ctx.relations {
		ids = append(ids, id)
	}
	return ids
}

// hookVars returns an os.Environ-style list of strings necessary to run a hook
// such that it can know what environment it's operating in, and can call back
// into ctx.
func (ctx *HookContext) hookVars(charmDir, toolsDir, socketPath string) []string {
	vars := []string{
		"APT_LISTCHANGES_FRONTEND=none",
		"DEBIAN_FRONTEND=noninteractive",
		"PATH=" + toolsDir + ":" + os.Getenv("PATH"),
		"CHARM_DIR=" + charmDir,
		"JUJU_CONTEXT_ID=" + ctx.id,
		"JUJU_AGENT_SOCKET=" + socketPath,
		"JUJU_UNIT_NAME=" + ctx.unit.Name(),
		"JUJU_ENV_UUID=" + ctx.uuid,
	}
	if r, found := ctx.HookRelation(); found {
		vars = append(vars, "JUJU_RELATION="+r.Name())
		vars = append(vars, "JUJU_RELATION_ID="+r.FakeId())
		name, _ := ctx.RemoteUnitName()
		vars = append(vars, "JUJU_REMOTE_UNIT="+name)
	}
	return vars
}

// RunHook executes a hook in an environment which allows it to to call back
// into ctx to execute jujuc tools.
func (ctx *HookContext) RunHook(hookName, charmDir, toolsDir, socketPath string) error {
	ps := exec.Command(filepath.Join(charmDir, "hooks", hookName))
	ps.Env = ctx.hookVars(charmDir, toolsDir, socketPath)
	ps.Dir = charmDir
	outReader, outWriter, err := os.Pipe()
	if err != nil {
		return fmt.Errorf("cannot make logging pipe: %v", err)
	}
	ps.Stdout = outWriter
	ps.Stderr = outWriter
	logger := &hookLogger{
		r:    outReader,
		done: make(chan struct{}),
	}
	go logger.run()
	err = ps.Start()
	outWriter.Close()
	if err == nil {
		err = ps.Wait()
	}
	logger.stop()
	if ee, ok := err.(*exec.Error); ok && err != nil {
		if os.IsNotExist(ee.Err) {
			// Missing hook is perfectly valid, but worth mentioning.
			log.Infof("worker/uniter: skipped %q hook (not implemented)", hookName)
			return nil
		}
	}
	write := err == nil
	for id, rctx := range ctx.relations {
		if write {
			if e := rctx.WriteSettings(); e != nil {
				e = fmt.Errorf(
					"could not write settings from %q to relation %d: %v",
					hookName, id, e,
				)
				log.Errorf("worker/uniter: %v", e)
				if err == nil {
					err = e
				}
			}
		}
		rctx.ClearCache()
	}
	return err
}

type hookLogger struct {
	r       io.ReadCloser
	done    chan struct{}
	mu      sync.Mutex
	stopped bool
}

func (l *hookLogger) run() {
	defer close(l.done)
	defer l.r.Close()
	br := bufio.NewReaderSize(l.r, 4096)
	for {
		line, _, err := br.ReadLine()
		if err != nil {
			if err != io.EOF {
				log.Errorf("worker/uniter: cannot read hook output: %v", err)
			}
			break
		}
		l.mu.Lock()
		if l.stopped {
			l.mu.Unlock()
			return
		}
		log.Infof("worker/uniter: HOOK %s", line)
		l.mu.Unlock()
	}
}

func (l *hookLogger) stop() {
	// We can see the process exit before the logger has processed
	// all its output, so allow a moment for the data buffered
	// in the pipe to be processed. We don't wait indefinitely though,
	// because the hook may have started a background process
	// that keeps the pipe open.
	select {
	case <-l.done:
	case <-time.After(100 * time.Millisecond):
	}
	// We can't close the pipe asynchronously, so just
	// stifle output instead.
	l.mu.Lock()
	l.stopped = true
	l.mu.Unlock()
}

// SettingsMap is a map from unit name to relation settings.
type SettingsMap map[string]map[string]interface{}

// ContextRelation is the implementation of jujuc.ContextRelation.
type ContextRelation struct {
	ru *state.RelationUnit

	// members contains settings for known relation members. Nil values
	// indicate members whose settings have not yet been cached.
	members SettingsMap

	// settings allows read and write access to the relation unit settings.
	settings *state.Settings

	// cache is a short-term cache that enables consistent access to settings
	// for units that are not currently participating in the relation. Its
	// contents should be cleared whenever a new hook is executed.
	cache SettingsMap
}

// NewContextRelation creates a new context for the given relation unit.
// The unit-name keys of members supplies the initial membership.
func NewContextRelation(ru *state.RelationUnit, members map[string]int64) *ContextRelation {
	ctx := &ContextRelation{ru: ru, members: SettingsMap{}}
	for unit := range members {
		ctx.members[unit] = nil
	}
	ctx.ClearCache()
	return ctx
}

// WriteSettings persists all changes made to the unit's relation settings.
func (ctx *ContextRelation) WriteSettings() (err error) {
	if ctx.settings != nil {
		_, err = ctx.settings.Write()
	}
	return
}

// ClearCache discards all cached settings for units that are not members
// of the relation, and all unwritten changes to the unit's relation settings.
// including any changes to Settings that have not been written.
func (ctx *ContextRelation) ClearCache() {
	ctx.settings = nil
	ctx.cache = make(SettingsMap)
}

// UpdateMembers ensures that the context is aware of every supplied member
// unit. For each supplied member that has non-nil settings, the cached
// settings will be overwritten; but nil settings will not overwrite cached
// ones.
func (ctx *ContextRelation) UpdateMembers(members SettingsMap) {
	for m, s := range members {
		_, found := ctx.members[m]
		if !found || s != nil {
			ctx.members[m] = s
		}
	}
}

// DeleteMember drops the membership and cache of a single remote unit, without
// perturbing settings for the remaining members.
func (ctx *ContextRelation) DeleteMember(unitName string) {
	delete(ctx.members, unitName)
}

func (ctx *ContextRelation) Id() int {
	return ctx.ru.Relation().Id()
}

func (ctx *ContextRelation) Name() string {
	return ctx.ru.Endpoint().Name
}

func (ctx *ContextRelation) FakeId() string {
	return fmt.Sprintf("%s:%d", ctx.Name(), ctx.ru.Relation().Id())
}

func (ctx *ContextRelation) UnitNames() (units []string) {
	for unit := range ctx.members {
		units = append(units, unit)
	}
	sort.Strings(units)
	return units
}

func (ctx *ContextRelation) Settings() (jujuc.Settings, error) {
	if ctx.settings == nil {
		node, err := ctx.ru.Settings()
		if err != nil {
			return nil, err
		}
		ctx.settings = node
	}
	return ctx.settings, nil
}

func (ctx *ContextRelation) ReadSettings(unit string) (settings map[string]interface{}, err error) {
	settings, member := ctx.members[unit]
	if settings == nil {
		if settings = ctx.cache[unit]; settings == nil {
			settings, err = ctx.ru.ReadSettings(unit)
			if err != nil {
				return nil, err
			}
		}
	}
	if member {
		ctx.members[unit] = settings
	} else {
		ctx.cache[unit] = settings
	}
	return settings, nil
}
<commit_msg>Add apiAddrs to hooks context as space separated addresses.<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package uniter

import (
	"bufio"
	"fmt"
	"io"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/worker/uniter/jujuc"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
)

// HookContext is the implementation of jujuc.Context.
type HookContext struct {
	unit *state.Unit

	// config holds the service configuration.
	config map[string]interface{}

	// id identifies the context.
	id string

	// uuid is the universally unique identifier of the environment.
	uuid string

	// relationId identifies the relation for which a relation hook is
	// executing. If it is -1, the context is not running a relation hook;
	// otherwise, its value must be a valid key into the relations map.
	relationId int

	// remoteUnitName identifies the changing unit of the executing relation
	// hook. It will be empty if the context is not running a relation hook,
	// or if it is running a relation-broken hook.
	remoteUnitName string

	// relations contains the context for every relation the unit is a member
	// of, keyed on relation id.
	relations map[int]*ContextRelation

	// apiAddrs contains the API server addresses.
	apiAddrs []string
}

func NewHookContext(unit *state.Unit, id, uuid string, relationId int,
	remoteUnitName string, relations map[int]*ContextRelation,
	apiAddrs []string) *HookContext {
	return &HookContext{
		unit:           unit,
		id:             id,
		uuid:           uuid,
		relationId:     relationId,
		remoteUnitName: remoteUnitName,
		relations:      relations,
		apiAddrs:       apiAddrs,
	}
}

func (ctx *HookContext) UnitName() string {
	return ctx.unit.Name()
}

func (ctx *HookContext) PublicAddress() (string, bool) {
	return ctx.unit.PublicAddress()
}

func (ctx *HookContext) PrivateAddress() (string, bool) {
	return ctx.unit.PrivateAddress()
}

func (ctx *HookContext) OpenPort(protocol string, port int) error {
	return ctx.unit.OpenPort(protocol, port)
}

func (ctx *HookContext) ClosePort(protocol string, port int) error {
	return ctx.unit.ClosePort(protocol, port)
}

func (ctx *HookContext) Config() (map[string]interface{}, error) {
	if ctx.config == nil {
		var err error
		ctx.config, err = ctx.unit.ServiceConfig()
		if err != nil {
			return nil, err
		}
	}
	return ctx.config, nil
}

func (ctx *HookContext) HookRelation() (jujuc.ContextRelation, bool) {
	return ctx.Relation(ctx.relationId)
}

func (ctx *HookContext) RemoteUnitName() (string, bool) {
	return ctx.remoteUnitName, ctx.remoteUnitName != ""
}

func (ctx *HookContext) Relation(id int) (jujuc.ContextRelation, bool) {
	r, found := ctx.relations[id]
	return r, found
}

func (ctx *HookContext) RelationIds() []int {
	ids := []int{}
	for id := range ctx.relations {
		ids = append(ids, id)
	}
	return ids
}

// hookVars returns an os.Environ-style list of strings necessary to run a hook
// such that it can know what environment it's operating in, and can call back
// into ctx.
func (ctx *HookContext) hookVars(charmDir, toolsDir, socketPath string) []string {
	vars := []string{
		"APT_LISTCHANGES_FRONTEND=none",
		"DEBIAN_FRONTEND=noninteractive",
		"PATH=" + toolsDir + ":" + os.Getenv("PATH"),
		"CHARM_DIR=" + charmDir,
		"JUJU_CONTEXT_ID=" + ctx.id,
		"JUJU_AGENT_SOCKET=" + socketPath,
		"JUJU_UNIT_NAME=" + ctx.unit.Name(),
		"JUJU_ENV_UUID=" + ctx.uuid,
		"JUJU_API_ADDRESSES=" + strings.Join(ctx.apiAddrs, " "),
	}
	if r, found := ctx.HookRelation(); found {
		vars = append(vars, "JUJU_RELATION="+r.Name())
		vars = append(vars, "JUJU_RELATION_ID="+r.FakeId())
		name, _ := ctx.RemoteUnitName()
		vars = append(vars, "JUJU_REMOTE_UNIT="+name)
	}
	return vars
}

// RunHook executes a hook in an environment which allows it to to call back
// into ctx to execute jujuc tools.
func (ctx *HookContext) RunHook(hookName, charmDir, toolsDir, socketPath string) error {
	ps := exec.Command(filepath.Join(charmDir, "hooks", hookName))
	ps.Env = ctx.hookVars(charmDir, toolsDir, socketPath)
	ps.Dir = charmDir
	outReader, outWriter, err := os.Pipe()
	if err != nil {
		return fmt.Errorf("cannot make logging pipe: %v", err)
	}
	ps.Stdout = outWriter
	ps.Stderr = outWriter
	logger := &hookLogger{
		r:    outReader,
		done: make(chan struct{}),
	}
	go logger.run()
	err = ps.Start()
	outWriter.Close()
	if err == nil {
		err = ps.Wait()
	}
	logger.stop()
	if ee, ok := err.(*exec.Error); ok && err != nil {
		if os.IsNotExist(ee.Err) {
			// Missing hook is perfectly valid, but worth mentioning.
			log.Infof("worker/uniter: skipped %q hook (not implemented)", hookName)
			return nil
		}
	}
	write := err == nil
	for id, rctx := range ctx.relations {
		if write {
			if e := rctx.WriteSettings(); e != nil {
				e = fmt.Errorf(
					"could not write settings from %q to relation %d: %v",
					hookName, id, e,
				)
				log.Errorf("worker/uniter: %v", e)
				if err == nil {
					err = e
				}
			}
		}
		rctx.ClearCache()
	}
	return err
}

type hookLogger struct {
	r       io.ReadCloser
	done    chan struct{}
	mu      sync.Mutex
	stopped bool
}

func (l *hookLogger) run() {
	defer close(l.done)
	defer l.r.Close()
	br := bufio.NewReaderSize(l.r, 4096)
	for {
		line, _, err := br.ReadLine()
		if err != nil {
			if err != io.EOF {
				log.Errorf("worker/uniter: cannot read hook output: %v", err)
			}
			break
		}
		l.mu.Lock()
		if l.stopped {
			l.mu.Unlock()
			return
		}
		log.Infof("worker/uniter: HOOK %s", line)
		l.mu.Unlock()
	}
}

func (l *hookLogger) stop() {
	// We can see the process exit before the logger has processed
	// all its output, so allow a moment for the data buffered
	// in the pipe to be processed. We don't wait indefinitely though,
	// because the hook may have started a background process
	// that keeps the pipe open.
	select {
	case <-l.done:
	case <-time.After(100 * time.Millisecond):
	}
	// We can't close the pipe asynchronously, so just
	// stifle output instead.
	l.mu.Lock()
	l.stopped = true
	l.mu.Unlock()
}

// SettingsMap is a map from unit name to relation settings.
type SettingsMap map[string]map[string]interface{}

// ContextRelation is the implementation of jujuc.ContextRelation.
type ContextRelation struct {
	ru *state.RelationUnit

	// members contains settings for known relation members. Nil values
	// indicate members whose settings have not yet been cached.
	members SettingsMap

	// settings allows read and write access to the relation unit settings.
	settings *state.Settings

	// cache is a short-term cache that enables consistent access to settings
	// for units that are not currently participating in the relation. Its
	// contents should be cleared whenever a new hook is executed.
	cache SettingsMap
}

// NewContextRelation creates a new context for the given relation unit.
// The unit-name keys of members supplies the initial membership.
func NewContextRelation(ru *state.RelationUnit, members map[string]int64) *ContextRelation {
	ctx := &ContextRelation{ru: ru, members: SettingsMap{}}
	for unit := range members {
		ctx.members[unit] = nil
	}
	ctx.ClearCache()
	return ctx
}

// WriteSettings persists all changes made to the unit's relation settings.
func (ctx *ContextRelation) WriteSettings() (err error) {
	if ctx.settings != nil {
		_, err = ctx.settings.Write()
	}
	return
}

// ClearCache discards all cached settings for units that are not members
// of the relation, and all unwritten changes to the unit's relation settings.
// including any changes to Settings that have not been written.
func (ctx *ContextRelation) ClearCache() {
	ctx.settings = nil
	ctx.cache = make(SettingsMap)
}

// UpdateMembers ensures that the context is aware of every supplied member
// unit. For each supplied member that has non-nil settings, the cached
// settings will be overwritten; but nil settings will not overwrite cached
// ones.
func (ctx *ContextRelation) UpdateMembers(members SettingsMap) {
	for m, s := range members {
		_, found := ctx.members[m]
		if !found || s != nil {
			ctx.members[m] = s
		}
	}
}

// DeleteMember drops the membership and cache of a single remote unit, without
// perturbing settings for the remaining members.
func (ctx *ContextRelation) DeleteMember(unitName string) {
	delete(ctx.members, unitName)
}

func (ctx *ContextRelation) Id() int {
	return ctx.ru.Relation().Id()
}

func (ctx *ContextRelation) Name() string {
	return ctx.ru.Endpoint().Name
}

func (ctx *ContextRelation) FakeId() string {
	return fmt.Sprintf("%s:%d", ctx.Name(), ctx.ru.Relation().Id())
}

func (ctx *ContextRelation) UnitNames() (units []string) {
	for unit := range ctx.members {
		units = append(units, unit)
	}
	sort.Strings(units)
	return units
}

func (ctx *ContextRelation) Settings() (jujuc.Settings, error) {
	if ctx.settings == nil {
		node, err := ctx.ru.Settings()
		if err != nil {
			return nil, err
		}
		ctx.settings = node
	}
	return ctx.settings, nil
}

func (ctx *ContextRelation) ReadSettings(unit string) (settings map[string]interface{}, err error) {
	settings, member := ctx.members[unit]
	if settings == nil {
		if settings = ctx.cache[unit]; settings == nil {
			settings, err = ctx.ru.ReadSettings(unit)
			if err != nil {
				return nil, err
			}
		}
	}
	if member {
		ctx.members[unit] = settings
	} else {
		ctx.cache[unit] = settings
	}
	return settings, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	log "github.com/Sirupsen/logrus"
)

type PortScannerConfig struct {
	Portrange string
	Ipaddress string
	Protocol  string
}

type PortScannerResult struct {
	portScannerResult portScannerResultMap
	running           int
	timeOut           int
}

type portScannerResultMap map[string]bool

var portScannerTuple PortScannerResult

func main() {
	log.SetLevel(log.DebugLevel)
	log.Infoln("*******************************************")
	log.Infoln("Port Scanner")
	log.Infoln("*******************************************")

	t := time.Now()
	defer func() {
		if e := recover(); e != nil {
			log.Debugln(e)
		}
	}()

	log.Debugln(loadConfig("config.properties"))

	log.Debugln("Parsed input data ", len(portScannerTuple.portScannerResult))
	CheckPort(&portScannerTuple)

	for key, value := range portScannerTuple.portScannerResult {
		if (value){
			log.Debugln("Port Scanner Result", key, " port is open :", value)
		}
	}

	log.Debugln("Total time taken %s to scan %d ports", time.Since(t), len(portScannerTuple.portScannerResult))
}

func CheckPort(portScannerTuple *PortScannerResult) {
	for record := range portScannerTuple.portScannerResult {
		for portScannerTuple.running >= 2000 {
			log.Debugln("Maximum threads spawned", portScannerTuple.running, " waiting ...", 1*time.Second)
			time.Sleep(1 * time.Second)
		}
		r := strings.Split(record, ":")
		port, _ := strconv.Atoi(r[1])
		portScannerTuple.running++
		go check(portScannerTuple, r[0], uint16(port))
	}

	for portScannerTuple.running != 0 {
		time.Sleep(1 * time.Second)
	}
}

func check(portScannerTuple *PortScannerResult, ip string, port uint16) {
	connection, err := net.DialTimeout("tcp", ip+":"+fmt.Sprintf("%d", port), time.Duration(portScannerTuple.timeOut)*time.Second)
	if err == nil {
		portScannerTuple.portScannerResult[fmt.Sprintf("%s:%d", ip, port)] = true
		//log.Debugln(fmt.Sprintf("%s:%d - true", ip, port))
		connection.Close()
	} else {
		portScannerTuple.portScannerResult[fmt.Sprintf("%s:%d", ip, port)] = false
		//log.Debugln(fmt.Sprintf("%s:%d - %s", ip, port, err))
	}
	portScannerTuple.running--
}

func loadConfig(file string) PortScannerConfig {
	var readConfigStruct PortScannerConfig
	if metaData, err := toml.DecodeFile(file, &readConfigStruct); err != nil {
		log.Debugln("Error Occured Reading file", err, metaData)
	}
	ports := strings.Split(readConfigStruct.Portrange,"-")
	p1, err := strconv.Atoi(ports[0])
    if err != nil {
        log.Errorln(err)
    }
	log.Debugln("p1",p1)
	p2, err := strconv.Atoi(ports[1])
    if err != nil {
        log.Errorln(err)
    }
	log.Debugln("p2",p2)
	portScannerTuple.portScannerResult = make(portScannerResultMap)
	for port := p1; port <= p2; port++ {
		portScannerTuple.timeOut = 5
		portScannerTuple.portScannerResult[readConfigStruct.Ipaddress+fmt.Sprintf(":%d", port)] = false
	}
	return readConfigStruct
}
<commit_msg>Initial commit of portscanner<commit_after>package main

import (
	"fmt"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	log "github.com/Sirupsen/logrus"
)

type PortScannerConfig struct {
	Portrange string
	Ipaddress string
	Protocol  string
}

type PortScannerResult struct {
	portScannerResult portScannerResultMap
	running           int
	timeOut           int
}

type portScannerResultMap map[string]bool

var portScannerTuple PortScannerResult

func main() {
	log.SetLevel(log.DebugLevel)
	log.Infoln("*******************************************")
	log.Infoln("Port Scanner")
	log.Infoln("*******************************************")

	t := time.Now()
	defer func() {
		if e := recover(); e != nil {
			log.Debugln(e)
		}
	}()

	log.Debugln(loadConfig("config.properties"))

	log.Debugln("Parsed input data ", len(portScannerTuple.portScannerResult))
	CheckPort(&portScannerTuple)

	for key, value := range portScannerTuple.portScannerResult {
		if value {
			log.Debugln("Port Scanner Result", key, " port is open :", value)
		}
	}

	log.Debugln("Total time taken %s to scan %d ports", time.Since(t), len(portScannerTuple.portScannerResult))
}

func CheckPort(portScannerTuple *PortScannerResult) {
	for record := range portScannerTuple.portScannerResult {
		for portScannerTuple.running >= 2000 {
			log.Debugln("Maximum threads spawned", portScannerTuple.running, " waiting ...", 1*time.Second)
			time.Sleep(1 * time.Second)
		}
		r := strings.Split(record, ":")
		port, _ := strconv.Atoi(r[1])
		portScannerTuple.running++
		go check(portScannerTuple, r[0], uint16(port))
	}

	for portScannerTuple.running != 0 {
		time.Sleep(1 * time.Second)
	}
}

func check(portScannerTuple *PortScannerResult, ip string, port uint16) {
	connection, err := net.DialTimeout("tcp", ip+":"+fmt.Sprintf("%d", port), time.Duration(portScannerTuple.timeOut)*time.Second)
	if err == nil {
		portScannerTuple.portScannerResult[fmt.Sprintf("%s:%d", ip, port)] = true
		//log.Debugln(fmt.Sprintf("%s:%d - true", ip, port))
		connection.Close()
	} else {
		portScannerTuple.portScannerResult[fmt.Sprintf("%s:%d", ip, port)] = false
		//log.Debugln(fmt.Sprintf("%s:%d - %s", ip, port, err))
	}
	portScannerTuple.running--
}

func loadConfig(file string) PortScannerConfig {
	var readConfigStruct PortScannerConfig
	if metaData, err := toml.DecodeFile(file, &readConfigStruct); err != nil {
		log.Debugln("Error Occured Reading file", err, metaData)
	}
	ports := strings.Split(readConfigStruct.Portrange, "-")
	p1, err := strconv.Atoi(ports[0])
	if err != nil {
		log.Errorln(err)
	}
	log.Debugln("p1", p1)
	p2, err := strconv.Atoi(ports[1])
	if err != nil {
		log.Errorln(err)
	}
	log.Debugln("p2", p2)
	portScannerTuple.portScannerResult = make(portScannerResultMap)
	for port := p1; port <= p2; port++ {
		portScannerTuple.timeOut = 5
		portScannerTuple.portScannerResult[readConfigStruct.Ipaddress+fmt.Sprintf(":%d", port)] = false
	}
	return readConfigStruct
}
<|endoftext|>


<commit_before>package server

import (
	"context"
	"fmt"
	"path"
	"time"

	log "github.com/sirupsen/logrus"
	"k8s.io/kubernetes/pkg/api"
	labelspkg "k8s.io/kubernetes/pkg/labels"
	kube_watch "k8s.io/kubernetes/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/util"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
	}
)

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	masterLock := dlock.NewDLock(a.etcdClient, path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)

		log.Infof("Launching PPS master process")

		pipelineWatcher, err := a.pipelines.ReadOnly(ctx).WatchWithPrev()
		if err != nil {
			return fmt.Errorf("error creating watch: %+v", err)
		}
		defer pipelineWatcher.Close()

		kubePipelineWatch, err := a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
			LabelSelector: labelspkg.SelectorFromSet(map[string]string{
				"component": "worker",
			}),
			Watch: true,
		})
		if err != nil {
			return err
		}
		defer kubePipelineWatch.Stop()

		for {
			select {
			case event := <-pipelineWatcher.Watch():
				if event.Err != nil {
					return fmt.Errorf("event err: %+v", event.Err)
				}
				switch event.Type {
				case watch.EventPut:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.Unmarshal(&pipelineName, &pipelineInfo); err != nil {
						return err
					}

					if pipelineInfo.Salt == "" {
						if _, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
							pipelines := a.pipelines.ReadWrite(stm)
							newPipelineInfo := new(pps.PipelineInfo)
							if err := pipelines.Get(pipelineInfo.Pipeline.Name, newPipelineInfo); err != nil {
								return fmt.Errorf("error getting pipeline %s: %+v", pipelineName, err)
							}
							if newPipelineInfo.Salt == "" {
								newPipelineInfo.Salt = uuid.NewWithoutDashes()
							}
							pipelines.Put(pipelineInfo.Pipeline.Name, newPipelineInfo)
							return nil
						}); err != nil {
							return err
						}
					}

					var prevPipelineInfo pps.PipelineInfo
					if event.PrevKey != nil {
						if err := event.UnmarshalPrev(&pipelineName, &prevPipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been stopped, delete workers
					if pipelineStateToStopped(pipelineInfo.State) {
						log.Infof("master: deleting workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been restarted, create workers
					if !pipelineStateToStopped(pipelineInfo.State) && event.PrevKey != nil && pipelineStateToStopped(prevPipelineInfo.State) {
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}

					// If the pipeline has been updated, create new workers
					if pipelineInfo.Version > prevPipelineInfo.Version {
						log.Infof("master: creating/updating workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if event.PrevKey != nil {
							if err := a.deleteWorkersForPipeline(&prevPipelineInfo); err != nil {
								return err
							}
						}
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}
				case watch.EventDelete:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.UnmarshalPrev(&pipelineName, &pipelineInfo); err != nil {
						return err
					}
					if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
						return err
					}
				}
			case event := <-kubePipelineWatch.ResultChan():
				if event.Type == kube_watch.Error {
					kubePipelineWatch, err = a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
						LabelSelector: labelspkg.SelectorFromSet(map[string]string{
							"component": "worker",
						}),
						Watch: true,
					})
					if err != nil {
						return err
					}
					defer kubePipelineWatch.Stop()
				}
				pod, ok := event.Object.(*api.Pod)
				if !ok {
					continue
				}
				if pod.Status.Phase == api.PodFailed {
					log.Errorf("pod failed because: %s", pod.Status.Message)
				}
				for _, status := range pod.Status.ContainerStatuses {
					if status.Name == "user" && status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
						if err := a.setPipelineFailure(ctx, pod.ObjectMeta.Annotations["pipelineName"], status.State.Waiting.Message); err != nil {
							return err
						}
					}
				}
			}
		}
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		log.Errorf("master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	// Set pipeline state to failure
	_, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
		pipelines := a.pipelines.ReadWrite(stm)
		pipelineInfo := new(pps.PipelineInfo)
		if err := pipelines.Get(pipelineName, pipelineInfo); err != nil {
			return err
		}
		pipelineInfo.State = pps.PipelineState_PIPELINE_FAILURE
		pipelineInfo.Reason = reason
		pipelines.Put(pipelineName, pipelineInfo)
		return nil
	})
	return err
}

func (a *apiServer) upsertWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	var errCount int
	return backoff.RetryNotify(func() error {
		parallelism, err := ppsserver.GetExpectedNumWorkers(a.kubeClient, pipelineInfo.ParallelismSpec)
		if err != nil {
			return err
		}
		var resources *api.ResourceList
		if pipelineInfo.ResourceSpec != nil {
			resources, err = util.GetResourceListFromPipeline(pipelineInfo)
			if err != nil {
				return err
			}
		}

		// Retrieve the current state of the RC.  If the RC is scaled down,
		// we want to ensure that it remains scaled down.
		rc := a.kubeClient.ReplicationControllers(a.namespace)
		workerRc, err := rc.Get(ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version))
		if err == nil {
			if (workerRc.Spec.Template.Spec.Containers[0].Resources.Requests == nil) && workerRc.Spec.Replicas == 1 {
				parallelism = 1
				resources = nil
			}
		}

		options := a.getWorkerOptions(
			pipelineInfo.Pipeline.Name,
			ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version),
			int32(parallelism),
			resources,
			pipelineInfo.Transform,
			pipelineInfo.CacheSize)
		// Set the pipeline name env
		options.workerEnv = append(options.workerEnv, api.EnvVar{
			Name:  client.PPSPipelineNameEnv,
			Value: pipelineInfo.Pipeline.Name,
		})
		return a.createWorkerRc(options)
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		errCount++
		if errCount >= 3 {
			return err
		}
		log.Errorf("error creating workers for pipeline %v: %v; retrying in %v", pipelineInfo.Pipeline.Name, err, d)
		return nil
	})
}

func (a *apiServer) deleteWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	rcName := ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	if err := a.kubeClient.Services(a.namespace).Delete(rcName); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	falseVal := false
	deleteOptions := &api.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	if err := a.kubeClient.ReplicationControllers(a.namespace).Delete(rcName, deleteOptions); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	return nil
}
<commit_msg>Fix an issue where when pachd restarts, it restarts paused pipelines<commit_after>package server

import (
	"context"
	"fmt"
	"path"
	"time"

	log "github.com/sirupsen/logrus"
	"k8s.io/kubernetes/pkg/api"
	labelspkg "k8s.io/kubernetes/pkg/labels"
	kube_watch "k8s.io/kubernetes/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/util"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
	}
)

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	masterLock := dlock.NewDLock(a.etcdClient, path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)

		log.Infof("Launching PPS master process")

		pipelineWatcher, err := a.pipelines.ReadOnly(ctx).WatchWithPrev()
		if err != nil {
			return fmt.Errorf("error creating watch: %+v", err)
		}
		defer pipelineWatcher.Close()

		kubePipelineWatch, err := a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
			LabelSelector: labelspkg.SelectorFromSet(map[string]string{
				"component": "worker",
			}),
			Watch: true,
		})
		if err != nil {
			return err
		}
		defer kubePipelineWatch.Stop()

		for {
			select {
			case event := <-pipelineWatcher.Watch():
				if event.Err != nil {
					return fmt.Errorf("event err: %+v", event.Err)
				}
				switch event.Type {
				case watch.EventPut:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.Unmarshal(&pipelineName, &pipelineInfo); err != nil {
						return err
					}

					if pipelineInfo.Salt == "" {
						if _, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
							pipelines := a.pipelines.ReadWrite(stm)
							newPipelineInfo := new(pps.PipelineInfo)
							if err := pipelines.Get(pipelineInfo.Pipeline.Name, newPipelineInfo); err != nil {
								return fmt.Errorf("error getting pipeline %s: %+v", pipelineName, err)
							}
							if newPipelineInfo.Salt == "" {
								newPipelineInfo.Salt = uuid.NewWithoutDashes()
							}
							pipelines.Put(pipelineInfo.Pipeline.Name, newPipelineInfo)
							return nil
						}); err != nil {
							return err
						}
					}

					var prevPipelineInfo pps.PipelineInfo
					if event.PrevKey != nil {
						if err := event.UnmarshalPrev(&pipelineName, &prevPipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been stopped, delete workers
					if pipelineStateToStopped(pipelineInfo.State) {
						log.Infof("master: deleting workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
							return err
						}
					}

					// If the pipeline has been restarted, create workers
					if !pipelineStateToStopped(pipelineInfo.State) && event.PrevKey != nil && pipelineStateToStopped(prevPipelineInfo.State) {
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}

					// If the pipeline has been updated, create new workers
					if pipelineInfo.Version > prevPipelineInfo.Version && !pipelineStateToStopped(pipelineInfo.State) {
						log.Infof("master: creating/updating workers for pipeline %s", pipelineInfo.Pipeline.Name)
						if event.PrevKey != nil {
							if err := a.deleteWorkersForPipeline(&prevPipelineInfo); err != nil {
								return err
							}
						}
						if err := a.upsertWorkersForPipeline(&pipelineInfo); err != nil {
							if err := a.setPipelineFailure(ctx, pipelineInfo.Pipeline.Name, fmt.Sprintf("failed to create workers: %s", err.Error())); err != nil {
								return err
							}
							continue
						}
					}
				case watch.EventDelete:
					var pipelineName string
					var pipelineInfo pps.PipelineInfo
					if err := event.UnmarshalPrev(&pipelineName, &pipelineInfo); err != nil {
						return err
					}
					if err := a.deleteWorkersForPipeline(&pipelineInfo); err != nil {
						return err
					}
				}
			case event := <-kubePipelineWatch.ResultChan():
				if event.Type == kube_watch.Error {
					kubePipelineWatch, err = a.kubeClient.Pods(a.namespace).Watch(api.ListOptions{
						LabelSelector: labelspkg.SelectorFromSet(map[string]string{
							"component": "worker",
						}),
						Watch: true,
					})
					if err != nil {
						return err
					}
					defer kubePipelineWatch.Stop()
				}
				pod, ok := event.Object.(*api.Pod)
				if !ok {
					continue
				}
				if pod.Status.Phase == api.PodFailed {
					log.Errorf("pod failed because: %s", pod.Status.Message)
				}
				for _, status := range pod.Status.ContainerStatuses {
					if status.Name == "user" && status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
						if err := a.setPipelineFailure(ctx, pod.ObjectMeta.Annotations["pipelineName"], status.State.Waiting.Message); err != nil {
							return err
						}
					}
				}
			}
		}
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		log.Errorf("master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	// Set pipeline state to failure
	_, err := col.NewSTM(ctx, a.etcdClient, func(stm col.STM) error {
		pipelines := a.pipelines.ReadWrite(stm)
		pipelineInfo := new(pps.PipelineInfo)
		if err := pipelines.Get(pipelineName, pipelineInfo); err != nil {
			return err
		}
		pipelineInfo.State = pps.PipelineState_PIPELINE_FAILURE
		pipelineInfo.Reason = reason
		pipelines.Put(pipelineName, pipelineInfo)
		return nil
	})
	return err
}

func (a *apiServer) upsertWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	var errCount int
	return backoff.RetryNotify(func() error {
		parallelism, err := ppsserver.GetExpectedNumWorkers(a.kubeClient, pipelineInfo.ParallelismSpec)
		if err != nil {
			return err
		}
		var resources *api.ResourceList
		if pipelineInfo.ResourceSpec != nil {
			resources, err = util.GetResourceListFromPipeline(pipelineInfo)
			if err != nil {
				return err
			}
		}

		// Retrieve the current state of the RC.  If the RC is scaled down,
		// we want to ensure that it remains scaled down.
		rc := a.kubeClient.ReplicationControllers(a.namespace)
		workerRc, err := rc.Get(ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version))
		if err == nil {
			if (workerRc.Spec.Template.Spec.Containers[0].Resources.Requests == nil) && workerRc.Spec.Replicas == 1 {
				parallelism = 1
				resources = nil
			}
		}

		options := a.getWorkerOptions(
			pipelineInfo.Pipeline.Name,
			ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version),
			int32(parallelism),
			resources,
			pipelineInfo.Transform,
			pipelineInfo.CacheSize)
		// Set the pipeline name env
		options.workerEnv = append(options.workerEnv, api.EnvVar{
			Name:  client.PPSPipelineNameEnv,
			Value: pipelineInfo.Pipeline.Name,
		})
		return a.createWorkerRc(options)
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		errCount++
		if errCount >= 3 {
			return err
		}
		log.Errorf("error creating workers for pipeline %v: %v; retrying in %v", pipelineInfo.Pipeline.Name, err, d)
		return nil
	})
}

func (a *apiServer) deleteWorkersForPipeline(pipelineInfo *pps.PipelineInfo) error {
	rcName := ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	if err := a.kubeClient.Services(a.namespace).Delete(rcName); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	falseVal := false
	deleteOptions := &api.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	if err := a.kubeClient.ReplicationControllers(a.namespace).Delete(rcName, deleteOptions); err != nil {
		if !isNotFoundErr(err) {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package server

import (
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/log"
	"github.com/pachyderm/pachyderm/src/server/pkg/metrics"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsdb"
	"github.com/pachyderm/pachyderm/src/server/pkg/serviceenv"
	txnenv "github.com/pachyderm/pachyderm/src/server/pkg/transactionenv"
)

// APIServer represents a PPS API server
type APIServer interface {
	ppsclient.APIServer
	txnenv.PpsTransactionServer
}

// NewAPIServer creates an APIServer.
func NewAPIServer(
	env *serviceenv.ServiceEnv,
	txnEnv *txnenv.TransactionEnv,
	etcdPrefix string,
	namespace string,
	workerImage string,
	workerSidecarImage string,
	workerImagePullPolicy string,
	storageRoot string,
	storageBackend string,
	storageHostPath string,
	iamRole string,
	imagePullSecret string,
	noExposeDockerSocket bool,
	reporter *metrics.Reporter,
	workerUsesRoot bool,
	workerGrpcPort uint16,
	port uint16,
	httpPort uint16,
	peerPort uint16,
) (APIServer, error) {
	apiServer := &apiServer{
		Logger:                log.NewLogger("pps.API"),
		env:                   env,
		txnEnv:                txnEnv,
		etcdPrefix:            etcdPrefix,
		namespace:             namespace,
		workerImage:           workerImage,
		workerSidecarImage:    workerSidecarImage,
		workerImagePullPolicy: workerImagePullPolicy,
		storageRoot:           storageRoot,
		storageBackend:        storageBackend,
		storageHostPath:       storageHostPath,
		iamRole:               iamRole,
		imagePullSecret:       imagePullSecret,
		noExposeDockerSocket:  noExposeDockerSocket,
		reporter:              reporter,
		workerUsesRoot:        workerUsesRoot,
		pipelines:             ppsdb.Pipelines(env.GetEtcdClient(), etcdPrefix),
		jobs:                  ppsdb.Jobs(env.GetEtcdClient(), etcdPrefix),
		monitorCancels:        make(map[string]func()),
		workerGrpcPort:        workerGrpcPort,
		port:                  port,
		httpPort:              httpPort,
		peerPort:              peerPort,
	}
	apiServer.validateKube()
	go apiServer.master()
	return apiServer, nil
}

// NewSidecarAPIServer creates an APIServer that has limited functionalities
// and is meant to be run as a worker sidecar.  It cannot, for instance,
// create pipelines.
func NewSidecarAPIServer(
	env *serviceenv.ServiceEnv,
	etcdPrefix string,
	iamRole string,
	reporter *metrics.Reporter,
	workerGrpcPort uint16,
	httpPort uint16,
	peerPort uint16,
) (APIServer, error) {
	apiServer := &apiServer{
		Logger:         log.NewLogger("pps.API"),
		env:            env,
		etcdPrefix:     etcdPrefix,
		iamRole:        iamRole,
		reporter:       reporter,
		workerUsesRoot: true,
		pipelines:      ppsdb.Pipelines(env.GetEtcdClient(), etcdPrefix),
		jobs:           ppsdb.Jobs(env.GetEtcdClient(), etcdPrefix),
		workerGrpcPort: workerGrpcPort,
		httpPort:       httpPort,
		peerPort:       peerPort,
	}
	return apiServer, nil
}
<commit_msg>pps/server.go: NewSidecar sets APIServer.namespace<commit_after>package server

import (
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/log"
	"github.com/pachyderm/pachyderm/src/server/pkg/metrics"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsdb"
	"github.com/pachyderm/pachyderm/src/server/pkg/serviceenv"
	txnenv "github.com/pachyderm/pachyderm/src/server/pkg/transactionenv"
)

// APIServer represents a PPS API server
type APIServer interface {
	ppsclient.APIServer
	txnenv.PpsTransactionServer
}

// NewAPIServer creates an APIServer.
func NewAPIServer(
	env *serviceenv.ServiceEnv,
	txnEnv *txnenv.TransactionEnv,
	etcdPrefix string,
	namespace string,
	workerImage string,
	workerSidecarImage string,
	workerImagePullPolicy string,
	storageRoot string,
	storageBackend string,
	storageHostPath string,
	iamRole string,
	imagePullSecret string,
	noExposeDockerSocket bool,
	reporter *metrics.Reporter,
	workerUsesRoot bool,
	workerGrpcPort uint16,
	port uint16,
	httpPort uint16,
	peerPort uint16,
) (APIServer, error) {
	apiServer := &apiServer{
		Logger:                log.NewLogger("pps.API"),
		env:                   env,
		txnEnv:                txnEnv,
		etcdPrefix:            etcdPrefix,
		namespace:             namespace,
		workerImage:           workerImage,
		workerSidecarImage:    workerSidecarImage,
		workerImagePullPolicy: workerImagePullPolicy,
		storageRoot:           storageRoot,
		storageBackend:        storageBackend,
		storageHostPath:       storageHostPath,
		iamRole:               iamRole,
		imagePullSecret:       imagePullSecret,
		noExposeDockerSocket:  noExposeDockerSocket,
		reporter:              reporter,
		workerUsesRoot:        workerUsesRoot,
		pipelines:             ppsdb.Pipelines(env.GetEtcdClient(), etcdPrefix),
		jobs:                  ppsdb.Jobs(env.GetEtcdClient(), etcdPrefix),
		monitorCancels:        make(map[string]func()),
		workerGrpcPort:        workerGrpcPort,
		port:                  port,
		httpPort:              httpPort,
		peerPort:              peerPort,
	}
	apiServer.validateKube()
	go apiServer.master()
	return apiServer, nil
}

// NewSidecarAPIServer creates an APIServer that has limited functionalities
// and is meant to be run as a worker sidecar.  It cannot, for instance,
// create pipelines.
func NewSidecarAPIServer(
	env *serviceenv.ServiceEnv,
	etcdPrefix string,
	namespace string,
	iamRole string,
	reporter *metrics.Reporter,
	workerGrpcPort uint16,
	httpPort uint16,
	peerPort uint16,
) (APIServer, error) {
	apiServer := &apiServer{
		Logger:         log.NewLogger("pps.API"),
		env:            env,
		etcdPrefix:     etcdPrefix,
		iamRole:        iamRole,
		reporter:       reporter,
		namespace:      namespace,
		workerUsesRoot: true,
		pipelines:      ppsdb.Pipelines(env.GetEtcdClient(), etcdPrefix),
		jobs:           ppsdb.Jobs(env.GetEtcdClient(), etcdPrefix),
		workerGrpcPort: workerGrpcPort,
		httpPort:       httpPort,
		peerPort:       peerPort,
	}
	return apiServer, nil
}
<|endoftext|>