<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sqlparser

// analyzer.go contains utility analysis functions.

import (
	"fmt"
	"strings"
	"unicode"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/vterrors"

	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
)

// StatementType encodes the type of a SQL statement
type StatementType int

// These constants are used to identify the SQL statement type.
// Changing this list will require reviewing all calls to Preview.
const (
	StmtSelect StatementType = iota
	StmtStream
	StmtInsert
	StmtReplace
	StmtUpdate
	StmtDelete
	StmtDDL
	StmtBegin
	StmtCommit
	StmtRollback
	StmtSet
	StmtShow
	StmtUse
	StmtOther
	StmtUnknown
	StmtComment
	StmtPriv
	StmtExplain
	StmtSavepoint
	StmtSRollback
	StmtRelease
)

//ASTToStatementType returns a StatementType from an AST stmt
func ASTToStatementType(stmt Statement) StatementType {
	switch stmt.(type) {
	case *Select, *Union:
		return StmtSelect
	case *Insert:
		return StmtInsert
	case *Update:
		return StmtUpdate
	case *Delete:
		return StmtDelete
	case *Set:
		return StmtSet
	case *Show:
		return StmtShow
	case *DDL, *DBDDL:
		return StmtDDL
	case *Use:
		return StmtUse
	case *OtherRead, *OtherAdmin:
		return StmtOther
	case *Explain:
		return StmtExplain
	case *Begin:
		return StmtBegin
	case *Commit:
		return StmtCommit
	case *Rollback:
		return StmtRollback
	case *Savepoint:
		return StmtSavepoint
	case *SRollback:
		return StmtSRollback
	case *Release:
		return StmtRelease
	default:
		return StmtUnknown
	}
}

//CanNormalize takes Statement and returns if the statement can be normalized.
func CanNormalize(stmt Statement) bool {
	switch stmt.(type) {
	case *Select, *Union, *Insert, *Update, *Delete, *Set:
		return true
	}
	return false
}

//IsSetStatement takes Statement and returns if the statement is set statement.
func IsSetStatement(stmt Statement) bool {
	switch stmt.(type) {
	case *Set:
		return true
	}
	return false
}

// Preview analyzes the beginning of the query using a simpler and faster
// textual comparison to identify the statement type.
func Preview(sql string) StatementType {
	trimmed := StripLeadingComments(sql)

	if strings.Index(trimmed, "/*!") == 0 {
		return StmtComment
	}

	isNotLetter := func(r rune) bool { return !unicode.IsLetter(r) }
	firstWord := strings.TrimLeftFunc(trimmed, isNotLetter)

	if end := strings.IndexFunc(firstWord, unicode.IsSpace); end != -1 {
		firstWord = firstWord[:end]
	}
	// Comparison is done in order of priority.
	loweredFirstWord := strings.ToLower(firstWord)
	switch loweredFirstWord {
	case "select":
		return StmtSelect
	case "stream":
		return StmtStream
	case "insert":
		return StmtInsert
	case "replace":
		return StmtReplace
	case "update":
		return StmtUpdate
	case "delete":
		return StmtDelete
	case "savepoint":
		return StmtSavepoint
	}
	// For the following statements it is not sufficient to rely
	// on loweredFirstWord. This is because they are not statements
	// in the grammar and we are relying on Preview to parse them.
	// For instance, we don't want: "BEGIN JUNK" to be parsed
	// as StmtBegin.
	trimmedNoComments, _ := SplitMarginComments(trimmed)
	switch strings.ToLower(trimmedNoComments) {
	case "begin", "start transaction":
		return StmtBegin
	case "commit":
		return StmtCommit
	case "rollback":
		return StmtRollback
	}
	switch loweredFirstWord {
	case "create", "alter", "rename", "drop", "truncate", "flush":
		return StmtDDL
	case "set":
		return StmtSet
	case "show":
		return StmtShow
	case "use":
		return StmtUse
	case "describe", "desc", "explain":
		return StmtExplain
	case "analyze", "repair", "optimize":
		return StmtOther
	case "grant", "revoke":
		return StmtPriv
	case "savepoint":
		return StmtSavepoint
	case "release":
		return StmtRelease
	case "rollback":
		return StmtSRollback
	}
	return StmtUnknown
}

func (s StatementType) String() string {
	switch s {
	case StmtSelect:
		return "SELECT"
	case StmtStream:
		return "STREAM"
	case StmtInsert:
		return "INSERT"
	case StmtReplace:
		return "REPLACE"
	case StmtUpdate:
		return "UPDATE"
	case StmtDelete:
		return "DELETE"
	case StmtDDL:
		return "DDL"
	case StmtBegin:
		return "BEGIN"
	case StmtCommit:
		return "COMMIT"
	case StmtRollback:
		return "ROLLBACK"
	case StmtSet:
		return "SET"
	case StmtShow:
		return "SHOW"
	case StmtUse:
		return "USE"
	case StmtOther:
		return "OTHER"
	case StmtPriv:
		return "PRIV"
	case StmtExplain:
		return "EXPLAIN"
	case StmtSavepoint:
		return "SAVEPOINT"
	case StmtSRollback:
		return "SAVEPOINT_ROLLBACK"
	case StmtRelease:
		return "RELEASE"
	default:
		return "UNKNOWN"
	}
}

// IsDML returns true if the query is an INSERT, UPDATE or DELETE statement.
func IsDML(sql string) bool {
	switch Preview(sql) {
	case StmtInsert, StmtReplace, StmtUpdate, StmtDelete:
		return true
	}
	return false
}

//IsDMLStatement returns true if the query is an INSERT, UPDATE or DELETE statement.
func IsDMLStatement(stmt Statement) bool {
	switch stmt.(type) {
	case *Insert, *Update, *Delete:
		return true
	}

	return false
}

//IsVschemaDDL returns true if the query is an Vschema alter ddl.
func IsVschemaDDL(ddl *DDL) bool {
	switch ddl.Action {
	case CreateVindexStr, DropVindexStr, AddVschemaTableStr, DropVschemaTableStr, AddColVindexStr, DropColVindexStr, AddSequenceStr, AddAutoIncStr:
		return true
	}
	return false
}

// SplitAndExpression breaks up the Expr into AND-separated conditions
// and appends them to filters. Outer parenthesis are removed. Precedence
// should be taken into account if expressions are recombined.
func SplitAndExpression(filters []Expr, node Expr) []Expr {
	if node == nil {
		return filters
	}
	switch node := node.(type) {
	case *AndExpr:
		filters = SplitAndExpression(filters, node.Left)
		return SplitAndExpression(filters, node.Right)
	}
	return append(filters, node)
}

// TableFromStatement returns the qualified table name for the query.
// This works only for select statements.
func TableFromStatement(sql string) (TableName, error) {
	stmt, err := Parse(sql)
	if err != nil {
		return TableName{}, err
	}
	sel, ok := stmt.(*Select)
	if !ok {
		return TableName{}, fmt.Errorf("unrecognized statement: %s", sql)
	}
	if len(sel.From) != 1 {
		return TableName{}, fmt.Errorf("table expression is complex")
	}
	aliased, ok := sel.From[0].(*AliasedTableExpr)
	if !ok {
		return TableName{}, fmt.Errorf("table expression is complex")
	}
	tableName, ok := aliased.Expr.(TableName)
	if !ok {
		return TableName{}, fmt.Errorf("table expression is complex")
	}
	return tableName, nil
}

// GetTableName returns the table name from the SimpleTableExpr
// only if it's a simple expression. Otherwise, it returns "".
func GetTableName(node SimpleTableExpr) TableIdent {
	if n, ok := node.(TableName); ok && n.Qualifier.IsEmpty() {
		return n.Name
	}
	// sub-select or '.' expression
	return NewTableIdent("")
}

// IsColName returns true if the Expr is a *ColName.
func IsColName(node Expr) bool {
	_, ok := node.(*ColName)
	return ok
}

// IsValue returns true if the Expr is a string, integral or value arg.
// NULL is not considered to be a value.
func IsValue(node Expr) bool {
	switch v := node.(type) {
	case *SQLVal:
		switch v.Type {
		case StrVal, HexVal, IntVal, ValArg:
			return true
		}
	}
	return false
}

// IsNull returns true if the Expr is SQL NULL
func IsNull(node Expr) bool {
	switch node.(type) {
	case *NullVal:
		return true
	}
	return false
}

// IsSimpleTuple returns true if the Expr is a ValTuple that
// contains simple values or if it's a list arg.
func IsSimpleTuple(node Expr) bool {
	switch vals := node.(type) {
	case ValTuple:
		for _, n := range vals {
			if !IsValue(n) {
				return false
			}
		}
		return true
	case ListArg:
		return true
	}
	// It's a subquery
	return false
}

// NewPlanValue builds a sqltypes.PlanValue from an Expr.
func NewPlanValue(node Expr) (sqltypes.PlanValue, error) {
	switch node := node.(type) {
	case *SQLVal:
		switch node.Type {
		case ValArg:
			return sqltypes.PlanValue{Key: string(node.Val[1:])}, nil
		case IntVal:
			n, err := sqltypes.NewIntegral(string(node.Val))
			if err != nil {
				return sqltypes.PlanValue{}, err
			}
			return sqltypes.PlanValue{Value: n}, nil
		case FloatVal:
			return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.Float64, node.Val)}, nil
		case StrVal:
			return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val)}, nil
		case HexVal:
			v, err := node.HexDecode()
			if err != nil {
				return sqltypes.PlanValue{}, err
			}
			return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, v)}, nil
		}
	case ListArg:
		return sqltypes.PlanValue{ListKey: string(node[2:])}, nil
	case ValTuple:
		pv := sqltypes.PlanValue{
			Values: make([]sqltypes.PlanValue, 0, len(node)),
		}
		for _, val := range node {
			innerpv, err := NewPlanValue(val)
			if err != nil {
				return sqltypes.PlanValue{}, err
			}
			if innerpv.ListKey != "" || innerpv.Values != nil {
				return sqltypes.PlanValue{}, vterrors.New(vtrpcpb.Code_UNIMPLEMENTED, "unsupported: nested lists")
			}
			pv.Values = append(pv.Values, innerpv)
		}
		return pv, nil
	case *NullVal:
		return sqltypes.PlanValue{}, nil
	case *UnaryExpr:
		switch node.Operator {
		case UBinaryStr, Utf8mb4Str, Utf8Str, Latin1Str: // for some charset introducers, we can just ignore them
			return NewPlanValue(node.Expr)
		}
	}
	return sqltypes.PlanValue{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "expression is too complex '%v'", String(node))
}
<commit_msg>Vrepl Savepoint: remove duplicate switch<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sqlparser

// analyzer.go contains utility analysis functions.

import (
	"fmt"
	"strings"
	"unicode"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/vterrors"

	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
)

// StatementType encodes the type of a SQL statement
type StatementType int

// These constants are used to identify the SQL statement type.
// Changing this list will require reviewing all calls to Preview.
const (
	StmtSelect StatementType = iota
	StmtStream
	StmtInsert
	StmtReplace
	StmtUpdate
	StmtDelete
	StmtDDL
	StmtBegin
	StmtCommit
	StmtRollback
	StmtSet
	StmtShow
	StmtUse
	StmtOther
	StmtUnknown
	StmtComment
	StmtPriv
	StmtExplain
	StmtSavepoint
	StmtSRollback
	StmtRelease
)

//ASTToStatementType returns a StatementType from an AST stmt
func ASTToStatementType(stmt Statement) StatementType {
	switch stmt.(type) {
	case *Select, *Union:
		return StmtSelect
	case *Insert:
		return StmtInsert
	case *Update:
		return StmtUpdate
	case *Delete:
		return StmtDelete
	case *Set:
		return StmtSet
	case *Show:
		return StmtShow
	case *DDL, *DBDDL:
		return StmtDDL
	case *Use:
		return StmtUse
	case *OtherRead, *OtherAdmin:
		return StmtOther
	case *Explain:
		return StmtExplain
	case *Begin:
		return StmtBegin
	case *Commit:
		return StmtCommit
	case *Rollback:
		return StmtRollback
	case *Savepoint:
		return StmtSavepoint
	case *SRollback:
		return StmtSRollback
	case *Release:
		return StmtRelease
	default:
		return StmtUnknown
	}
}

//CanNormalize takes Statement and returns if the statement can be normalized.
func CanNormalize(stmt Statement) bool {
	switch stmt.(type) {
	case *Select, *Union, *Insert, *Update, *Delete, *Set:
		return true
	}
	return false
}

//IsSetStatement takes Statement and returns if the statement is set statement.
func IsSetStatement(stmt Statement) bool {
	switch stmt.(type) {
	case *Set:
		return true
	}
	return false
}

// Preview analyzes the beginning of the query using a simpler and faster
// textual comparison to identify the statement type.
func Preview(sql string) StatementType {
	trimmed := StripLeadingComments(sql)

	if strings.Index(trimmed, "/*!") == 0 {
		return StmtComment
	}

	isNotLetter := func(r rune) bool { return !unicode.IsLetter(r) }
	firstWord := strings.TrimLeftFunc(trimmed, isNotLetter)

	if end := strings.IndexFunc(firstWord, unicode.IsSpace); end != -1 {
		firstWord = firstWord[:end]
	}
	// Comparison is done in order of priority.
	loweredFirstWord := strings.ToLower(firstWord)
	switch loweredFirstWord {
	case "select":
		return StmtSelect
	case "stream":
		return StmtStream
	case "insert":
		return StmtInsert
	case "replace":
		return StmtReplace
	case "update":
		return StmtUpdate
	case "delete":
		return StmtDelete
	case "savepoint":
		return StmtSavepoint
	}
	// For the following statements it is not sufficient to rely
	// on loweredFirstWord. This is because they are not statements
	// in the grammar and we are relying on Preview to parse them.
	// For instance, we don't want: "BEGIN JUNK" to be parsed
	// as StmtBegin.
	trimmedNoComments, _ := SplitMarginComments(trimmed)
	switch strings.ToLower(trimmedNoComments) {
	case "begin", "start transaction":
		return StmtBegin
	case "commit":
		return StmtCommit
	case "rollback":
		return StmtRollback
	}
	switch loweredFirstWord {
	case "create", "alter", "rename", "drop", "truncate", "flush":
		return StmtDDL
	case "set":
		return StmtSet
	case "show":
		return StmtShow
	case "use":
		return StmtUse
	case "describe", "desc", "explain":
		return StmtExplain
	case "analyze", "repair", "optimize":
		return StmtOther
	case "grant", "revoke":
		return StmtPriv
	case "release":
		return StmtRelease
	case "rollback":
		return StmtSRollback
	}
	return StmtUnknown
}

func (s StatementType) String() string {
	switch s {
	case StmtSelect:
		return "SELECT"
	case StmtStream:
		return "STREAM"
	case StmtInsert:
		return "INSERT"
	case StmtReplace:
		return "REPLACE"
	case StmtUpdate:
		return "UPDATE"
	case StmtDelete:
		return "DELETE"
	case StmtDDL:
		return "DDL"
	case StmtBegin:
		return "BEGIN"
	case StmtCommit:
		return "COMMIT"
	case StmtRollback:
		return "ROLLBACK"
	case StmtSet:
		return "SET"
	case StmtShow:
		return "SHOW"
	case StmtUse:
		return "USE"
	case StmtOther:
		return "OTHER"
	case StmtPriv:
		return "PRIV"
	case StmtExplain:
		return "EXPLAIN"
	case StmtSavepoint:
		return "SAVEPOINT"
	case StmtSRollback:
		return "SAVEPOINT_ROLLBACK"
	case StmtRelease:
		return "RELEASE"
	default:
		return "UNKNOWN"
	}
}

// IsDML returns true if the query is an INSERT, UPDATE or DELETE statement.
func IsDML(sql string) bool {
	switch Preview(sql) {
	case StmtInsert, StmtReplace, StmtUpdate, StmtDelete:
		return true
	}
	return false
}

//IsDMLStatement returns true if the query is an INSERT, UPDATE or DELETE statement.
func IsDMLStatement(stmt Statement) bool {
	switch stmt.(type) {
	case *Insert, *Update, *Delete:
		return true
	}

	return false
}

//IsVschemaDDL returns true if the query is an Vschema alter ddl.
func IsVschemaDDL(ddl *DDL) bool {
	switch ddl.Action {
	case CreateVindexStr, DropVindexStr, AddVschemaTableStr, DropVschemaTableStr, AddColVindexStr, DropColVindexStr, AddSequenceStr, AddAutoIncStr:
		return true
	}
	return false
}

// SplitAndExpression breaks up the Expr into AND-separated conditions
// and appends them to filters. Outer parenthesis are removed. Precedence
// should be taken into account if expressions are recombined.
func SplitAndExpression(filters []Expr, node Expr) []Expr {
	if node == nil {
		return filters
	}
	switch node := node.(type) {
	case *AndExpr:
		filters = SplitAndExpression(filters, node.Left)
		return SplitAndExpression(filters, node.Right)
	}
	return append(filters, node)
}

// TableFromStatement returns the qualified table name for the query.
// This works only for select statements.
func TableFromStatement(sql string) (TableName, error) {
	stmt, err := Parse(sql)
	if err != nil {
		return TableName{}, err
	}
	sel, ok := stmt.(*Select)
	if !ok {
		return TableName{}, fmt.Errorf("unrecognized statement: %s", sql)
	}
	if len(sel.From) != 1 {
		return TableName{}, fmt.Errorf("table expression is complex")
	}
	aliased, ok := sel.From[0].(*AliasedTableExpr)
	if !ok {
		return TableName{}, fmt.Errorf("table expression is complex")
	}
	tableName, ok := aliased.Expr.(TableName)
	if !ok {
		return TableName{}, fmt.Errorf("table expression is complex")
	}
	return tableName, nil
}

// GetTableName returns the table name from the SimpleTableExpr
// only if it's a simple expression. Otherwise, it returns "".
func GetTableName(node SimpleTableExpr) TableIdent {
	if n, ok := node.(TableName); ok && n.Qualifier.IsEmpty() {
		return n.Name
	}
	// sub-select or '.' expression
	return NewTableIdent("")
}

// IsColName returns true if the Expr is a *ColName.
func IsColName(node Expr) bool {
	_, ok := node.(*ColName)
	return ok
}

// IsValue returns true if the Expr is a string, integral or value arg.
// NULL is not considered to be a value.
func IsValue(node Expr) bool {
	switch v := node.(type) {
	case *SQLVal:
		switch v.Type {
		case StrVal, HexVal, IntVal, ValArg:
			return true
		}
	}
	return false
}

// IsNull returns true if the Expr is SQL NULL
func IsNull(node Expr) bool {
	switch node.(type) {
	case *NullVal:
		return true
	}
	return false
}

// IsSimpleTuple returns true if the Expr is a ValTuple that
// contains simple values or if it's a list arg.
func IsSimpleTuple(node Expr) bool {
	switch vals := node.(type) {
	case ValTuple:
		for _, n := range vals {
			if !IsValue(n) {
				return false
			}
		}
		return true
	case ListArg:
		return true
	}
	// It's a subquery
	return false
}

// NewPlanValue builds a sqltypes.PlanValue from an Expr.
func NewPlanValue(node Expr) (sqltypes.PlanValue, error) {
	switch node := node.(type) {
	case *SQLVal:
		switch node.Type {
		case ValArg:
			return sqltypes.PlanValue{Key: string(node.Val[1:])}, nil
		case IntVal:
			n, err := sqltypes.NewIntegral(string(node.Val))
			if err != nil {
				return sqltypes.PlanValue{}, err
			}
			return sqltypes.PlanValue{Value: n}, nil
		case FloatVal:
			return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.Float64, node.Val)}, nil
		case StrVal:
			return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val)}, nil
		case HexVal:
			v, err := node.HexDecode()
			if err != nil {
				return sqltypes.PlanValue{}, err
			}
			return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, v)}, nil
		}
	case ListArg:
		return sqltypes.PlanValue{ListKey: string(node[2:])}, nil
	case ValTuple:
		pv := sqltypes.PlanValue{
			Values: make([]sqltypes.PlanValue, 0, len(node)),
		}
		for _, val := range node {
			innerpv, err := NewPlanValue(val)
			if err != nil {
				return sqltypes.PlanValue{}, err
			}
			if innerpv.ListKey != "" || innerpv.Values != nil {
				return sqltypes.PlanValue{}, vterrors.New(vtrpcpb.Code_UNIMPLEMENTED, "unsupported: nested lists")
			}
			pv.Values = append(pv.Values, innerpv)
		}
		return pv, nil
	case *NullVal:
		return sqltypes.PlanValue{}, nil
	case *UnaryExpr:
		switch node.Operator {
		case UBinaryStr, Utf8mb4Str, Utf8Str, Latin1Str: // for some charset introducers, we can just ignore them
			return NewPlanValue(node.Expr)
		}
	}
	return sqltypes.PlanValue{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "expression is too complex '%v'", String(node))
}
<|endoftext|>


<commit_before>package ui

import "hivejam/config"
import "hivejam/data"

import "log"
import "net/http"
import "strconv"
import "strings"

import _ "github.com/jteeuwen/go-bindata"

func configHandler(w http.ResponseWriter, r *http.Request) {
	jsConfig, err := config.JsConfig()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
	w.Write(jsConfig)
}

func jsHandler(w http.ResponseWriter, r *http.Request) {
	data, err := data.Asset("hive-jam/resources/public/js/compiled/hive_jam.js")
	if err != nil {
		http.NotFound(w, r)
		return
	}
	dataString := string(data)
	dataString = strings.Replace(dataString, "UI_BRIDGE_PORT", strconv.Itoa(*config.Flags.UiBridgePort), -1)
	w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
	w.Write([]byte(dataString))
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	resource, err := data.Asset("hive-jam/resources/public/index.html")
	if err != nil {
		http.NotFound(w, r)
		return
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.Write(resource)
}

func cssHandler(w http.ResponseWriter, r *http.Request) {
	resourceId := r.URL.Path[len("/css/"):]
	resource, err := data.Asset("hive-jam/resources/public/css/"+resourceId)
	if err != nil {
		http.NotFound(w, r)
		return
	}
	w.Header().Set("Content-Type", "text/css; charset=utf-8")
	w.Write(resource)
}

func Serve(done chan bool) {
	http.HandleFunc("/", rootHandler)
	http.HandleFunc("/config/config.js", configHandler)
	http.HandleFunc("/js/compiled/hive_jam.js", jsHandler)
	http.HandleFunc("/css/", cssHandler)
	endpoint := *config.Flags.UiIp+":"+strconv.Itoa(*config.Flags.UiPort)
	externalEndpoint := *config.Flags.UiExternalIp+":"+strconv.Itoa(*config.Flags.UiPort)
	log.Print("Starting the UI webserver on "+externalEndpoint)
	http.ListenAndServe(endpoint, nil)
}

func Shutdown() {
	log.Print("Shutting down.")
}
<commit_msg>Fix rename error in UI server.<commit_after>package ui

import "hivejam/config"
import "hivejam/data"

import "log"
import "net/http"
import "strconv"
import "strings"

import _ "github.com/jteeuwen/go-bindata"

func configHandler(w http.ResponseWriter, r *http.Request) {
	jsConfig, err := config.JsConfig()
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
	w.Write(jsConfig)
}

func jsHandler(w http.ResponseWriter, r *http.Request) {
	data, err := data.Asset("cljs/resources/public/js/compiled/hive_jam.js")
	if err != nil {
		http.NotFound(w, r)
		return
	}
	dataString := string(data)
	dataString = strings.Replace(dataString, "UI_BRIDGE_PORT", strconv.Itoa(*config.Flags.UiBridgePort), -1)
	w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
	w.Write([]byte(dataString))
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	resource, err := data.Asset("cljs/resources/public/index.html")
	if err != nil {
		http.NotFound(w, r)
		return
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.Write(resource)
}

func cssHandler(w http.ResponseWriter, r *http.Request) {
	resourceId := r.URL.Path[len("/css/"):]
	resource, err := data.Asset("cljs/resources/public/css/"+resourceId)
	if err != nil {
		http.NotFound(w, r)
		return
	}
	w.Header().Set("Content-Type", "text/css; charset=utf-8")
	w.Write(resource)
}

func Serve(done chan bool) {
	http.HandleFunc("/", rootHandler)
	http.HandleFunc("/config/config.js", configHandler)
	http.HandleFunc("/js/compiled/hive_jam.js", jsHandler)
	http.HandleFunc("/css/", cssHandler)
	endpoint := *config.Flags.UiIp+":"+strconv.Itoa(*config.Flags.UiPort)
	externalEndpoint := *config.Flags.UiExternalIp+":"+strconv.Itoa(*config.Flags.UiPort)
	log.Print("Starting the UI webserver on "+externalEndpoint)
	http.ListenAndServe(endpoint, nil)
}

func Shutdown() {
	log.Print("Shutting down.")
}
<|endoftext|>


<commit_before>// Package boolq lets you build generic query expressions.

package boolq

import (
	"fmt"

	"github.com/kylebrandt/boolq/parse"
)

// An Asker is something that can be queried using boolq. The string passed
// to Ask will be the component in an expression. For example with the expression
// `(foo:bar AND baz:biz)` Ask will be called twice, once with the argument "foo:bar"
// and another time with the argument "baz:biz"
type Asker interface {
	Ask(string) (bool, error)
}

// AskExpr takes an expression and an Asker. It then parses the expression
// calling the Asker's Ask on expressions AskNodes and returns if the
// expression is true or not for the given asker.
func AskExpr(expr string, asker Asker) (bool, error) {
	q, err := parse.Parse(expr)
	if err != nil {
		return false, err
	}
	return walk(q.Root, asker)
}

// AskParsedExpr is like AskExpr but takes an expression that has already
// been parsed by parse.Parse on the expression. This is useful if you are calling
// the same expression multiple times.
func AskParsedExpr(q *Tree, asker Asker) (bool, error) {
	return walk(q.Root, asker)
}

type Tree struct {
	*parse.Tree
}

// Parse parses an expression and returns the parsed expression.
// It can be used wtih AskParsedExpr
func Parse(text string) (*Tree, error) {
	tree := &Tree{}
	var err error
	tree.Tree, err = parse.Parse(text)
	return tree, err
}

func walk(node parse.Node, asker Asker) (bool, error) {
	switch node := node.(type) {
	case *parse.AskNode:
		return asker.Ask(node.Text)
	case *parse.BinaryNode:
		return walkBinary(node, asker)
	case *parse.UnaryNode:
		return walkUnary(node, asker)
	default:
		return false, fmt.Errorf("can not walk type %v", node.Type())
	}
}

func walkBinary(node *parse.BinaryNode, asker Asker) (bool, error) {
	l, err := walk(node.Args[0], asker)
	if err != nil {
		return false, err
	}
	r, err := walk(node.Args[1], asker)
	if err != nil {
		return false, err
	}
	if node.OpStr == "AND" {
		return l && r, nil
	}
	if node.OpStr == "OR" {
		return l || r, nil
	}
	return false, fmt.Errorf("Unrecognized operator: %v", node.OpStr)
}

func walkUnary(node *parse.UnaryNode, asker Asker) (bool, error) {
	r, err := walk(node.Arg, asker)
	if err != nil {
		return false, err
	}
	if node.OpStr == "!" {
		return !r, nil
	}
	return false, fmt.Errorf("unknown unary operator: %v", node.OpStr)
}
<commit_msg>return true on empty expression.<commit_after>// Package boolq lets you build generic query expressions.

package boolq

import (
	"fmt"

	"github.com/kylebrandt/boolq/parse"
)

// An Asker is something that can be queried using boolq. The string passed
// to Ask will be the component in an expression. For example with the expression
// `(foo:bar AND baz:biz)` Ask will be called twice, once with the argument "foo:bar"
// and another time with the argument "baz:biz"
type Asker interface {
	Ask(string) (bool, error)
}

// AskExpr takes an expression and an Asker. It then parses the expression
// calling the Asker's Ask on expressions AskNodes and returns if the
// expression is true or not for the given asker.
func AskExpr(expr string, asker Asker) (bool, error) {
	q, err := parse.Parse(expr)
	if err != nil {
		return false, err
	}
	return walk(q.Root, asker)
}

// AskParsedExpr is like AskExpr but takes an expression that has already
// been parsed by parse.Parse on the expression. This is useful if you are calling
// the same expression multiple times.
func AskParsedExpr(q *Tree, asker Asker) (bool, error) {
	if q.Tree.Root == nil {
		return true, nil
	}
	return walk(q.Root, asker)
}

type Tree struct {
	*parse.Tree
}

// Parse parses an expression and returns the parsed expression.
// It can be used wtih AskParsedExpr
func Parse(text string) (*Tree, error) {
	tree := &Tree{}
	if text == "" {
		tree.Tree = &parse.Tree{}
		return tree, nil
	}
	var err error
	tree.Tree, err = parse.Parse(text)
	return tree, err
}

func walk(node parse.Node, asker Asker) (bool, error) {
	switch node := node.(type) {
	case *parse.AskNode:
		return asker.Ask(node.Text)
	case *parse.BinaryNode:
		return walkBinary(node, asker)
	case *parse.UnaryNode:
		return walkUnary(node, asker)
	default:
		return false, fmt.Errorf("can not walk type %v", node.Type())
	}
}

func walkBinary(node *parse.BinaryNode, asker Asker) (bool, error) {
	l, err := walk(node.Args[0], asker)
	if err != nil {
		return false, err
	}
	r, err := walk(node.Args[1], asker)
	if err != nil {
		return false, err
	}
	if node.OpStr == "AND" {
		return l && r, nil
	}
	if node.OpStr == "OR" {
		return l || r, nil
	}
	return false, fmt.Errorf("Unrecognized operator: %v", node.OpStr)
}

func walkUnary(node *parse.UnaryNode, asker Asker) (bool, error) {
	r, err := walk(node.Arg, asker)
	if err != nil {
		return false, err
	}
	if node.OpStr == "!" {
		return !r, nil
	}
	return false, fmt.Errorf("unknown unary operator: %v", node.OpStr)
}
<|endoftext|>


<commit_before>package dashboard

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/Cepave/fe/http/base"
	"github.com/Cepave/fe/model/dashboard"
	"github.com/Cepave/fe/model/uic"
)

type DashBoardController struct {
	base.BaseController
}

func (this *DashBoardController) EndpRegxqury() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	queryStr := this.GetString("queryStr", "")
	if queryStr == "" {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	limitNum, _ := this.GetInt("limit", 0)
	enp, err := dashboard.QueryEndpintByNameRegx(queryStr, limitNum)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(enp) > 0 {
		baseResp.Data["endpoints"] = enp
	} else {
		baseResp.Data["endpoints"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

//counter query by endpoints
func (this *DashBoardController) CounterQuery() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	endpoints := this.GetString("endpoints", "")
	endpointcheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if endpoints == "" || endpointcheck.MatchString(endpoints) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	endpointsArr := strings.Split(rexstr.ReplaceAllString(endpoints, ""), ",")
	limitNum, _ := this.GetInt("limit", 0)
	counters, err := dashboard.QueryCounterByEndpoints(endpointsArr, limitNum)
	switch {
	case err != nil:
		this.ResposeError(baseResp, err.Error())
		return
	case len(counters) == 0:
		baseResp.Data["counters"] = []string{}
	default:
		baseResp.Data["counters"] = counters
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) HostGroupQuery() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	queryStr := this.GetString("queryStr", "")
	if queryStr == "" {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	limitNum, _ := this.GetInt("limit", 0)
	hostgroupList, err := dashboard.QueryHostGroupByNameRegx(queryStr, limitNum)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(hostgroupList) > 0 {
		baseResp.Data["hostgroups"] = hostgroupList
	} else {
		baseResp.Data["hostgroups"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) HostsQueryByHostGroups() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	hostgroups := this.GetString("hostgroups", "")
	hostgroupscheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if hostgroups == "" || hostgroupscheck.MatchString(hostgroups) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	hostgroupsArr := strings.Split(rexstr.ReplaceAllString(hostgroups, ""), ",")
	hosts_resp, err := dashboard.GetHostsByHostGroupName(hostgroupsArr)

	if len(hosts_resp) > 0 {
		baseResp.Data["hosts"] = hosts_resp
	} else {
		baseResp.Data["hosts"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) CounterQueryByHostGroup() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	hostgroups := this.GetString("hostgroups", "")
	hostgroupscheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if hostgroups == "" || hostgroupscheck.MatchString(hostgroups) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	hostgroupsArr := strings.Split(rexstr.ReplaceAllString(hostgroups, ""), ",")

	hosts, err := dashboard.GetHostsByHostGroupName(hostgroupsArr)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(hosts) > 0 {
		var endpoints []string
		for _, v := range hosts {
			endpoints = append(endpoints, fmt.Sprintf("\"%v\"", v.Hostname))
		}
		limitNum, _ := this.GetInt("limit", 0)
		counters, err := dashboard.QueryCounterByEndpoints(endpoints, limitNum)
		if err != nil {
			this.ResposeError(baseResp, err.Error())
			return
		} else if len(counters) > 0 {
			baseResp.Data["counters"] = counters
		} else {
			baseResp.Data["counters"] = []string{}
		}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) CountNumOfHostGroup() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()

	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	} else {
		numberOfteam, err := dashboard.CountNumOfHostGroup()
		if err != nil {
			this.ResposeError(baseResp, err.Error())
			return
		}
		baseResp.Data["count"] = numberOfteam
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) EndpRegxquryForOps() {
	sig := this.Ctx.GetCookie("sig")
	session := uic.ReadSessionBySig(sig)
	if sig == "" || session.Uid <= 0 {
		this.Data["SessionFlag"] = true
		this.Data["ErrorMsg"] = "Session is not vaild"
	} else {
		this.Data["SessionFlag"] = false
	}
	username := uic.SelectUserById(session.Uid)
	if username.Name != "root" {
		this.Data["SessionFlag"] = true
		this.Data["ErrorMsg"] = "You don't have permission to access this page"
	}

	queryStr := this.GetString("queryStr", "")
	this.Data["QueryCondstion"] = queryStr
	if queryStr == "" || this.Data["SessionFlag"] == true {
		this.Data["Init"] = true
	} else {
		enp, _ := dashboard.QueryEndpintByNameRegxForOps(queryStr)
		if len(enp) > 0 {
			var ips []string
			this.Data["Endopints"] = enp
			this.Data["Len"] = len(enp)
			for _, en := range enp {
				if en.Ip != "" {
					ips = append(ips, en.Ip)
				}
			}
			this.Data["IP"] = strings.Join(ips, ",")
		} else {
			this.Data["Endopints"] = []string{}
			this.Data["Len"] = 0
			this.Data["IP"] = ""
		}
	}
	this.TplName = "dashboard/endpoints.html"
}
<commit_msg>fix issue: 1. "/ops/endpoints" got error when user not login yet. 2. site bar links broken.<commit_after>package dashboard

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/Cepave/fe/g"
	"github.com/Cepave/fe/http/base"
	"github.com/Cepave/fe/model/dashboard"
	"github.com/Cepave/fe/model/uic"
)

type DashBoardController struct {
	base.BaseController
}

func (this *DashBoardController) EndpRegxqury() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	queryStr := this.GetString("queryStr", "")
	if queryStr == "" {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	limitNum, _ := this.GetInt("limit", 0)
	enp, err := dashboard.QueryEndpintByNameRegx(queryStr, limitNum)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(enp) > 0 {
		baseResp.Data["endpoints"] = enp
	} else {
		baseResp.Data["endpoints"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

//counter query by endpoints
func (this *DashBoardController) CounterQuery() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	endpoints := this.GetString("endpoints", "")
	endpointcheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if endpoints == "" || endpointcheck.MatchString(endpoints) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	endpointsArr := strings.Split(rexstr.ReplaceAllString(endpoints, ""), ",")
	limitNum, _ := this.GetInt("limit", 0)
	counters, err := dashboard.QueryCounterByEndpoints(endpointsArr, limitNum)
	switch {
	case err != nil:
		this.ResposeError(baseResp, err.Error())
		return
	case len(counters) == 0:
		baseResp.Data["counters"] = []string{}
	default:
		baseResp.Data["counters"] = counters
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) HostGroupQuery() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	queryStr := this.GetString("queryStr", "")
	if queryStr == "" {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	limitNum, _ := this.GetInt("limit", 0)
	hostgroupList, err := dashboard.QueryHostGroupByNameRegx(queryStr, limitNum)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(hostgroupList) > 0 {
		baseResp.Data["hostgroups"] = hostgroupList
	} else {
		baseResp.Data["hostgroups"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) HostsQueryByHostGroups() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	hostgroups := this.GetString("hostgroups", "")
	hostgroupscheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if hostgroups == "" || hostgroupscheck.MatchString(hostgroups) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	hostgroupsArr := strings.Split(rexstr.ReplaceAllString(hostgroups, ""), ",")
	hosts_resp, err := dashboard.GetHostsByHostGroupName(hostgroupsArr)

	if len(hosts_resp) > 0 {
		baseResp.Data["hosts"] = hosts_resp
	} else {
		baseResp.Data["hosts"] = []string{}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) CounterQueryByHostGroup() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}

	hostgroups := this.GetString("hostgroups", "")
	hostgroupscheck, _ := regexp.Compile("^\\s*\\[\\s*\\]\\s*$")
	if hostgroups == "" || hostgroupscheck.MatchString(hostgroups) {
		this.ResposeError(baseResp, "query string is empty, please check it")
		return
	}
	rexstr, _ := regexp.Compile("^\\s*\\[\\s*|\\s*\\]\\s*$")
	hostgroupsArr := strings.Split(rexstr.ReplaceAllString(hostgroups, ""), ",")

	hosts, err := dashboard.GetHostsByHostGroupName(hostgroupsArr)
	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	}
	if len(hosts) > 0 {
		var endpoints []string
		for _, v := range hosts {
			endpoints = append(endpoints, fmt.Sprintf("\"%v\"", v.Hostname))
		}
		limitNum, _ := this.GetInt("limit", 0)
		counters, err := dashboard.QueryCounterByEndpoints(endpoints, limitNum)
		if err != nil {
			this.ResposeError(baseResp, err.Error())
			return
		} else if len(counters) > 0 {
			baseResp.Data["counters"] = counters
		} else {
			baseResp.Data["counters"] = []string{}
		}
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) CountNumOfHostGroup() {
	baseResp := this.BasicRespGen()
	_, err := this.SessionCheck()

	if err != nil {
		this.ResposeError(baseResp, err.Error())
		return
	} else {
		numberOfteam, err := dashboard.CountNumOfHostGroup()
		if err != nil {
			this.ResposeError(baseResp, err.Error())
			return
		}
		baseResp.Data["count"] = numberOfteam
	}
	this.ServeApiJson(baseResp)
	return
}

func (this *DashBoardController) EndpRegxquryForOps() {
	this.Data["Shortcut"] = g.Config().Shortcut
	sig := this.Ctx.GetCookie("sig")
	session := uic.ReadSessionBySig(sig)
	var username *uic.User
	if sig == "" || session.Uid <= 0 {
		this.Data["SessionFlag"] = true
		this.Data["ErrorMsg"] = "Session is not vaild"
	} else {
		this.Data["SessionFlag"] = false
		username = uic.SelectUserById(session.Uid)
		if username.Name != "root" {
			this.Data["SessionFlag"] = true
			this.Data["ErrorMsg"] = "You don't have permission to access this page"
		}
	}
	queryStr := this.GetString("queryStr", "")
	this.Data["QueryCondstion"] = queryStr
	if queryStr == "" || this.Data["SessionFlag"] == true {
		this.Data["Init"] = true
	} else {
		enp, _ := dashboard.QueryEndpintByNameRegxForOps(queryStr)
		if len(enp) > 0 {
			var ips []string
			this.Data["Endopints"] = enp
			this.Data["Len"] = len(enp)
			for _, en := range enp {
				if en.Ip != "" {
					ips = append(ips, en.Ip)
				}
			}
			this.Data["IP"] = strings.Join(ips, ",")
		} else {
			this.Data["Endopints"] = []string{}
			this.Data["Len"] = 0
			this.Data["IP"] = ""
		}
	}
	this.TplName = "dashboard/endpoints.html"
}
<|endoftext|>


<commit_before>package web

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/evepraisal/go-evepraisal"

	"golang.org/x/oauth2"
)

// GetCurrentUser returns the current user, if there is one in the request cookies
func (ctx *Context) GetCurrentUser(r *http.Request) *evepraisal.User {
	user, ok := ctx.getSessionValue(r, "user").(evepraisal.User)
	if !ok {
		return nil
	}
	return &user
}

// IsAppraisalOwner determines if a user owns the given appraisal
func IsAppraisalOwner(user *evepraisal.User, appraisal *evepraisal.Appraisal) bool {
	if user == nil {
		return false
	}

	if appraisal.User == nil {
		return false
	}

	return appraisal.User.CharacterOwnerHash == user.CharacterOwnerHash
}

// HandleLogin handles /login
func (ctx *Context) HandleLogin(w http.ResponseWriter, r *http.Request) {
	if ctx.OauthConfig == nil {
		ctx.renderErrorPage(r, w, http.StatusBadRequest, "Feature Unavailable", "SSO is not configured")
		return
	}
	url := ctx.OauthConfig.AuthCodeURL("state", oauth2.AccessTypeOnline)
	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
}

// HandleLogout handles /logout
func (ctx *Context) HandleLogout(w http.ResponseWriter, r *http.Request) {
	if ctx.OauthConfig == nil {
		ctx.renderErrorPage(r, w, http.StatusBadRequest, "Feature Unavailable", "SSO is not configured")
		return
	}
	ctx.setSessionValue(r, w, "user", nil)
	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}

// HandleAuthCallback handles /oauthcallback
func (ctx *Context) HandleAuthCallback(w http.ResponseWriter, r *http.Request) {
	if ctx.OauthConfig == nil {
		ctx.renderErrorPage(r, w, http.StatusBadRequest, "Feature Unavailable", "SSO is not configured")
		return
	}
	code := r.FormValue("code")
	tok, err := ctx.OauthConfig.Exchange(r.Context(), code)
	if err != nil {
		ctx.renderServerError(r, w, err)
	}

	client := ctx.OauthConfig.Client(r.Context(), tok)
	resp, err := client.Get(ctx.OauthVerifyURL)
	if err != nil {
		ctx.renderServerError(r, w, err)
	}

	if resp.StatusCode != 200 {
		ctx.renderErrorPage(r, w, http.StatusInternalServerError, "Something bad happened", fmt.Sprintf("Invalid response when verifying identity (%d)", resp.StatusCode))
	}

	user := &evepraisal.User{}
	err = json.NewDecoder(resp.Body).Decode(user)
	defer resp.Body.Close()
	if err != nil {
		ctx.renderServerError(r, w, err)
	}

	ctx.setSessionValue(r, w, "user", &user)
	log.Printf("User logged in: %s", user.CharacterName)

	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}
<commit_msg>Fix error handling with failures during auth<commit_after>package web

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/evepraisal/go-evepraisal"

	"golang.org/x/oauth2"
)

// GetCurrentUser returns the current user, if there is one in the request cookies
func (ctx *Context) GetCurrentUser(r *http.Request) *evepraisal.User {
	user, ok := ctx.getSessionValue(r, "user").(evepraisal.User)
	if !ok {
		return nil
	}
	return &user
}

// IsAppraisalOwner determines if a user owns the given appraisal
func IsAppraisalOwner(user *evepraisal.User, appraisal *evepraisal.Appraisal) bool {
	if user == nil {
		return false
	}

	if appraisal.User == nil {
		return false
	}

	return appraisal.User.CharacterOwnerHash == user.CharacterOwnerHash
}

// HandleLogin handles /login
func (ctx *Context) HandleLogin(w http.ResponseWriter, r *http.Request) {
	if ctx.OauthConfig == nil {
		ctx.renderErrorPage(r, w, http.StatusBadRequest, "Feature Unavailable", "SSO is not configured")
		return
	}
	url := ctx.OauthConfig.AuthCodeURL("state", oauth2.AccessTypeOnline)
	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
}

// HandleLogout handles /logout
func (ctx *Context) HandleLogout(w http.ResponseWriter, r *http.Request) {
	if ctx.OauthConfig == nil {
		ctx.renderErrorPage(r, w, http.StatusBadRequest, "Feature Unavailable", "SSO is not configured")
		return
	}
	ctx.setSessionValue(r, w, "user", nil)
	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}

// HandleAuthCallback handles /oauthcallback
func (ctx *Context) HandleAuthCallback(w http.ResponseWriter, r *http.Request) {
	if ctx.OauthConfig == nil {
		ctx.renderErrorPage(r, w, http.StatusBadRequest, "Feature Unavailable", "SSO is not configured")
		return
	}
	code := r.FormValue("code")
	tok, err := ctx.OauthConfig.Exchange(r.Context(), code)
	if err != nil {
		ctx.renderServerError(r, w, err)
	}

	client := ctx.OauthConfig.Client(r.Context(), tok)
	resp, err := client.Get(ctx.OauthVerifyURL)
	if err != nil {
		ctx.renderServerError(r, w, err)
		return
	}

	if resp.StatusCode != 200 {
		ctx.renderErrorPage(r, w, http.StatusInternalServerError, "Something bad happened", fmt.Sprintf("Invalid response when verifying identity (%d)", resp.StatusCode))
		return
	}

	user := &evepraisal.User{}
	err = json.NewDecoder(resp.Body).Decode(user)
	defer resp.Body.Close()
	if err != nil {
		ctx.renderServerError(r, w, err)
		return
	}

	ctx.setSessionValue(r, w, "user", &user)
	log.Printf("User logged in: %s", user.CharacterName)

	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}
<|endoftext|>


<commit_before>// stardew.rocks web server
//
// It contains a few HTTP handlers:
// - a static handler that serves files from /home/stardew/www.
// - a handler that serves the latest complete file save in XML format.
package main

import (
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/nictuku/stardew-rocks/stardb"

	logging "github.com/op/go-logging"

	"github.com/NYTimes/gziphandler"
	hs "github.com/gorilla/handlers"
	"gopkg.in/natefinch/lumberjack.v2"
)

var log = logging.MustGetLogger("stardew.rocks")

func wwwDir() string {
	home := os.Getenv("HOME")
	if home == "" {
		home = string(filepath.Separator)
	}
	return filepath.Clean(filepath.Join(home, "www"))
}

func Index(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, filepath.Join(wwwDir(), "index.html"))
}

func GetFarms(w http.ResponseWriter, r *http.Request) {
	b, err := stardb.FarmsJSON()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	_, err = w.Write(b)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	return
}

func GetFarm(w http.ResponseWriter, r *http.Request) {
	farmid := strings.TrimPrefix(r.URL.Path, "/api/farm/")

	b, err := stardb.FarmJSON(farmid)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	_, err = w.Write(b)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	return
}

func Root(w http.ResponseWriter, r *http.Request) {
	if strings.HasPrefix(r.URL.Path, "/map") {
		LegacyMap(w, r)
		return
	}
	// Allow URLs such as http://stardew.farm/56f360131700d41646571433
	if len(r.URL.Path) == 25 { // len("/56f360131700d41646571433")
		Index(w, r)
		return
	}
	dir := wwwDir()
	http.FileServer(http.Dir(dir)).ServeHTTP(w, r)
}

func StaticFiles(w http.ResponseWriter, r *http.Request) {
	dir := wwwDir()
	http.FileServer(http.Dir(dir)).ServeHTTP(w, r)
}

// SearchFarms searches for farms or farmers. It looks for a query paramater "q".
func SearchFarms(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()

	q := r.Form.Get("q")
	if len(q) == 0 {
		w.Write([]byte("{}"))
		return
	}
	b, err := stardb.SearchFarmsJSON(q)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	_, err = w.Write(b)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	return
}

func RunHTTPServer() {
	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		log.Error("Error starting www server:", err)
		// os.IsPermission doesn't match.
		if strings.Contains(err.Error(), "permission denied") {
			// Only for linux and port 80. Keeping here for future convenience.
			log.Error("Try: sudo setcap 'cap_net_bind_service=+ep' www")
		}
	}
}

func main() {

	serverlog := logging.NewBackendFormatter(logging.NewLogBackend(&lumberjack.Logger{
		Filename:   "/var/log/stardew/server.log",
		MaxSize:    500, // megabytes
		MaxBackups: 3,
		MaxAge:     28, //days
	}, "", 0), logging.GlogFormatter)

	formatted := logging.AddModuleLevel(serverlog)
	formatted.SetLevel(logging.INFO, "")

	stderr := logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), logging.GlogFormatter)

	// Set the backends to be used and the default level.
	logging.SetBackend(stderr, formatted)

	combinedLog := &lumberjack.Logger{
		Filename:   "/var/log/stardew/access.log",
		MaxSize:    500, // megabytes
		MaxBackups: 3,
		MaxAge:     28, //days
	}

	dir := wwwDir()
	log.Infof("Serving files from %v", dir)
	http.Handle("/api/farms", hs.CombinedLoggingHandler(combinedLog, http.HandlerFunc(GetFarms)))
	http.Handle("/api/farm/", hs.CombinedLoggingHandler(combinedLog, http.HandlerFunc(GetFarm)))
	http.Handle("/api/search/farm", hs.CombinedLoggingHandler(combinedLog, http.HandlerFunc(SearchFarms)))

	http.Handle("/screenshot/", hs.CombinedLoggingHandler(combinedLog, http.HandlerFunc(ServeScreenshot)))
	//http.Handle("/saveGames/", logHandler(http.HandlerFunc(ServeGFSFile)))

	// This is served by the web app.
	http.Handle("/farm/", hs.CombinedLoggingHandler(combinedLog, http.HandlerFunc(Index)))

	// This is served from the filesystem, but / goes to index.html which has our web app.
	http.Handle("/", hs.CombinedLoggingHandler(combinedLog, gziphandler.GzipHandler(http.HandlerFunc(Root))))

	// Useful during development.
	http.Handle("/src/", hs.CombinedLoggingHandler(combinedLog, gziphandler.GzipHandler(http.HandlerFunc(StaticFiles))))
	http.Handle("/jspm_packages/", hs.CombinedLoggingHandler(combinedLog, gziphandler.GzipHandler(http.HandlerFunc(StaticFiles))))

	RunHTTPServer()
}
<commit_msg>web/server: allow running on different port, for the staging environment<commit_after>// stardew.rocks web server
//
// It contains a few HTTP handlers:
// - a static handler that serves files from /home/stardew/www.
// - a handler that serves the latest complete file save in XML format.
package main

import (
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/nictuku/stardew-rocks/stardb"

	logging "github.com/op/go-logging"

	"github.com/NYTimes/gziphandler"
	hs "github.com/gorilla/handlers"
	"gopkg.in/natefinch/lumberjack.v2"
)

var log = logging.MustGetLogger("stardew.rocks")

func wwwDir() string {
	home := os.Getenv("HOME")
	if home == "" {
		home = string(filepath.Separator)
	}
	return filepath.Clean(filepath.Join(home, "www"))
}

func Index(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, filepath.Join(wwwDir(), "index.html"))
}

func GetFarms(w http.ResponseWriter, r *http.Request) {
	b, err := stardb.FarmsJSON()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	_, err = w.Write(b)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	return
}

func GetFarm(w http.ResponseWriter, r *http.Request) {
	farmid := strings.TrimPrefix(r.URL.Path, "/api/farm/")

	b, err := stardb.FarmJSON(farmid)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	_, err = w.Write(b)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	return
}

func Root(w http.ResponseWriter, r *http.Request) {
	if strings.HasPrefix(r.URL.Path, "/map") {
		LegacyMap(w, r)
		return
	}
	// Allow URLs such as http://stardew.farm/56f360131700d41646571433
	if len(r.URL.Path) == 25 { // len("/56f360131700d41646571433")
		Index(w, r)
		return
	}
	dir := wwwDir()
	http.FileServer(http.Dir(dir)).ServeHTTP(w, r)
}

func StaticFiles(w http.ResponseWriter, r *http.Request) {
	dir := wwwDir()
	http.FileServer(http.Dir(dir)).ServeHTTP(w, r)
}

// SearchFarms searches for farms or farmers. It looks for a query paramater "q".
func SearchFarms(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()

	q := r.Form.Get("q")
	if len(q) == 0 {
		w.Write([]byte("{}"))
		return
	}
	b, err := stardb.SearchFarmsJSON(q)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	_, err = w.Write(b)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	return
}

func RunHTTPServer() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	err := http.ListenAndServe(":"+port, nil)
	if err != nil {
		log.Error("Error starting www server:", err)
		// os.IsPermission doesn't match.
		if strings.Contains(err.Error(), "permission denied") {
			// Only for linux and port 80. Keeping here for future convenience.
			log.Error("Try: sudo setcap 'cap_net_bind_service=+ep' www")
		}
	}
}

func main() {

	serverlog := logging.NewBackendFormatter(logging.NewLogBackend(&lumberjack.Logger{
		Filename:   "/var/log/stardew/server.log",
		MaxSize:    500, // megabytes
		MaxBackups: 3,
		MaxAge:     28, //days
	}, "", 0), logging.GlogFormatter)

	formatted := logging.AddModuleLevel(serverlog)
	formatted.SetLevel(logging.INFO, "")

	stderr := logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), logging.GlogFormatter)

	// Set the backends to be used and the default level.
	logging.SetBackend(stderr, formatted)

	combinedLog := &lumberjack.Logger{
		Filename:   "/var/log/stardew/access.log",
		MaxSize:    500, // megabytes
		MaxBackups: 3,
		MaxAge:     28, //days
	}

	dir := wwwDir()
	log.Infof("Serving files from %v", dir)
	http.Handle("/api/farms", hs.CombinedLoggingHandler(combinedLog, http.HandlerFunc(GetFarms)))
	http.Handle("/api/farm/", hs.CombinedLoggingHandler(combinedLog, http.HandlerFunc(GetFarm)))
	http.Handle("/api/search/farm", hs.CombinedLoggingHandler(combinedLog, http.HandlerFunc(SearchFarms)))

	http.Handle("/screenshot/", hs.CombinedLoggingHandler(combinedLog, http.HandlerFunc(ServeScreenshot)))
	//http.Handle("/saveGames/", logHandler(http.HandlerFunc(ServeGFSFile)))

	// This is served by the web app.
	http.Handle("/farm/", hs.CombinedLoggingHandler(combinedLog, http.HandlerFunc(Index)))

	// This is served from the filesystem, but / goes to index.html which has our web app.
	http.Handle("/", hs.CombinedLoggingHandler(combinedLog, gziphandler.GzipHandler(http.HandlerFunc(Root))))

	// Useful during development.
	http.Handle("/src/", hs.CombinedLoggingHandler(combinedLog, gziphandler.GzipHandler(http.HandlerFunc(StaticFiles))))
	http.Handle("/jspm_packages/", hs.CombinedLoggingHandler(combinedLog, gziphandler.GzipHandler(http.HandlerFunc(StaticFiles))))

	RunHTTPServer()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"time"

	docker "github.com/bywan/go-dockercommand"
	commons "github.com/haklop/bazooka/commons"
	l "github.com/haklop/bazooka/commons/logger"
	"github.com/haklop/bazooka/commons/mongo"
)

type Runner struct {
	BuildImages []BuiltImage
	Env         map[string]string
	Mongo       *mongo.MongoConnector
	client      *docker.Docker
}

func (r *Runner) Run(logger Logger) (bool, error) {
	client, err := docker.NewDocker(DockerEndpoint)
	if err != nil {
		return false, err
	}
	r.client = client

	errChanRun := make(chan error)
	successChanRun := make(chan bool)
	remainingRuns := len(r.BuildImages)
	for _, buildImage := range r.BuildImages {
		go r.runContainer(logger, buildImage, r.Env, successChanRun, errChanRun)
	}

	success := true
	var lastError error
	for {
		select {
		case result := <-successChanRun:
			success = success && result
		case err := <-errChanRun:
			lastError = err
		}

		remainingRuns--
		if remainingRuns == 0 {
			break
		}
	}

	l.Info.Printf("Dockerfiles builds finished\n")
	return success, lastError
}

func (r *Runner) runContainer(logger Logger, buildImage BuiltImage, env map[string]string, successChan chan bool, errChan chan error) {
	success := true
	servicesFile := fmt.Sprintf("%s/work/%d/services", BazookaInput, buildImage.VariantID)

	variant := &commons.Variant{
		Started:    time.Now(),
		BuildImage: buildImage.Image,
		Number:     buildImage.VariantID,
		JobID:      env[BazookaEnvJobID],
	}
	err := r.Mongo.AddVariant(variant)
	if err != nil {
		errChan <- err
		return
	}

	servicesList, err := listServices(servicesFile)
	if err != nil {
		mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
		if mongoErr != nil {
			errChan <- mongoErr
			return
		}
		errChan <- err
		return
	}

	serviceContainers := []*docker.Container{}
	containerLinks := []string{}
	for _, service := range servicesList {
		name := fmt.Sprintf("service-%s-%s-%d", env[BazookaEnvProjectID], env[BazookaEnvJobID], buildImage.VariantID)
		containerLinks = append(containerLinks, fmt.Sprintf("%s:%s", name, service))
		serviceContainer, err := r.client.Run(&docker.RunOptions{
			Name:  name,
			Image: service,
			VolumeBinds: []string{
				fmt.Sprintf("%s:/var/run/docker.sock", DockerSock),
			},
			Detach: true,
		})
		if err != nil {
			mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
			if mongoErr != nil {
				errChan <- mongoErr
				return
			}
			errChan <- err
			return
		}
		serviceContainers = append(serviceContainers, serviceContainer)
	}

	// TODO link containers
	container, err := r.client.Run(&docker.RunOptions{
		Image:  buildImage.Image,
		Links:  containerLinks,
		Detach: true,
	})
	if err != nil {
		mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
		if mongoErr != nil {
			errChan <- mongoErr
			return
		}
		errChan <- err
		return
	}

	container.LogsWith(buildImage.Image, l.Docker)
	logger(buildImage.Image, variant.ID, container)

	exitCode, err := container.Wait()
	if err != nil {
		mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
		if mongoErr != nil {
			errChan <- mongoErr
			return
		}
		errChan <- err
		return
	}
	if exitCode != 0 {
		if exitCode == 42 {
			mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
			if mongoErr != nil {
				errChan <- mongoErr
				return
			}
			errChan <- fmt.Errorf("Run failed\n Check Docker container logs, id is %s\n", container.ID())
			return
		}
		success = false
	}
	err = container.Remove(&docker.RemoveOptions{
		Force:         true,
		RemoveVolumes: true,
	})
	if err != nil {
		mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
		if mongoErr != nil {
			errChan <- mongoErr
			return
		}
		errChan <- err
		return
	}

	for _, serviceContainer := range serviceContainers {
		err = serviceContainer.Remove(&docker.RemoveOptions{
			Force:         true,
			RemoveVolumes: true,
		})
		if err != nil {
			mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
			if mongoErr != nil {
				errChan <- mongoErr
				return
			}
			errChan <- err
			return
		}
	}
	var status commons.JobStatus
	if success {
		status = commons.JOB_SUCCESS
	} else {
		status = commons.JOB_FAILED
	}
	err = r.Mongo.FinishVariant(variant.ID, status, time.Now())
	if err != nil {
		errChan <- err
		return
	}
	successChan <- success
}

func listServices(servicesFile string) ([]string, error) {
	file, err := os.Open(servicesFile)
	if err != nil {
		if os.IsNotExist(err) {
			return []string{}, nil
		}
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	var services []string
	for scanner.Scan() {
		services = append(services, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return services, nil
}
<commit_msg>This time, docker sock is really added<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"time"

	docker "github.com/bywan/go-dockercommand"
	commons "github.com/haklop/bazooka/commons"
	l "github.com/haklop/bazooka/commons/logger"
	"github.com/haklop/bazooka/commons/mongo"
)

type Runner struct {
	BuildImages []BuiltImage
	Env         map[string]string
	Mongo       *mongo.MongoConnector
	client      *docker.Docker
}

func (r *Runner) Run(logger Logger) (bool, error) {
	client, err := docker.NewDocker(DockerEndpoint)
	if err != nil {
		return false, err
	}
	r.client = client

	errChanRun := make(chan error)
	successChanRun := make(chan bool)
	remainingRuns := len(r.BuildImages)
	for _, buildImage := range r.BuildImages {
		go r.runContainer(logger, buildImage, r.Env, successChanRun, errChanRun)
	}

	success := true
	var lastError error
	for {
		select {
		case result := <-successChanRun:
			success = success && result
		case err := <-errChanRun:
			lastError = err
		}

		remainingRuns--
		if remainingRuns == 0 {
			break
		}
	}

	l.Info.Printf("Dockerfiles builds finished\n")
	return success, lastError
}

func (r *Runner) runContainer(logger Logger, buildImage BuiltImage, env map[string]string, successChan chan bool, errChan chan error) {
	success := true
	servicesFile := fmt.Sprintf("%s/work/%d/services", BazookaInput, buildImage.VariantID)

	variant := &commons.Variant{
		Started:    time.Now(),
		BuildImage: buildImage.Image,
		Number:     buildImage.VariantID,
		JobID:      env[BazookaEnvJobID],
	}
	err := r.Mongo.AddVariant(variant)
	if err != nil {
		errChan <- err
		return
	}

	servicesList, err := listServices(servicesFile)
	if err != nil {
		mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
		if mongoErr != nil {
			errChan <- mongoErr
			return
		}
		errChan <- err
		return
	}

	serviceContainers := []*docker.Container{}
	containerLinks := []string{}
	for _, service := range servicesList {
		name := fmt.Sprintf("service-%s-%s-%d", env[BazookaEnvProjectID], env[BazookaEnvJobID], buildImage.VariantID)
		containerLinks = append(containerLinks, fmt.Sprintf("%s:%s", name, service))
		serviceContainer, err := r.client.Run(&docker.RunOptions{
			Name:   name,
			Image:  service,
			Detach: true,
		})
		if err != nil {
			mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
			if mongoErr != nil {
				errChan <- mongoErr
				return
			}
			errChan <- err
			return
		}
		serviceContainers = append(serviceContainers, serviceContainer)
	}

	// TODO link containers
	container, err := r.client.Run(&docker.RunOptions{
		Image: buildImage.Image,
		Links: containerLinks,
		VolumeBinds: []string{
			fmt.Sprintf("%s:/var/run/docker.sock", DockerSock),
		},
		Detach: true,
	})
	if err != nil {
		mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
		if mongoErr != nil {
			errChan <- mongoErr
			return
		}
		errChan <- err
		return
	}

	container.LogsWith(buildImage.Image, l.Docker)
	logger(buildImage.Image, variant.ID, container)

	exitCode, err := container.Wait()
	if err != nil {
		mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
		if mongoErr != nil {
			errChan <- mongoErr
			return
		}
		errChan <- err
		return
	}
	if exitCode != 0 {
		if exitCode == 42 {
			mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
			if mongoErr != nil {
				errChan <- mongoErr
				return
			}
			errChan <- fmt.Errorf("Run failed\n Check Docker container logs, id is %s\n", container.ID())
			return
		}
		success = false
	}
	err = container.Remove(&docker.RemoveOptions{
		Force:         true,
		RemoveVolumes: true,
	})
	if err != nil {
		mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
		if mongoErr != nil {
			errChan <- mongoErr
			return
		}
		errChan <- err
		return
	}

	for _, serviceContainer := range serviceContainers {
		err = serviceContainer.Remove(&docker.RemoveOptions{
			Force:         true,
			RemoveVolumes: true,
		})
		if err != nil {
			mongoErr := r.Mongo.FinishVariant(variant.ID, commons.JOB_ERRORED, time.Now())
			if mongoErr != nil {
				errChan <- mongoErr
				return
			}
			errChan <- err
			return
		}
	}
	var status commons.JobStatus
	if success {
		status = commons.JOB_SUCCESS
	} else {
		status = commons.JOB_FAILED
	}
	err = r.Mongo.FinishVariant(variant.ID, status, time.Now())
	if err != nil {
		errChan <- err
		return
	}
	successChan <- success
}

func listServices(servicesFile string) ([]string, error) {
	file, err := os.Open(servicesFile)
	if err != nil {
		if os.IsNotExist(err) {
			return []string{}, nil
		}
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	var services []string
	for scanner.Scan() {
		services = append(services, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return services, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"

	docker "github.com/bywan/go-dockercommand"
)

type Runner struct {
	BuildImages []BuiltImage
	Env         map[string]string
}

func (r *Runner) Run() error {
	client, err := docker.NewDocker(DockerEndpoint)
	if err != nil {
		return err
	}

	errChanRun := make(chan error)
	successChanRun := make(chan bool)
	remainingRuns := len(r.BuildImages)
	for _, buildImage := range r.BuildImages {
		go runContainer(client, buildImage, r.Env, successChanRun, errChanRun)
	}

	for {
		select {
		case _ = <-successChanRun:
			remainingRuns--
		case err := <-errChanRun:
			return err
		}

		if remainingRuns == 0 {
			break
		}
	}

	log.Printf("Dockerfiles builds finished\n")
	return nil
}

func runContainer(client *docker.Docker, buildImage BuiltImage, env map[string]string, successChan chan bool, errChan chan error) {
	servicesFile := fmt.Sprintf("%s/work/%d/services", BazookaInput, buildImage.VariantID)

	servicesList, err := listServices(servicesFile)
	if err != nil {
		errChan <- err
		return
	}

	serviceContainers := []*docker.Container{}
	containerLinks := []string{}
	for _, service := range servicesList {
		name := fmt.Sprintf("service-%s-%s-%d", env[BazookaEnvProjectID], env[BazookaEnvJobID], buildImage.VariantID)
		containerLinks = append(containerLinks, fmt.Sprintf("%s:%s", name, service))
		serviceContainer, err := client.Run(&docker.RunOptions{
			Name:   name,
			Image:  service,
			Detach: true,
		})
		if err != nil {
			errChan <- err
			return
		}
		serviceContainers = append(serviceContainers, serviceContainer)
	}

	// TODO link containers
	container, err := client.Run(&docker.RunOptions{
		Image:  buildImage.Image,
		Links:  containerLinks,
		Detach: true,
	})
	if err != nil {
		errChan <- err
		return
	}

	container.Logs(buildImage.Image)

	exitCode, err := container.Wait()
	if err != nil {
		errChan <- err
		return
	}
	if exitCode != 0 {
		errChan <- fmt.Errorf("Run failed\n Check Docker container logs, id is %s\n", container.ID())
		return
	}
	successChan <- true

	for _, serviceContainer := range serviceContainers {
		err := serviceContainer.Stop(300)
		if err != nil {
			errChan <- err
			return
		}
		_, err = serviceContainer.Wait()
		if err != nil {
			errChan <- err
			return
		}
		err = container.Remove(docker.RemoveOptions{
			Force:         true,
			RemoveVolumes: true,
		})
		if err != nil {
			errChan <- err
			return
		}
	}
}

func listServices(servicesFile string) ([]string, error) {
	file, err := os.Open(servicesFile)
	if err != nil {
		if os.IsNotExist(err) {
			return []string{}, nil
		}
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	var services []string
	for scanner.Scan() {
		services = append(services, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return services, nil
}
<commit_msg>remove services containers with volumes<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"

	docker "github.com/bywan/go-dockercommand"
)

type Runner struct {
	BuildImages []BuiltImage
	Env         map[string]string
}

func (r *Runner) Run() error {
	client, err := docker.NewDocker(DockerEndpoint)
	if err != nil {
		return err
	}

	errChanRun := make(chan error)
	successChanRun := make(chan bool)
	remainingRuns := len(r.BuildImages)
	for _, buildImage := range r.BuildImages {
		go runContainer(client, buildImage, r.Env, successChanRun, errChanRun)
	}

	for {
		select {
		case _ = <-successChanRun:
			remainingRuns--
		case err := <-errChanRun:
			return err
		}

		if remainingRuns == 0 {
			break
		}
	}

	log.Printf("Dockerfiles builds finished\n")
	return nil
}

func runContainer(client *docker.Docker, buildImage BuiltImage, env map[string]string, successChan chan bool, errChan chan error) {
	servicesFile := fmt.Sprintf("%s/work/%d/services", BazookaInput, buildImage.VariantID)

	servicesList, err := listServices(servicesFile)
	if err != nil {
		errChan <- err
		return
	}

	serviceContainers := []*docker.Container{}
	containerLinks := []string{}
	for _, service := range servicesList {
		name := fmt.Sprintf("service-%s-%s-%d", env[BazookaEnvProjectID], env[BazookaEnvJobID], buildImage.VariantID)
		containerLinks = append(containerLinks, fmt.Sprintf("%s:%s", name, service))
		serviceContainer, err := client.Run(&docker.RunOptions{
			Name:   name,
			Image:  service,
			Detach: true,
		})
		if err != nil {
			errChan <- err
			return
		}
		serviceContainers = append(serviceContainers, serviceContainer)
	}

	// TODO link containers
	container, err := client.Run(&docker.RunOptions{
		Image:  buildImage.Image,
		Links:  containerLinks,
		Detach: true,
	})
	if err != nil {
		errChan <- err
		return
	}

	container.Logs(buildImage.Image)

	exitCode, err := container.Wait()
	if err != nil {
		errChan <- err
		return
	}
	if exitCode != 0 {
		errChan <- fmt.Errorf("Run failed\n Check Docker container logs, id is %s\n", container.ID())
		return
	}
	successChan <- true

	fmt.Printf("$$$$$$$$$$$$%s\n", serviceContainers)
	for _, serviceContainer := range serviceContainers {
		err = serviceContainer.Remove(docker.RemoveOptions{
			Force:         true,
			RemoveVolumes: true,
		})
		if err != nil {
			errChan <- err
			return
		}
	}
}

func listServices(servicesFile string) ([]string, error) {
	file, err := os.Open(servicesFile)
	if err != nil {
		if os.IsNotExist(err) {
			return []string{}, nil
		}
		return nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	var services []string
	for scanner.Scan() {
		services = append(services, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return services, nil
}
<|endoftext|>


<commit_before>package penalty

import (
	"sync"
)

type Score int

type PenaltyType int

const (
	P_COOKIE            PenaltyType = 1
	P_EXTERNAL_LINK     PenaltyType = 2
	P_HTTP_LINK         PenaltyType = 3
	P_EXTERNAL_RESOURCE PenaltyType = 4
	P_NO_HTTPS          PenaltyType = 5
	P_JS                PenaltyType = 6
	P_NO_SECURE_HEADER  PenaltyType = 7
	P_IFRAME            PenaltyType = 8
)

const baseScore Score = 100

type Penalty struct {
	Description string
	DetailLink  string
	Notes       []string
	value       Score
}

type PenaltyContainer struct {
	sync.RWMutex
	penalties map[PenaltyType]*Penalty
}

func NewPenaltyContainer() *PenaltyContainer {
	return &PenaltyContainer{penalties: make(map[PenaltyType]*Penalty)}
}

func (c *PenaltyContainer) GetAll() map[PenaltyType]*Penalty {
	return c.penalties
}

func (c *PenaltyContainer) Add(pt PenaltyType, notes ...string) {
	if p, found := c.penalties[pt]; found {
		for _, n := range notes {
			if n == "" {
				continue
			}
			note_found := false
			for _, pn := range p.Notes {
				if pn == n {
					note_found = true
					break
				}
			}
			if !note_found {
				c.Lock()
				p.Notes = append(p.Notes, n)
				c.Unlock()
			}
		}
	} else {
		c.Lock()
		c.penalties[pt] = New(pt)
		c.penalties[pt].Notes = notes
		c.Unlock()
	}
}

func (c *PenaltyContainer) GetScore() Score {
	score := baseScore
	for _, p := range c.penalties {
		score -= p.GetValue()
	}
	return score
}

func (p *Penalty) GetValue() Score {
	if len(p.Notes) > 0 {
		return p.value * Score(len(p.Notes))
	}
	return p.value
}

func New(p PenaltyType) *Penalty {
	desc := ""
	link := ""
	var score Score = 0
	switch p {
	case P_COOKIE:
		desc = "Automatically sets cookies"
		link = "https://en.wikipedia.org/wiki/Internet_privacy#HTTP_cookies"
		score = 3
	case P_EXTERNAL_LINK:
		desc = "Leaks HTTP referrer to foreign host"
		link = "https://randomoracle.wordpress.com/2013/11/23/privacy-and-http-referer-header-12/"
		score = 3
	case P_HTTP_LINK:
		desc = "Has link to unencrypted service (no HTTPS)"
		link = "https://en.wikipedia.org/wiki/HTTP_Secure"
		score = 3
	case P_EXTERNAL_RESOURCE:
		desc = "Loads external resource"
		link = "https://jonathanmayer.org/papers_data/trackingsurvey12.pdf"
		score = 3
	case P_NO_HTTPS:
		desc = "Uses unencrypted transport layer (no HTTPS)"
		link = "https://en.wikipedia.org/wiki/HTTP_Secure"
		score = 3
	case P_JS:
		desc = "Uses JavaScript"
		link = "todo"
		score = 3
	case P_NO_SECURE_HEADER:
		desc = "Missing secure HTTP header"
		link = "https://scotthelme.co.uk/hardening-your-http-response-headers/"
		score = 3
	case P_IFRAME:
		desc = "Loads external content to iframe"
		link = "http://stackoverflow.com/questions/7289139/why-are-iframes-considered-dangerous-and-a-security-risk"
		score = 3
	}
	return &Penalty{desc, link, make([]string, 0, 8), score}
}
<commit_msg>[mod] scoring<commit_after>package penalty

import (
	"sync"
)

type Score int

type PenaltyType int

const (
	P_COOKIE            PenaltyType = 1
	P_EXTERNAL_LINK     PenaltyType = 2
	P_HTTP_LINK         PenaltyType = 3
	P_EXTERNAL_RESOURCE PenaltyType = 4
	P_NO_HTTPS          PenaltyType = 5
	P_JS                PenaltyType = 6
	P_NO_SECURE_HEADER  PenaltyType = 7
	P_IFRAME            PenaltyType = 8
)

const baseScore Score = 100

type Penalty struct {
	Description string
	DetailLink  string
	Notes       []string
	value       Score
}

type PenaltyContainer struct {
	sync.RWMutex
	penalties map[PenaltyType]*Penalty
}

func NewPenaltyContainer() *PenaltyContainer {
	return &PenaltyContainer{penalties: make(map[PenaltyType]*Penalty)}
}

func (c *PenaltyContainer) GetAll() map[PenaltyType]*Penalty {
	return c.penalties
}

func (c *PenaltyContainer) Add(pt PenaltyType, notes ...string) {
	if p, found := c.penalties[pt]; found {
		for _, n := range notes {
			if n == "" {
				continue
			}
			note_found := false
			for _, pn := range p.Notes {
				if pn == n {
					note_found = true
					break
				}
			}
			if !note_found {
				c.Lock()
				p.Notes = append(p.Notes, n)
				c.Unlock()
			}
		}
	} else {
		c.Lock()
		c.penalties[pt] = New(pt)
		c.penalties[pt].Notes = notes
		c.Unlock()
	}
}

func (c *PenaltyContainer) GetScore() Score {
	score := baseScore
	for _, p := range c.penalties {
		score -= p.GetValue()
	}
	return score
}

func (p *Penalty) GetValue() Score {
	if len(p.Notes) > 0 {
		return p.value * Score(len(p.Notes))
	}
	return p.value
}

func New(p PenaltyType) *Penalty {
	desc := ""
	link := ""
	var score Score = 0
	switch p {
	case P_COOKIE:
		desc = "Automatically sets cookies"
		link = "https://en.wikipedia.org/wiki/Internet_privacy#HTTP_cookies"
		score = 3
	case P_EXTERNAL_LINK:
		desc = "Leaks HTTP referrer to foreign host"
		link = "https://randomoracle.wordpress.com/2013/11/23/privacy-and-http-referer-header-12/"
		score = 1
	case P_HTTP_LINK:
		desc = "Has link to unencrypted service (no HTTPS)"
		link = "https://en.wikipedia.org/wiki/HTTP_Secure"
		score = 3
	case P_EXTERNAL_RESOURCE:
		desc = "Loads external resource"
		link = "https://jonathanmayer.org/papers_data/trackingsurvey12.pdf"
		score = 8
	case P_NO_HTTPS:
		desc = "Uses unencrypted transport layer (no HTTPS)"
		link = "https://en.wikipedia.org/wiki/HTTP_Secure"
		score = 6
	case P_JS:
		desc = "Uses JavaScript"
		link = "todo"
		score = 5
	case P_NO_SECURE_HEADER:
		desc = "Missing secure HTTP header"
		link = "https://scotthelme.co.uk/hardening-your-http-response-headers/"
		score = 3
	case P_IFRAME:
		desc = "Loads external content to iframe"
		link = "http://stackoverflow.com/questions/7289139/why-are-iframes-considered-dangerous-and-a-security-risk"
		score = 3
	}
	return &Penalty{desc, link, make([]string, 0, 8), score}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"github.com/jasdel/harvester/internal/queue"
	"github.com/jasdel/harvester/internal/storage"
	"log"
	"net/http"
	"os"
	"path"
)

// Web server for exposing an interface for scheduling jobs, checking their status, and
// receiving their result.
//
// The endpoints exposed are:
// POST: /
//		- Schedule Job. Body is newline separated list of URls to scheduled to be crawled.
//
// GET: /status/:jobId
//		- Get the status of an already scheduled job.
//
// GET: /result/:jobId
//		- Get the result of an already scheduled job
//
// Queues Used:
// Publish to URL Queue:
// Scheduled Job URLs will be sent to the URL Queue to be filtered and later crawled.
//
func main() {
	cfgFilename := flag.String("config", "config.json", "The web server configuration file.")
	flag.Parse()

	cfg, err := LoadConfig(*cfgFilename)
	if err != nil {
		log.Fatalln(err)
	}

	// Initialize the queue for publishing scheduled Job URLs
	urlQueuePub, err := queue.NewPublisher(cfg.URLQueueConfig)
	if err != nil {
		log.Fatalln("Queue Publisher initialization failed:", err)
	}
	defer urlQueuePub.Close()

	// Initialize the storage for checking the status and results of jobs
	sc, err := storage.NewClient(cfg.StorageConfig)
	if err != nil {
		log.Fatalln("Storage NewClient failed:", err)
	}
	defer sc.Close()

	// Create the HTTP handlers to be able to provide an interface for serving
	// job schedule, status, and result requests. The Trailing '/' have to be append
	// because path.Join will strip off the trailing '/'
	http.Handle(path.Join("/", cfg.HTTPRootPath), &JobScheduleHandler{urlQueuePub: urlQueuePub, sc: sc})
	http.Handle(path.Join("/", cfg.HTTPRootPath, "/status/")+"/", &JobStatusHandler{sc: sc})
	http.Handle(path.Join("/", cfg.HTTPRootPath, "/result/")+"/", &JobResultHandler{sc: sc})

	log.Println("Listening on", cfg.HTTPAddr)
	if err := http.ListenAndServe(cfg.HTTPAddr, nil); err != nil {
		log.Fatalln(err)
	}
}

// Provides the web server's configuration information. For connecting to
// Queues, storage, and other runtime settings.
type Config struct {
	// Storage connection configuration
	StorageConfig storage.ClientConfig `json:"storage"`

	// URL queue for publishing scheduled job URLs to the foreman
	URLQueueConfig queue.QueueConfig `json:"urlQueue"`

	// HTTP address to service content from
	HTTPAddr string `json:"httpAddr"`

	// Root path the HTTP routes should be based of of. Useful when
	// nesting the service behind a reverse proxy
	HTTPRootPath string `json:"httpRootPath"`
}

// Loads the configuration file from disk in as a JSON blob.
func LoadConfig(filename string) (Config, error) {
	cfg := Config{}

	file, err := os.Open(filename)
	if err != nil {
		return cfg, err
	}
	defer file.Close()

	if err = json.NewDecoder(file).Decode(&cfg); err != nil {
		return cfg, err
	}

	return cfg, nil
}
<commit_msg>added ability to configure host address from commandline<commit_after>package main

import (
	"encoding/json"
	"flag"
	"github.com/jasdel/harvester/internal/queue"
	"github.com/jasdel/harvester/internal/storage"
	"log"
	"net/http"
	"os"
	"path"
)

// Web server for exposing an interface for scheduling jobs, checking their status, and
// receiving their result.
//
// The endpoints exposed are:
// POST: /
//		- Schedule Job. Body is newline separated list of URls to scheduled to be crawled.
//
// GET: /status/:jobId
//		- Get the status of an already scheduled job.
//
// GET: /result/:jobId
//		- Get the result of an already scheduled job
//
// Queues Used:
// Publish to URL Queue:
// Scheduled Job URLs will be sent to the URL Queue to be filtered and later crawled.
//
func main() {
	cfgFilename := flag.String("config", "config.json", "The web server configuration file.")
	hostAddr := flag.String("addr", "", "Host address to override config file")
	flag.Parse()

	cfg, err := LoadConfig(*cfgFilename)
	if err != nil {
		log.Fatalln(err)
	}

	// Allow the host address to be overridden via command line, for multiple instances
	if *hostAddr != "" {
		cfg.HTTPAddr = *hostAddr
	}

	// Initialize the queue for publishing scheduled Job URLs
	urlQueuePub, err := queue.NewPublisher(cfg.URLQueueConfig)
	if err != nil {
		log.Fatalln("Queue Publisher initialization failed:", err)
	}
	defer urlQueuePub.Close()

	// Initialize the storage for checking the status and results of jobs
	sc, err := storage.NewClient(cfg.StorageConfig)
	if err != nil {
		log.Fatalln("Storage NewClient failed:", err)
	}
	defer sc.Close()

	// Create the HTTP handlers to be able to provide an interface for serving
	// job schedule, status, and result requests. The Trailing '/' have to be append
	// because path.Join will strip off the trailing '/'
	http.Handle(path.Join("/", cfg.HTTPRootPath), &JobScheduleHandler{urlQueuePub: urlQueuePub, sc: sc})
	http.Handle(path.Join("/", cfg.HTTPRootPath, "/status/")+"/", &JobStatusHandler{sc: sc})
	http.Handle(path.Join("/", cfg.HTTPRootPath, "/result/")+"/", &JobResultHandler{sc: sc})

	log.Println("Listening on", cfg.HTTPAddr)
	if err := http.ListenAndServe(cfg.HTTPAddr, nil); err != nil {
		log.Fatalln(err)
	}
}

// Provides the web server's configuration information. For connecting to
// Queues, storage, and other runtime settings.
type Config struct {
	// Storage connection configuration
	StorageConfig storage.ClientConfig `json:"storage"`

	// URL queue for publishing scheduled job URLs to the foreman
	URLQueueConfig queue.QueueConfig `json:"urlQueue"`

	// HTTP address to service content from
	HTTPAddr string `json:"httpAddr"`

	// Root path the HTTP routes should be based of of. Useful when
	// nesting the service behind a reverse proxy
	HTTPRootPath string `json:"httpRootPath"`
}

// Loads the configuration file from disk in as a JSON blob.
func LoadConfig(filename string) (Config, error) {
	cfg := Config{}

	file, err := os.Open(filename)
	if err != nil {
		return cfg, err
	}
	defer file.Close()

	if err = json.NewDecoder(file).Decode(&cfg); err != nil {
		return cfg, err
	}

	return cfg, nil
}
<|endoftext|>


<commit_before>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2017, 4, 1, 0, 0, 0, 0, time.Local)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	indent         = "\t"
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<commit_msg>go 1.8.1<commit_after>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2017, 4, 10, 0, 0, 0, 0, time.Local)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	indent         = "\t"
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/hex"
	"flag"
	csv "github.com/whosonfirst/go-whosonfirst-csv"
	log "github.com/whosonfirst/go-whosonfirst-log"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type WOFClone struct {
	Source         string
	Dest           string
	Count          int64
	Success        int64
	Error          int64
	Skipped        int64
	Scheduled      int64
	Completed      int64
	client         *http.Client
	connections    int64
	maxconnections int64
	cond           *sync.Cond
	logger         *log.WOFLogger
}

func NewWOFClone(source string, dest string, logger *log.WOFLogger) *WOFClone {

	// to do - add logging

	cd := &sync.Cond{L: &sync.Mutex{}}

	cl := &http.Client{Timeout: 3}

	c := WOFClone{
		Count:          0,
		Success:        0,
		Error:          0,
		Skipped:        0,
		Source:         source,
		Dest:           dest,
		logger:         logger,
		client:         cl,
		connections:    0,
		maxconnections: 200,
		cond:           cd,
	}

	return &c
}

func (c *WOFClone) ParseMetaFile(file string) error {

	abs_path, _ := filepath.Abs(file)
	c.logger.Debug("Parse meta file %s", abs_path)

	reader, read_err := csv.NewDictReader(abs_path)

	if read_err != nil {
		c.logger.Error("Failed to read %s, because %v", abs_path, read_err)
		return read_err
	}

	wg := new(sync.WaitGroup)

	for {
		row, err := reader.Read()

		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		rel_path, ok := row["path"]

		if !ok {
			continue
		}

		wg.Add(1)

		go func() {

			atomic.AddInt64(&c.Scheduled, 1)

			defer wg.Done()
			complete, _ := c.Clone(rel_path)

			c.logger.Debug("Finised cloning %s: %t", rel_path, complete)

			atomic.AddInt64(&c.Completed, 1)
		}()
	}

	go func() {

		for c.Completed < c.Scheduled {
			c.logger.Info("scheduled: %d completed: %d connections: %d", c.Scheduled, c.Completed, c.connections)
		}
	}()

	wg.Wait()

	return nil
}

func (c *WOFClone) Clone(rel_path string) (bool, error) {

	c.logger.Debug("Pre-process %s", rel_path)

	atomic.AddInt64(&c.Count, 1)

	remote := c.Source + rel_path
	local := path.Join(c.Dest, rel_path)

	/*
		_, err := os.Stat(local)

		if !os.IsNotExist(err) {

			change, _ := c.HasChanged(local, remote)

			if ! change {
				atomic.AddInt64(&c.Skipped, 1)
				return true, nil
			}

		}
	*/

	process_err := c.Process(remote, local)

	if process_err != nil {
		atomic.AddInt64(&c.Error, 1)
		return false, process_err
	}

	atomic.AddInt64(&c.Success, 1)
	return true, nil
}

func (c *WOFClone) HasChanged(local string, remote string) (bool, error) {

	change := true

	body, err := ioutil.ReadFile(local)

	if err != nil {
		c.logger.Error("Failed to read %s, becase %v", local, err)
		return change, err
	}

	hash := md5.Sum(body)
	local_hash := hex.EncodeToString(hash[:])

	rsp, err := c.Fetch("HEAD", remote)

	if err != nil {
		return change, err
	}

	defer rsp.Body.Close()

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		change = false
	}

	// log.Printf("hash %s etag %s change %t\n", local_hash, remote_hash, change)

	return change, nil
}

func (c *WOFClone) Process(remote string, local string) error {

	c.logger.Debug("fetch %s and store in %s", remote, local)

	local_root := path.Dir(local)

	_, err := os.Stat(local_root)

	if os.IsNotExist(err) {
		c.logger.Info("create %s", local_root)
		os.MkdirAll(local_root, 0755)
	}

	rsp, fetch_err := c.Fetch("GET", remote)

	if fetch_err != nil {
		return fetch_err
	}

	defer rsp.Body.Close()

	contents, read_err := ioutil.ReadAll(rsp.Body)

	if read_err != nil {
		c.logger.Error("failed to read body for %s, because %v", remote, read_err)
		return read_err
	}

	go func() error {

		write_err := ioutil.WriteFile(local, contents, 0644)

		if write_err != nil {
			c.logger.Error("Failed to write %s, because %v", local, write_err)

			// See the way we're in a goroutine? That means the parent function
			// will return 'okay' without an error. So we're just going to account
			// for that here...

			atomic.AddInt64(&c.Success, -1)
			atomic.AddInt64(&c.Error, 1)

			return write_err
		}

		c.logger.Debug("Wrote %s to disk", local)
		return nil
	}()

	return nil
}

func (c *WOFClone) Fetch(method string, url string) (*http.Response, error) {

	c.logger.Debug("%s %s", method, url)

	for {
		c.cond.L.Lock()

		for c.connections > c.maxconnections {
			c.logger.Debug("%d still > %d", c.connections, c.maxconnections)
			c.cond.Wait()
		}

		atomic.AddInt64(&c.connections, 1)

		c.cond.L.Unlock()
		c.cond.Broadcast()
		break
	}

	req, _ := http.NewRequest(method, url, nil)
	req.Close = true

	rsp, err := c.client.Do(req)

	atomic.AddInt64(&c.connections, -1)

	if err != nil {
		c.logger.Error("Failed to %s %s, because %v", method, url, err)
		return nil, err
	}

	return rsp, err
}

func main() {

	var source = flag.String("source", "http://whosonfirst.mapzen.com/data/", "Where to look for files")
	var dest = flag.String("dest", "", "Where to write files")
	var loglevel = flag.String("loglevel", "debug", "The level of detail for logging")

	flag.Parse()
	args := flag.Args()

	writer := io.MultiWriter(os.Stdout)

	lg := log.NewWOFLogger(writer, "[clone] ", *loglevel)

	cl := NewWOFClone(*source, *dest, lg)

	start := time.Now()

	wg := new(sync.WaitGroup)

	for _, file := range args {

		wg.Add(1)

		go func() {
			defer wg.Done()
			cl.ParseMetaFile(file)
		}()
	}

	wg.Wait()

	since := time.Since(start)
	secs := float64(since) / 1e9

	cl.logger.Info("processed %d files (ok: %d error: %d) in %f seconds\n", cl.Count, cl.Success, cl.Error, secs)
}
<commit_msg>still exhausts all the filehandles with reckless abandon but now manages to check ETags correctly<commit_after>package main

import (
	"crypto/md5"
	"encoding/hex"
	"flag"
	csv "github.com/whosonfirst/go-whosonfirst-csv"
	log "github.com/whosonfirst/go-whosonfirst-log"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type WOFClone struct {
	Source         string
	Dest           string
	Count          int64
	Success        int64
	Error          int64
	Skipped        int64
	Scheduled      int64
	Completed      int64
	client         *http.Client
	connections    int64
	maxconnections int64
	cond           *sync.Cond
	logger         *log.WOFLogger
}

func NewWOFClone(source string, dest string, logger *log.WOFLogger) *WOFClone {

	// to do - add logging

	cd := &sync.Cond{L: &sync.Mutex{}}

	cl := &http.Client{}

	c := WOFClone{
		Count:          0,
		Success:        0,
		Error:          0,
		Skipped:        0,
		Source:         source,
		Dest:           dest,
		logger:         logger,
		client:         cl,
		connections:    0,
		maxconnections: 200,
		cond:           cd,
	}

	return &c
}

func (c *WOFClone) ParseMetaFile(file string) error {

	abs_path, _ := filepath.Abs(file)
	c.logger.Debug("Parse meta file %s", abs_path)

	reader, read_err := csv.NewDictReader(abs_path)

	if read_err != nil {
		c.logger.Error("Failed to read %s, because %v", abs_path, read_err)
		return read_err
	}

	wg := new(sync.WaitGroup)

	for {
		row, err := reader.Read()

		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		rel_path, ok := row["path"]

		if !ok {
			continue
		}

		wg.Add(1)

		go func() {

			atomic.AddInt64(&c.Scheduled, 1)

			defer wg.Done()
			complete, _ := c.Clone(rel_path)

			c.logger.Debug("Finised cloning %s: %t", rel_path, complete)

			atomic.AddInt64(&c.Completed, 1)
		}()
	}

	go func() {

		for c.Completed < c.Scheduled {
			c.logger.Info("scheduled: %d completed: %d connections: %d", c.Scheduled, c.Completed, c.connections)
			time.Sleep(1 * time.Second)
		}
	}()

	wg.Wait()

	return nil
}

func (c *WOFClone) Clone(rel_path string) (bool, error) {

	c.logger.Debug("Pre-process %s", rel_path)

	atomic.AddInt64(&c.Count, 1)

	remote := c.Source + rel_path
	local := path.Join(c.Dest, rel_path)

	_, err := os.Stat(local)

	if !os.IsNotExist(err) {

		change, _ := c.HasChanged(local, remote)

		if !change {

			c.logger.Debug("%s has not changed so skipping", local)

			atomic.AddInt64(&c.Success, 1)
			atomic.AddInt64(&c.Skipped, 1)
			return true, nil
		}

	}

	process_err := c.Process(remote, local)

	if process_err != nil {
		atomic.AddInt64(&c.Error, 1)
		return false, process_err
	}

	atomic.AddInt64(&c.Success, 1)
	return true, nil
}

/*
Okay, I figured out the problem I was seeing yesterday when checking
ETags (now that the WOF meta files contain an MD5 hash of the records
they list).

For example:

$> md5 /usr/local/mapzen/whosonfirst-data/data/404/529/391/404529391.geojson
71714a9cb8d286eb50b9a300b7faa1c3

If I request the same file from AWS proper everything is All Good (tm) :

$> curl -v https://s3.amazonaws.com/whosonfirst.mapzen.com/data/404/529/391/404529391.geojson
> /dev/null
* Server certificate: s3.amazonaws.com
< ETag: "71714a9cb8d286eb50b9a300b7faa1c3"

OTOH if I request the same file from wof.mz.com/data I get this:

$> curl -v https://whosonfirst.mapzen.com/data/404/529/391/404529391.geojson
> /dev/null
* Server certificate: whosonfirst.mapzen.com
< ETag: "93ad7d36a39f0e98f4aa777f22c73c03193fe390"

Which seems all weird-and-wtf until you remember that the nginx config
for the spelunker tries to proxy files from the wof-data GH repository
first and fall backs on AWS as a last resort:

https://github.com/whosonfirst/whosonfirst-www-spelunker/blob/master/nginx/whosonfirst-www-spelunker.conf.example

Who knows how GH is setting their ETags. I guess I don't really
understand why they aren't just doing MD5 hashes too but it's their
party so...

So.

The short answer is to also do a HEAD / ETag check against:

https://s3.amazonaws.com/whosonfirst.mapzen.com/data/

(20151027/thisisaaronland)
*/

func (c *WOFClone) HasChanged(local string, remote string) (bool, error) {

	change := true

	body, err := ioutil.ReadFile(local)

	if err != nil {
		c.logger.Error("Failed to read %s, becase %v", local, err)
		return change, err
	}

	hash := md5.Sum(body)
	local_hash := hex.EncodeToString(hash[:])

	rsp, err := c.Fetch("HEAD", remote)

	if err != nil {
		return change, err
	}

	defer rsp.Body.Close()

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		change = false
	}

	// log.Printf("hash %s etag %s change %t\n", local_hash, remote_hash, change)

	return change, nil
}

func (c *WOFClone) Process(remote string, local string) error {

	c.logger.Debug("fetch %s and store in %s", remote, local)

	local_root := path.Dir(local)

	_, err := os.Stat(local_root)

	if os.IsNotExist(err) {
		c.logger.Info("create %s", local_root)
		os.MkdirAll(local_root, 0755)
	}

	rsp, fetch_err := c.Fetch("GET", remote)

	if fetch_err != nil {
		return fetch_err
	}

	defer rsp.Body.Close()

	contents, read_err := ioutil.ReadAll(rsp.Body)

	if read_err != nil {
		c.logger.Error("failed to read body for %s, because %v", remote, read_err)
		return read_err
	}

	go func() error {

		write_err := ioutil.WriteFile(local, contents, 0644)

		if write_err != nil {
			c.logger.Error("Failed to write %s, because %v", local, write_err)

			// See the way we're in a goroutine? That means the parent function
			// will return 'okay' without an error. So we're just going to account
			// for that here...

			atomic.AddInt64(&c.Success, -1)
			atomic.AddInt64(&c.Error, 1)

			return write_err
		}

		c.logger.Debug("Wrote %s to disk", local)
		return nil
	}()

	return nil
}

func (c *WOFClone) Fetch(method string, url string) (*http.Response, error) {

	c.logger.Debug("%s %s", method, url)

	for {
		c.cond.L.Lock()

		for c.connections > c.maxconnections {
			c.logger.Debug("%d still > %d", c.connections, c.maxconnections)
			c.cond.Wait()
		}

		atomic.AddInt64(&c.connections, 1)

		c.cond.L.Unlock()
		c.cond.Broadcast()
		break
	}

	req, _ := http.NewRequest(method, url, nil)
	req.Close = true

	rsp, err := c.client.Do(req)

	atomic.AddInt64(&c.connections, -1)

	if err != nil {
		c.logger.Error("Failed to %s %s, because %v", method, url, err)
		return nil, err
	}

	return rsp, err
}

func main() {

	// See notes inre source and Etags in the `HasChanged` method (20151027/thisisaaronland)

	var source = flag.String("source", "https://s3.amazonaws.com/whosonfirst.mapzen.com/data/", "Where to look for files")
	var dest = flag.String("dest", "", "Where to write files")
	var loglevel = flag.String("loglevel", "debug", "The level of detail for logging")

	flag.Parse()
	args := flag.Args()

	writer := io.MultiWriter(os.Stdout)

	lg := log.NewWOFLogger(writer, "[clone] ", *loglevel)

	cl := NewWOFClone(*source, *dest, lg)

	start := time.Now()

	wg := new(sync.WaitGroup)

	for _, file := range args {

		wg.Add(1)

		go func() {
			defer wg.Done()
			cl.ParseMetaFile(file)
		}()
	}

	wg.Wait()

	since := time.Since(start)
	secs := float64(since) / 1e9

	cl.logger.Info("processed %d files (ok: %d error: %d skipped: %d) in %f seconds\n", cl.Count, cl.Success, cl.Error, cl.Skipped, secs)
}
<|endoftext|>