<commit_before>package runtime

import (
	"io"
	"net/http"

	"context"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/any"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/status"
)

// HTTPStatusFromCode converts a gRPC error code into the corresponding HTTP response status.
// See: https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto
func HTTPStatusFromCode(code codes.Code) int {
	switch code {
	case codes.OK:
		return http.StatusOK
	case codes.Canceled:
		return http.StatusRequestTimeout
	case codes.Unknown:
		return http.StatusInternalServerError
	case codes.InvalidArgument:
		return http.StatusBadRequest
	case codes.DeadlineExceeded:
		return http.StatusGatewayTimeout
	case codes.NotFound:
		return http.StatusNotFound
	case codes.AlreadyExists:
		return http.StatusConflict
	case codes.PermissionDenied:
		return http.StatusForbidden
	case codes.Unauthenticated:
		return http.StatusUnauthorized
	case codes.ResourceExhausted:
		return http.StatusTooManyRequests
	case codes.FailedPrecondition:
		return http.StatusBadRequest
	case codes.Aborted:
		return http.StatusConflict
	case codes.OutOfRange:
		return http.StatusBadRequest
	case codes.Unimplemented:
		return http.StatusNotImplemented
	case codes.Internal:
		return http.StatusInternalServerError
	case codes.Unavailable:
		return http.StatusServiceUnavailable
	case codes.DataLoss:
		return http.StatusInternalServerError
	}

	grpclog.Printf("Unknown gRPC error code: %v", code)
	return http.StatusInternalServerError
}

var (
	// HTTPError replies to the request with the error.
	// You can set a custom function to this variable to customize error format.
	HTTPError = DefaultHTTPError
	// OtherErrorHandler handles the following error used by the gateway: StatusMethodNotAllowed StatusNotFound and StatusBadRequest
	OtherErrorHandler = DefaultOtherErrorHandler
)

type errorBody struct {
	Error   string     `protobuf:"bytes,1,name=error" json:"error"`
	Code    int32      `protobuf:"varint,2,name=code" json:"code"`
	Details []*any.Any `protobuf:"bytes,3,rep,name=details" json:"details,omitempty"`
}

// Make this also conform to proto.Message for builtin JSONPb Marshaler
func (e *errorBody) Reset()         { *e = errorBody{} }
func (e *errorBody) String() string { return proto.CompactTextString(e) }
func (*errorBody) ProtoMessage()    {}

// DefaultHTTPError is the default implementation of HTTPError.
// If "err" is an error from gRPC system, the function replies with the status code mapped by HTTPStatusFromCode.
// If otherwise, it replies with http.StatusInternalServerError.
//
// The response body returned by this function is a JSON object,
// which contains a member whose key is "error" and whose value is err.Error().
func DefaultHTTPError(ctx context.Context, mux *ServeMux, marshaler Marshaler, w http.ResponseWriter, _ *http.Request, err error) {
	const fallback = `{"error": "failed to marshal error message"}`

	w.Header().Del("Trailer")
	if w.Header().Get("Content-Type") == "" {
		w.Header().Set("Content-Type", marshaler.ContentType())
	}

	s, ok := status.FromError(err)
	if !ok {
		s = status.New(codes.Unknown, err.Error())
	}

	body := &errorBody{
		Error: s.Message(),
		Code:  int32(s.Code()),
	}

	for _, detail := range s.Details() {
		if det, ok := detail.(proto.Message); ok {
			a, err := ptypes.MarshalAny(det)
			if err != nil {
				grpclog.Printf("Failed to marshal any: %v", err)
			} else {
				body.Details = append(body.Details, a)
			}
		}
	}

	buf, merr := marshaler.Marshal(body)
	if merr != nil {
		grpclog.Printf("Failed to marshal error message %q: %v", body, merr)
		w.WriteHeader(http.StatusInternalServerError)
		if _, err := io.WriteString(w, fallback); err != nil {
			grpclog.Printf("Failed to write response: %v", err)
		}
		return
	}

	md, ok := ServerMetadataFromContext(ctx)
	if !ok {
		grpclog.Printf("Failed to extract ServerMetadata from context")
	}

	handleForwardResponseServerMetadata(w, mux, md)
	handleForwardResponseTrailerHeader(w, md)
	st := HTTPStatusFromCode(s.Code())
	w.WriteHeader(st)
	if _, err := w.Write(buf); err != nil {
		grpclog.Printf("Failed to write response: %v", err)
	}

	handleForwardResponseTrailer(w, md)
}

// DefaultOtherErrorHandler is the default implementation of OtherErrorHandler.
// It simply writes a string representation of the given error into "w".
func DefaultOtherErrorHandler(w http.ResponseWriter, _ *http.Request, msg string, code int) {
	http.Error(w, msg, code)
}
<commit_msg>do not add error details to REST/JSON body in DefaultHTTPError<commit_after>package runtime

import (
	"io"
	"net/http"

	"context"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes/any"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/grpclog"
	"google.golang.org/grpc/status"
)

// HTTPStatusFromCode converts a gRPC error code into the corresponding HTTP response status.
// See: https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto
func HTTPStatusFromCode(code codes.Code) int {
	switch code {
	case codes.OK:
		return http.StatusOK
	case codes.Canceled:
		return http.StatusRequestTimeout
	case codes.Unknown:
		return http.StatusInternalServerError
	case codes.InvalidArgument:
		return http.StatusBadRequest
	case codes.DeadlineExceeded:
		return http.StatusGatewayTimeout
	case codes.NotFound:
		return http.StatusNotFound
	case codes.AlreadyExists:
		return http.StatusConflict
	case codes.PermissionDenied:
		return http.StatusForbidden
	case codes.Unauthenticated:
		return http.StatusUnauthorized
	case codes.ResourceExhausted:
		return http.StatusTooManyRequests
	case codes.FailedPrecondition:
		return http.StatusBadRequest
	case codes.Aborted:
		return http.StatusConflict
	case codes.OutOfRange:
		return http.StatusBadRequest
	case codes.Unimplemented:
		return http.StatusNotImplemented
	case codes.Internal:
		return http.StatusInternalServerError
	case codes.Unavailable:
		return http.StatusServiceUnavailable
	case codes.DataLoss:
		return http.StatusInternalServerError
	}

	grpclog.Printf("Unknown gRPC error code: %v", code)
	return http.StatusInternalServerError
}

var (
	// HTTPError replies to the request with the error.
	// You can set a custom function to this variable to customize error format.
	HTTPError = DefaultHTTPError
	// OtherErrorHandler handles the following error used by the gateway: StatusMethodNotAllowed StatusNotFound and StatusBadRequest
	OtherErrorHandler = DefaultOtherErrorHandler
)

type errorBody struct {
	Error   string     `protobuf:"bytes,1,name=error" json:"error"`
	Code    int32      `protobuf:"varint,2,name=code" json:"code"`
	Details []*any.Any `protobuf:"bytes,3,rep,name=details" json:"details,omitempty"`
}

// Make this also conform to proto.Message for builtin JSONPb Marshaler
func (e *errorBody) Reset()         { *e = errorBody{} }
func (e *errorBody) String() string { return proto.CompactTextString(e) }
func (*errorBody) ProtoMessage()    {}

// DefaultHTTPError is the default implementation of HTTPError.
// If "err" is an error from gRPC system, the function replies with the status code mapped by HTTPStatusFromCode.
// If otherwise, it replies with http.StatusInternalServerError.
//
// The response body returned by this function is a JSON object,
// which contains a member whose key is "error" and whose value is err.Error().
func DefaultHTTPError(ctx context.Context, mux *ServeMux, marshaler Marshaler, w http.ResponseWriter, _ *http.Request, err error) {
	const fallback = `{"error": "failed to marshal error message"}`

	w.Header().Del("Trailer")
	if w.Header().Get("Content-Type") == "" {
		w.Header().Set("Content-Type", marshaler.ContentType())
	}

	s, ok := status.FromError(err)
	if !ok {
		s = status.New(codes.Unknown, err.Error())
	}

	body := &errorBody{
		Error: s.Message(),
		Code:  int32(s.Code()),
	}

	// for _, detail := range s.Details() {
	// 	if det, ok := detail.(proto.Message); ok {
	// 		a, err := ptypes.MarshalAny(det)
	// 		if err != nil {
	// 			grpclog.Printf("Failed to marshal any: %v", err)
	// 		} else {
	// 			body.Details = append(body.Details, a)
	// 		}
	// 	}
	// }

	buf, merr := marshaler.Marshal(body)
	if merr != nil {
		grpclog.Printf("Failed to marshal error message %q: %v", body, merr)
		w.WriteHeader(http.StatusInternalServerError)
		if _, err := io.WriteString(w, fallback); err != nil {
			grpclog.Printf("Failed to write response: %v", err)
		}
		return
	}

	md, ok := ServerMetadataFromContext(ctx)
	if !ok {
		grpclog.Printf("Failed to extract ServerMetadata from context")
	}

	handleForwardResponseServerMetadata(w, mux, md)
	handleForwardResponseTrailerHeader(w, md)
	st := HTTPStatusFromCode(s.Code())
	w.WriteHeader(st)
	if _, err := w.Write(buf); err != nil {
		grpclog.Printf("Failed to write response: %v", err)
	}

	handleForwardResponseTrailer(w, md)
}

// DefaultOtherErrorHandler is the default implementation of OtherErrorHandler.
// It simply writes a string representation of the given error into "w".
func DefaultOtherErrorHandler(w http.ResponseWriter, _ *http.Request, msg string, code int) {
	http.Error(w, msg, code)
}
<|endoftext|>


<commit_before>package event

import ()

const (
	CLIENT_REGISTRATION = "CR" //client registered (for the first time)
	CLIENT_AUTO_REREGI  = "CA" //client automatically re-registered
	CLIENT_UNREGISTER   = "CU" //client unregistered
	WORK_CHECKOUT       = "WC" //workunit checkout
	WORK_FAIL           = "WF" //workunit fails running
	WORK_FAILED         = "W!" //workunit fails running (not recoverable)
	//server only events
	SERVER_START         = "SS" //awe-server start
	SERVER_RECOVER       = "SR" //awe-server start with recover option  (-recover)
	DEBUG_LEVEL          = "DL" //debug level changed
	QUEUE_RESUME         = "QR" //awe-server queue resumed if suspended
	QUEUE_SUSPEND        = "QS" //awe-server queue suspended, not handing out work
	JOB_SUBMISSION       = "JQ" //job submitted
	JOB_IMPORT           = "JI" //job imported
	TASK_ENQUEUE         = "TQ" //task parsed and enqueue
	WORK_DONE            = "WD" //workunit received successful feedback from client
	WORK_REQUEUE         = "WR" //workunit requeue after receive failed feedback from client
	WORK_SUSPEND         = "WP" //workunit suspend after failing for conf.Max_Failure times
	TASK_DONE            = "TD" //task done (all the workunits in the task have finished)
	TASK_SKIPPED         = "TS" //task skipped (skip option > 0)
	JOB_DONE             = "JD" //job done (all the tasks in the job have finished)
	JOB_SUSPEND          = "JP" //job suspended
	JOB_DELETED          = "JL" //job deleted
	JOB_EXPIRED          = "JE" //job expired
	JOB_FULL_DELETE      = "JR" //job removed form mongodb (deleted fully)
	JOB_FAILED_PERMANENT = "JF"
	//client only events
	WORK_START     = "WS" //workunit command start running
	WORK_END       = "WE" //workunit command finish running
	WORK_RETURN    = "WR" //send back failed workunit to server
	WORK_DISCARD   = "WI" //workunit discarded after receiving discard signal from server
	PRE_WORK_START = "PS" //workunit command start running
	PRE_WORK_END   = "PE" //workunit command finish running
	PRE_IN         = "PI" //start fetching predata file from url
	PRE_READY      = "PR" //predata file is available
	FILE_IN        = "FI" //start fetching input file from shock
	FILE_READY     = "FR" //finish fetching input file from shock
	FILE_OUT       = "FO" //start pushing output file to shock
	FILE_DONE      = "FD" //finish pushing output file to shock
	ATTR_IN        = "AI" //start fetching input attributes from shock
	ATTR_READY     = "AR" //finish fetching input attributes from shock
	//proxy only events
	WORK_QUEUED = "WQ" //workunit queued at proxy
)

var EventDiscription = map[string]map[string]string{
	"general": map[string]string{
		"CR": "client registered (for the first time)",
		"CA": "client automatically re-registered",
		"CU": "client unregistered",
		"WC": "workunit checkout",
		"WF": "workunit fails running",
		"W!": "workunit failed running (not recoverable)",
	},
	"server": map[string]string{
		"SS": "awe-server start",
		"SR": "awe-server start with recover option  (-recover)",
		"DL": "debug level changed",
		"QR": "awe-server queue resumed if suspended",
		"QS": "awe-server queue suspended, not handing out work",
		"JQ": "job submitted",
		"JI": "job imported",
		"TQ": "task parsed and enqueue",
		"WD": "workunit received successful feedback from client",
		"WR": "workunit requeue after receive failed feedback from client",
		"WP": "workunit suspend after failing for conf.Max_Failure times",
		"TD": "task done (all the workunits in the task have finished)",
		"TS": "task skipped (skip option > 0)",
		"JD": "job done (all the tasks in the job have finished)",
		"JP": "job suspended",
		"JL": "job deleted",
		"JE": "job expired",
		"JR": "job removed form mongodb (deleted fully)",
		"JF": "job failed permanently",
	},
	"client": map[string]string{
		"WS": "workunit command start running",
		"WE": "workunit command finish running",
		"WR": "send back failed workunit to server",
		"WI": "workunit discarded after receiving discard signal from server",
		"PS": "workunit command start running",
		"PE": "workunit command finish running",
		"PI": "start fetching predata file from url",
		"PR": "predata file is available",
		"FI": "start fetching input file from shock",
		"FR": "finish fetching input file from shock",
		"FO": "start pushing output file to shock",
		"FD": "finish pushing output file to shock",
		"AI": "start fetching input attributes from shock",
		"AR": "finish fetching input attributes from shock",
	},
	"proxy": map[string]string{
		"WQ": "workunit queued at proxy",
	},
}
<commit_msg>comments<commit_after>package event

import ()

const (
	CLIENT_REGISTRATION = "CR" //client registered (for the first time)
	CLIENT_AUTO_REREGI  = "CA" //client automatically re-registered
	CLIENT_UNREGISTER   = "CU" //client unregistered
	WORK_CHECKOUT       = "WC" //workunit checkout
	WORK_FAIL           = "WF" //workunit fails running
	WORK_FAILED         = "W!" //workunit fails running (not recoverable)
	//server only events
	SERVER_START         = "SS" //awe-server start
	SERVER_RECOVER       = "SR" //awe-server start with recover option  (-recover)
	DEBUG_LEVEL          = "DL" //debug level changed
	QUEUE_RESUME         = "QR" //awe-server queue resumed if suspended
	QUEUE_SUSPEND        = "QS" //awe-server queue suspended, not handing out work
	JOB_SUBMISSION       = "JQ" //job submitted
	JOB_IMPORT           = "JI" //job imported
	TASK_ENQUEUE         = "TQ" //task parsed and enqueue
	WORK_DONE            = "WD" //workunit received successful feedback from client
	WORK_REQUEUE         = "WR" //workunit requeue after receive failed feedback from client
	WORK_SUSPEND         = "WP" //workunit suspend after failing for conf.Max_Failure times
	TASK_DONE            = "TD" //task done (all the workunits in the task have finished)
	TASK_SKIPPED         = "TS" //task skipped (skip option > 0)
	JOB_DONE             = "JD" //job done (all the tasks in the job have finished)
	JOB_SUSPEND          = "JP" //job suspended
	JOB_DELETED          = "JL" //job deleted
	JOB_EXPIRED          = "JE" //job expired
	JOB_FULL_DELETE      = "JR" //job removed form mongodb (deleted fully)
	JOB_FAILED_PERMANENT = "JF" //job failed permanently
	//client only events
	WORK_START     = "WS" //workunit command start running
	WORK_END       = "WE" //workunit command finish running
	WORK_RETURN    = "WR" //send back failed workunit to server
	WORK_DISCARD   = "WI" //workunit discarded after receiving discard signal from server
	PRE_WORK_START = "PS" //workunit command start running
	PRE_WORK_END   = "PE" //workunit command finish running
	PRE_IN         = "PI" //start fetching predata file from url
	PRE_READY      = "PR" //predata file is available
	FILE_IN        = "FI" //start fetching input file from shock
	FILE_READY     = "FR" //finish fetching input file from shock
	FILE_OUT       = "FO" //start pushing output file to shock
	FILE_DONE      = "FD" //finish pushing output file to shock
	ATTR_IN        = "AI" //start fetching input attributes from shock
	ATTR_READY     = "AR" //finish fetching input attributes from shock
	//proxy only events
	WORK_QUEUED = "WQ" //workunit queued at proxy
)

var EventDiscription = map[string]map[string]string{
	"general": map[string]string{
		"CR": "client registered (for the first time)",
		"CA": "client automatically re-registered",
		"CU": "client unregistered",
		"WC": "workunit checkout",
		"WF": "workunit fails running",
		"W!": "workunit failed running (not recoverable)",
	},
	"server": map[string]string{
		"SS": "awe-server start",
		"SR": "awe-server start with recover option  (-recover)",
		"DL": "debug level changed",
		"QR": "awe-server queue resumed if suspended",
		"QS": "awe-server queue suspended, not handing out work",
		"JQ": "job submitted",
		"JI": "job imported",
		"TQ": "task parsed and enqueue",
		"WD": "workunit received successful feedback from client",
		"WR": "workunit requeue after receive failed feedback from client",
		"WP": "workunit suspend after failing for conf.Max_Failure times",
		"TD": "task done (all the workunits in the task have finished)",
		"TS": "task skipped (skip option > 0)",
		"JD": "job done (all the tasks in the job have finished)",
		"JP": "job suspended",
		"JL": "job deleted",
		"JE": "job expired",
		"JR": "job removed form mongodb (deleted fully)",
		"JF": "job failed permanently",
	},
	"client": map[string]string{
		"WS": "workunit command start running",
		"WE": "workunit command finish running",
		"WR": "send back failed workunit to server",
		"WI": "workunit discarded after receiving discard signal from server",
		"PS": "workunit command start running",
		"PE": "workunit command finish running",
		"PI": "start fetching predata file from url",
		"PR": "predata file is available",
		"FI": "start fetching input file from shock",
		"FR": "finish fetching input file from shock",
		"FO": "start pushing output file to shock",
		"FD": "finish pushing output file to shock",
		"AI": "start fetching input attributes from shock",
		"AR": "finish fetching input attributes from shock",
	},
	"proxy": map[string]string{
		"WQ": "workunit queued at proxy",
	},
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package stream

import (
	"encoding/json"
	"testing"
)

func TestConfigSanitised(t *testing.T) {
	var err error
	var dat interface{}
	var actBytes []byte

	c := NewConfig()
	c.Input.Processors = nil
	c.Output.Processors = nil

	exp := `{` +
		`"input":{"type":"stdin","stdin":{"delimiter":"","max_buffer":1000000,"multipart":false}},` +
		`"buffer":{"type":"none","none":{}},` +
		`"pipeline":{"processors":[],"threads":1},` +
		`"output":{"type":"stdout","stdout":{"delimiter":""}}` +
		`}`

	if dat, err = c.Sanitised(); err != nil {
		t.Fatal(err)
	}
	if actBytes, err = json.Marshal(dat); err != nil {
		t.Fatal(err)
	}
	if act := string(actBytes); exp != act {
		t.Errorf("Wrong sanitised output: %v != %v", act, exp)
	}

	c = NewConfig()
	c.Input.Processors = nil
	c.Input.Type = "file"
	c.Output.Processors = nil
	c.Output.Type = "kafka"

	{
	}
	exp = `{` +
		`"input":{"type":"file","file":{"delimiter":"","max_buffer":1000000,"multipart":false,"path":""}},` +
		`"buffer":{"type":"none","none":{}},` +
		`"pipeline":{"processors":[],"threads":1},` +
		`"output":{"type":"kafka","kafka":{"ack_replicas":false,"addresses":["localhost:9092"],"client_id":"benthos_kafka_output","compression":"none","key":"","max_msg_bytes":1000000,"round_robin_partitions":false,"target_version":"1.0.0","timeout_ms":5000,"topic":"benthos_stream"}}` +
		`}`

	if dat, err = c.Sanitised(); err != nil {
		t.Fatal(err)
	}
	if actBytes, err = json.Marshal(dat); err != nil {
		t.Fatal(err)
	}
	if act := string(actBytes); exp != act {
		t.Errorf("Wrong sanitised output: %v != %v", act, exp)
	}
}
<commit_msg>Fix config test<commit_after>// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package stream

import (
	"encoding/json"
	"testing"
)

func TestConfigSanitised(t *testing.T) {
	var err error
	var dat interface{}
	var actBytes []byte

	c := NewConfig()
	c.Input.Processors = nil
	c.Output.Processors = nil

	exp := `{` +
		`"input":{"type":"stdin","stdin":{"delimiter":"","max_buffer":1000000,"multipart":false}},` +
		`"buffer":{"type":"none","none":{}},` +
		`"pipeline":{"processors":[],"threads":1},` +
		`"output":{"type":"stdout","stdout":{"delimiter":""}}` +
		`}`

	if dat, err = c.Sanitised(); err != nil {
		t.Fatal(err)
	}
	if actBytes, err = json.Marshal(dat); err != nil {
		t.Fatal(err)
	}
	if act := string(actBytes); exp != act {
		t.Errorf("Wrong sanitised output: %v != %v", act, exp)
	}

	c = NewConfig()
	c.Input.Processors = nil
	c.Input.Type = "file"
	c.Output.Processors = nil
	c.Output.Type = "kafka"

	exp = `{` +
		`"input":{"type":"file","file":{"delimiter":"","max_buffer":1000000,"multipart":false,"path":""}},` +
		`"buffer":{"type":"none","none":{}},` +
		`"pipeline":{"processors":[],"threads":1},` +
		`"output":{"type":"kafka","kafka":{"ack_replicas":false,"addresses":["localhost:9092"],"client_id":"benthos_kafka_output","compression":"none","key":"","max_msg_bytes":1000000,"round_robin_partitions":false,"skip_cert_verify":false,"target_version":"1.0.0","timeout_ms":5000,"tls_enable":false,"topic":"benthos_stream"}}` +
		`}`

	if dat, err = c.Sanitised(); err != nil {
		t.Fatal(err)
	}
	if actBytes, err = json.Marshal(dat); err != nil {
		t.Fatal(err)
	}
	if act := string(actBytes); exp != act {
		t.Errorf("Wrong sanitised output: %v != %v", act, exp)
	}
}
<|endoftext|>


<commit_before>package peer

import (
	"net"

	"github.com/drausin/libri/libri/librarian/api"
	"google.golang.org/grpc"
)

// Connector creates and destroys connections with a peer.
type Connector interface {
	// Connect establishes the TCP connection with the peer if it doesn't already exist
	// and returns an api.LibrarianClient.
	Connect() (api.LibrarianClient, error)

	// Disconnect closes the connection with the peer.
	Disconnect() error

	// Equals determines whether the Connector has the same underlying address as the other.
	Equals(other Connector) bool

	// String returns a string representation of the public address.
	String() string
}

type connector struct {

	// RPC TCP address
	publicAddress *net.TCPAddr

	// Librarian client to peer
	client api.LibrarianClient

	// client connection to the peer
	clientConn *grpc.ClientConn
}

// NewConnector creates a Connector instance from an address.
func NewConnector(address *net.TCPAddr) Connector {
	return &connector{publicAddress: address}
}

// Connect establishes the TCP connection with the peer and establishes the Librarian client with
// it.
func (c *connector) Connect() (api.LibrarianClient, error) {
	if c.client == nil {
		// TODO (drausin) add SSL
		conn, err := grpc.Dial(c.publicAddress.String(), grpc.WithInsecure())
		if err != nil {
			return nil, err
		}
		c.client = api.NewLibrarianClient(conn)
		c.clientConn = conn
	}
	return c.client, nil
}

// Disconnect closes the connection with the peer.
func (c *connector) Disconnect() error {
	if c.client != nil {
		c.client = nil
		return c.clientConn.Close()
	}
	return nil
}

func (c *connector) Equals(other Connector) bool {
	return c.publicAddress.String() == other.(*connector).publicAddress.String()
}

func (c *connector) String() string {
	return c.publicAddress.String()
}
<commit_msg>remove SSL comment b/c not going to do it (#30)<commit_after>package peer

import (
	"net"

	"github.com/drausin/libri/libri/librarian/api"
	"google.golang.org/grpc"
)

// Connector creates and destroys connections with a peer.
type Connector interface {
	// Connect establishes the TCP connection with the peer if it doesn't already exist
	// and returns an api.LibrarianClient.
	Connect() (api.LibrarianClient, error)

	// Disconnect closes the connection with the peer.
	Disconnect() error

	// Equals determines whether the Connector has the same underlying address as the other.
	Equals(other Connector) bool

	// String returns a string representation of the public address.
	String() string
}

type connector struct {

	// RPC TCP address
	publicAddress *net.TCPAddr

	// Librarian client to peer
	client api.LibrarianClient

	// client connection to the peer
	clientConn *grpc.ClientConn
}

// NewConnector creates a Connector instance from an address.
func NewConnector(address *net.TCPAddr) Connector {
	return &connector{publicAddress: address}
}

// Connect establishes the TCP connection with the peer and establishes the Librarian client with
// it.
func (c *connector) Connect() (api.LibrarianClient, error) {
	if c.client == nil {
		conn, err := grpc.Dial(c.publicAddress.String(), grpc.WithInsecure())
		if err != nil {
			return nil, err
		}
		c.client = api.NewLibrarianClient(conn)
		c.clientConn = conn
	}
	return c.client, nil
}

// Disconnect closes the connection with the peer.
func (c *connector) Disconnect() error {
	if c.client != nil {
		c.client = nil
		return c.clientConn.Close()
	}
	return nil
}

func (c *connector) Equals(other Connector) bool {
	return c.publicAddress.String() == other.(*connector).publicAddress.String()
}

func (c *connector) String() string {
	return c.publicAddress.String()
}
<|endoftext|>


<commit_before>package tsdb

import (
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/yuuki/diamondb/lib/model"
)

func TestFetchMetricsFromDynamoDB(t *testing.T) {
	name := "roleA.r.{1,2}.loadavg"
	expected := []*model.Metric{
		model.NewMetric(
			"roleA.r.1.loadavg",
			[]*model.DataPoint{
				{120, 10.0},
				{180, 11.2},
				{240, 13.1},
			},
			60,
		),
		model.NewMetric(
			"roleA.r.2.loadavg",
			[]*model.DataPoint{
				{120, 1.0},
				{180, 1.2},
				{240, 1.1},
			},
			60,
		),
	}
	ctrl := SetMockDynamoDB(t, &MockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 0,
		Metrics:   expected,
	})
	defer ctrl.Finish()
	metrics, err := FetchMetricsFromDynamoDB(name, time.Unix(100, 0), time.Unix(300, 0))
	if assert.NoError(t, err) {
		assert.Exactly(t, expected, metrics)
	}
}

func TestGroupNames(t *testing.T) {
	var names []string
	for i := 1; i <= 5; i++ {
		names = append(names, fmt.Sprintf("server%d.loadavg5", i))
	}
	nameGroups := groupNames(names, 2)
	expected := [][]string{
		{"server1.loadavg5", "server2.loadavg5"},
		{"server3.loadavg5", "server4.loadavg5"},
		{"server5.loadavg5"},
	}
	assert.Exactly(t, expected, nameGroups)
}

func TestBatchGet(t *testing.T) {
	expected := []*model.Metric{
		model.NewMetric(
			"server1.loadavg5",
			[]*model.DataPoint{
				{1465516810, 10.0},
			},
			60,
		),
		model.NewMetric(
			"server2.loadavg5",
			[]*model.DataPoint{
				{1465516810, 15.0},
			},
			60,
		),
	}
	ctrl := SetMockDynamoDB(t, &MockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 1000,
		Metrics:   expected,
	})
	defer ctrl.Finish()
	metrics, err := batchGet(
		&timeSlot{DynamoDBTableOneHour + "-0", 1000},
		[]string{"server1.loadavg5", "server2.loadavg5"},
		60,
	)
	assert.NoError(t, err)
	assert.Exactly(t, expected, metrics)
}

func TestConcurrentBatchGet(t *testing.T) {
	expected := []*model.Metric{
		model.NewMetric(
			"server1.loadavg5",
			[]*model.DataPoint{
				{1465516810, 10.0},
			},
			60,
		),
		model.NewMetric(
			"server2.loadavg5",
			[]*model.DataPoint{
				{1465516810, 15.0},
			},
			60,
		),
	}
	ctrl := SetMockDynamoDB(t, &MockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 1000,
		Metrics:   expected,
	})
	defer ctrl.Finish()
	c := make(chan interface{})
	concurrentBatchGet(
		&timeSlot{DynamoDBTableOneHour + "-0", 1000},
		[]string{"server1.loadavg5", "server2.loadavg5"},
		60,
		c,
	)
	var metrics []*model.Metric
	ret := <-c
	metrics = append(metrics, ret.([]*model.Metric)...)
	assert.Exactly(t, expected, metrics)
}

func TestSplitName(t *testing.T) {
	name := "roleA.r.{1,2,3,4}.loadavg"
	names := splitName(name)
	expected := []string{
		"roleA.r.1.loadavg",
		"roleA.r.2.loadavg",
		"roleA.r.3.loadavg",
		"roleA.r.4.loadavg",
	}
	assert.Exactly(t, expected, names)
}

func TestListTablesByRange_1m1h(t *testing.T) {
	s, e := time.Unix(100, 0), time.Unix(6000, 0)
	slots, step := listTimeSlots(s, e)
	assert.Exactly(t, 60, step)
	expected := []*timeSlot{
		{
			tableName: DynamoDBTableOneHour + "-0",
			itemEpoch: 0,
		},
		{
			tableName: DynamoDBTableOneHour + "-0",
			itemEpoch: 3600,
		},
	}
	assert.Exactly(t, expected, slots)
}

func TestListTablesByRange_5m1d(t *testing.T) {
	s, e := time.Unix(10000, 0), time.Unix(100000, 0)
	slots, step := listTimeSlots(s, e)
	assert.Exactly(t, 300, step)
	expected := []*timeSlot{
		{
			tableName: DynamoDBTableOneDay + "-0",
			itemEpoch: 0,
		},
		{
			tableName: DynamoDBTableOneDay + "-86400",
			itemEpoch: 86400,
		},
	}
	assert.Exactly(t, expected, slots)
}

func TestListTablesByRange_1h7d(t *testing.T) {
	s, e := time.Unix(100000, 0), time.Unix(1000000, 0)
	slots, step := listTimeSlots(s, e)
	assert.Exactly(t, 3600, step)
	expected := []*timeSlot{
		{
			tableName: DynamoDBTableOneWeek + "-0",
			itemEpoch: 0,
		},
		{
			tableName: DynamoDBTableOneWeek + "-604800",
			itemEpoch: 604800,
		},
	}
	assert.Exactly(t, expected, slots)
}

func TestListTablesByRange_1d360d(t *testing.T) {
	s, e := time.Unix(1000000, 0), time.Unix(100000000, 0)
	slots, step := listTimeSlots(s, e)
	assert.Exactly(t, 86400, step)
	expected := []*timeSlot{
		{
			tableName: DynamoDBTableOneYear + "-0",
			itemEpoch: 0,
		},
		{
			tableName: DynamoDBTableOneYear + "-31104000",
			itemEpoch: 31104000,
		},
		{
			tableName: DynamoDBTableOneYear + "-62208000",
			itemEpoch: 62208000,
		},
		{
			tableName: DynamoDBTableOneYear + "-93312000",
			itemEpoch: 93312000,
		},
	}
	assert.Exactly(t, expected, slots)
}
<commit_msg>Remove assert from lib/tsdb/dynamodb_test.go<commit_after>package tsdb

import (
	"fmt"
	"reflect"
	"testing"
	"time"

	"github.com/yuuki/diamondb/lib/model"
)

func TestFetchMetricsFromDynamoDB(t *testing.T) {
	name := "roleA.r.{1,2}.loadavg"
	expected := []*model.Metric{
		model.NewMetric(
			"roleA.r.1.loadavg",
			[]*model.DataPoint{
				{120, 10.0},
				{180, 11.2},
				{240, 13.1},
			},
			60,
		),
		model.NewMetric(
			"roleA.r.2.loadavg",
			[]*model.DataPoint{
				{120, 1.0},
				{180, 1.2},
				{240, 1.1},
			},
			60,
		),
	}
	ctrl := SetMockDynamoDB(t, &MockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 0,
		Metrics:   expected,
	})
	defer ctrl.Finish()

	metrics, err := FetchMetricsFromDynamoDB(name, time.Unix(100, 0), time.Unix(300, 0))
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if !reflect.DeepEqual(metrics, expected) {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", expected, metrics)
	}
}

func TestGroupNames(t *testing.T) {
	var names []string
	for i := 1; i <= 5; i++ {
		names = append(names, fmt.Sprintf("server%d.loadavg5", i))
	}
	nameGroups := groupNames(names, 2)
	expected := [][]string{
		{"server1.loadavg5", "server2.loadavg5"},
		{"server3.loadavg5", "server4.loadavg5"},
		{"server5.loadavg5"},
	}
	if !reflect.DeepEqual(nameGroups, expected) {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", expected, nameGroups)
	}
}

func TestBatchGet(t *testing.T) {
	expected := []*model.Metric{
		model.NewMetric(
			"server1.loadavg5",
			[]*model.DataPoint{
				{1465516810, 10.0},
			},
			60,
		),
		model.NewMetric(
			"server2.loadavg5",
			[]*model.DataPoint{
				{1465516810, 15.0},
			},
			60,
		),
	}
	ctrl := SetMockDynamoDB(t, &MockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 1000,
		Metrics:   expected,
	})
	defer ctrl.Finish()

	metrics, err := batchGet(
		&timeSlot{DynamoDBTableOneHour + "-0", 1000},
		[]string{"server1.loadavg5", "server2.loadavg5"},
		60,
	)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if !reflect.DeepEqual(metrics, expected) {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", expected, metrics)
	}
}

func TestConcurrentBatchGet(t *testing.T) {
	expected := []*model.Metric{
		model.NewMetric(
			"server1.loadavg5",
			[]*model.DataPoint{
				{1465516810, 10.0},
			},
			60,
		),
		model.NewMetric(
			"server2.loadavg5",
			[]*model.DataPoint{
				{1465516810, 15.0},
			},
			60,
		),
	}
	ctrl := SetMockDynamoDB(t, &MockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 1000,
		Metrics:   expected,
	})
	defer ctrl.Finish()

	c := make(chan interface{})
	concurrentBatchGet(
		&timeSlot{DynamoDBTableOneHour + "-0", 1000},
		[]string{"server1.loadavg5", "server2.loadavg5"},
		60,
		c,
	)
	var metrics []*model.Metric
	ret := <-c
	metrics = append(metrics, ret.([]*model.Metric)...)
	if !reflect.DeepEqual(metrics, expected) {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", expected, metrics)
	}
}

func TestSplitName(t *testing.T) {
	name := "roleA.r.{1,2,3,4}.loadavg"
	names := splitName(name)
	expected := []string{
		"roleA.r.1.loadavg",
		"roleA.r.2.loadavg",
		"roleA.r.3.loadavg",
		"roleA.r.4.loadavg",
	}
	if !reflect.DeepEqual(names, expected) {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", expected, names)
	}
}

func TestListTablesByRange_1m1h(t *testing.T) {
	slots, step := listTimeSlots(time.Unix(100, 0), time.Unix(6000, 0))

	if step != 60 {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", 60, step)
	}
	expected := []*timeSlot{
		{
			tableName: DynamoDBTableOneHour + "-0",
			itemEpoch: 0,
		},
		{
			tableName: DynamoDBTableOneHour + "-0",
			itemEpoch: 3600,
		},
	}
	if !reflect.DeepEqual(slots, expected) {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", expected, slots)
	}
}

func TestListTablesByRange_5m1d(t *testing.T) {
	slots, step := listTimeSlots(time.Unix(10000, 0), time.Unix(100000, 0))

	if step != 300 {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", 300, step)
	}
	expected := []*timeSlot{
		{
			tableName: DynamoDBTableOneDay + "-0",
			itemEpoch: 0,
		},
		{
			tableName: DynamoDBTableOneDay + "-86400",
			itemEpoch: 86400,
		},
	}
	if !reflect.DeepEqual(slots, expected) {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", expected, slots)
	}
}

func TestListTablesByRange_1h7d(t *testing.T) {
	slots, step := listTimeSlots(time.Unix(100000, 0), time.Unix(1000000, 0))

	if step != 3600 {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", 3600, step)
	}
	expected := []*timeSlot{
		{
			tableName: DynamoDBTableOneWeek + "-0",
			itemEpoch: 0,
		},
		{
			tableName: DynamoDBTableOneWeek + "-604800",
			itemEpoch: 604800,
		},
	}
	if !reflect.DeepEqual(slots, expected) {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", expected, slots)
	}
}

func TestListTablesByRange_1d360d(t *testing.T) {
	slots, step := listTimeSlots(time.Unix(1000000, 0), time.Unix(100000000, 0))

	if step != 86400 {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", 86400, step)
	}
	expected := []*timeSlot{
		{
			tableName: DynamoDBTableOneYear + "-0",
			itemEpoch: 0,
		},
		{
			tableName: DynamoDBTableOneYear + "-31104000",
			itemEpoch: 31104000,
		},
		{
			tableName: DynamoDBTableOneYear + "-62208000",
			itemEpoch: 62208000,
		},
		{
			tableName: DynamoDBTableOneYear + "-93312000",
			itemEpoch: 93312000,
		},
	}
	if !reflect.DeepEqual(slots, expected) {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", expected, slots)
	}
}
<|endoftext|>


<commit_before>package container551

import (
	"github.com/go51/cookie551"
	"github.com/go51/log551"
	"net/http"
)

type Container struct {
	sid    string
	w      http.ResponseWriter
	r      *http.Request
	logger *log551.Log551
	cookie *cookie551.Cookie
}

func New() *Container {
	return &Container{}
}

func (c *Container) SetSID(sid string) {
	c.sid = sid
}

func (c *Container) SID() string {
	return c.sid
}

func (c *Container) SetResponseWriter(w http.ResponseWriter) {
	c.w = w
}

func (c *Container) ResponseWriter() http.ResponseWriter {
	return c.w
}

func (c *Container) SetRequest(r *http.Request) {
	c.r = r
}

func (c *Container) Request() *http.Request {
	return c.r
}

func (c *Container) SetLogger(logger *log551.Log551) {
	c.logger = logger
}

func (c *Container) Logger() *log551.Log551 {
	return c.logger
}

func (c *Container) SetCookie(cookie *cookie551.Cookie) {
	c.cookie = cookie
}

func (c *Container) Cookie() *cookie551.Cookie {
	return c.cookie
}
<commit_msg>refs #7 mysql551 を保持できるようにする<commit_after>package container551

import (
	"github.com/go51/cookie551"
	"github.com/go51/log551"
	"github.com/go51/mysql551"
	"net/http"
)

type Container struct {
	sid    string
	w      http.ResponseWriter
	r      *http.Request
	logger *log551.Log551
	cookie *cookie551.Cookie
	db     *mysql551.Mysql
}

func New() *Container {
	return &Container{}
}

func (c *Container) SetSID(sid string) {
	c.sid = sid
}

func (c *Container) SID() string {
	return c.sid
}

func (c *Container) SetResponseWriter(w http.ResponseWriter) {
	c.w = w
}

func (c *Container) ResponseWriter() http.ResponseWriter {
	return c.w
}

func (c *Container) SetRequest(r *http.Request) {
	c.r = r
}

func (c *Container) Request() *http.Request {
	return c.r
}

func (c *Container) SetLogger(logger *log551.Log551) {
	c.logger = logger
}

func (c *Container) Logger() *log551.Log551 {
	return c.logger
}

func (c *Container) SetCookie(cookie *cookie551.Cookie) {
	c.cookie = cookie
}

func (c *Container) Cookie() *cookie551.Cookie {
	return c.cookie
}

func (c *Container) SetDb(db *mysql551.Mysql) {
	c.db = db
}

func (c *Container) Db() *mysql551.Mysql {
	return c.db
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net"
	"sync"
	"time"

	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
)

var (
	// ErrNoActiveConn is an error returned when this component
	// is not yet connected to the block server.
	ErrNoActiveConn = errors.New("Not connected to block server")
	// ErrConnTimeout is an error returned timed out (repeatedly)
	// while trying to connect to the block server.
	ErrConnTimeout = errors.New("Repeatedly failed to connect to block server")
	// BServerTimeout is the timeout for communications with block server.
	BServerTimeout = 60 * time.Second
)

// BlockServerRemote implements the BlockServer interface and
// represents a remote KBFS block server.
type BlockServerRemote struct {
	config Config

	srvAddr  string
	certFile string

	conn      net.Conn
	clt       keybase1.BlockClient
	connected bool

	lastTried time.Time
	retryMu   sync.Mutex
}

// NewBlockServerRemote constructs a new BlockServerRemote for the
// given address.
func NewBlockServerRemote(config Config, blkSrvAddr string) *BlockServerRemote {
	b := &BlockServerRemote{
		config:   config,
		srvAddr:  blkSrvAddr,
		certFile: "./cert.pem",
	}

	if err := b.ConnectOnce(); err != nil {
		go b.Reconnect()
	}

	return b
}

// TLSConnect connects over TLS to the given server, expecting the
// connection to be authenticated with the given certificate.
func TLSConnect(cFile string, Addr string) (conn net.Conn, err error) {
	CAPool := x509.NewCertPool()
	var cacert []byte
	cacert, err = ioutil.ReadFile(cFile)
	if err != nil {
		return
	}
	CAPool.AppendCertsFromPEM(cacert)

	config := tls.Config{RootCAs: CAPool}
	conn, err = tls.Dial("tcp", Addr, &config)
	if err != nil {
		return
	}
	return
}

// Config returns the configuration object
func (b *BlockServerRemote) Config() Config {
	return b.config
}

// ConnectOnce tries once to connect to the remote block server.
func (b *BlockServerRemote) ConnectOnce() error {
	var err error
	if b.conn, err = TLSConnect(b.certFile, b.srvAddr); err != nil {
		return err
	}

	b.clt = keybase1.BlockClient{Cli: rpc2.NewClient(
		rpc2.NewTransport(b.conn, libkb.NewRpcLogFactory(), libkb.WrapError), libkb.UnwrapError)}

	var session *libkb.Session
	if session, err = b.config.KBPKI().GetSession(); err != nil {
		libkb.G.Log.Warning("error getting session, disconnect from backend: %q", err)
		b.conn.Close()
		return err
	}
	/*
		var token []byte
		if token, err = base64.StdEncoding.DecodeString(session.GetToken()); err != nil {
			b.conn.Close()
			return err
		}
	*/
	if err = b.clt.EstablishSession(session.GetToken()); err != nil {
		b.conn.Close()
		return err
	}

	b.connected = true
	return nil
}

// WaitForReconnect waits for the timeout period to reconnect to the
// server.
func (b *BlockServerRemote) WaitForReconnect() error {
	timeout := time.Now().Add(BServerTimeout)

	b.retryMu.Lock()
	defer b.retryMu.Unlock()

	for !b.connected {
		b.retryMu.Unlock()
		if time.Now().After(timeout) {
			return ErrConnTimeout
		}
		time.Sleep(1 * time.Second)
		b.retryMu.Lock()
	}
	return nil
}

// Reconnect reconnects to block server.
func (b *BlockServerRemote) Reconnect() {
	b.retryMu.Lock()
	defer b.retryMu.Unlock()

	for b.ConnectOnce() != nil {
		b.retryMu.Unlock()
		time.Sleep(1 * time.Second)
		b.retryMu.Lock()
	}
	return
}

// Shutdown closes the connection to this remote block server.
func (b *BlockServerRemote) Shutdown() {
	b.conn.Close()
}

// Get implements the BlockServer interface for BlockServerRemote.
func (b *BlockServerRemote) Get(id BlockID, context BlockContext) (
	[]byte, BlockCryptKeyServerHalf, error) {
	if !b.connected {
		if err := b.WaitForReconnect(); err != nil {
			return nil, BlockCryptKeyServerHalf{}, err
		}
	}
	bid := keybase1.BlockIdCombo{
		BlockHash: hex.EncodeToString(id[:]),
		Size:      int(context.GetQuotaSize()),
		ChargedTo: context.GetWriter(),
	}
	//XXX: if fails due to connection problem, should reconnect
	res, err := b.clt.GetBlock(bid)
	if err != nil {
		return nil, BlockCryptKeyServerHalf{}, err
	}

	// TODO: return the server-half of the block key
	bk := BlockCryptKeyServerHalf{}
	if kbuf, err := hex.DecodeString(res.BlockKey); err == nil {
		copy(bk.ServerHalf[:], kbuf)
	}
	return res.Buf, bk, err
}

// Put implements the BlockServer interface for BlockServerRemote.
// TODO: store the server-half of the block key
func (b *BlockServerRemote) Put(id BlockID, tlfID DirID, context BlockContext,
	buf []byte, serverHalf BlockCryptKeyServerHalf) error {
	if !b.connected {
		if err := b.WaitForReconnect(); err != nil {
			return err
		}
	}
	arg := keybase1.PutBlockArg{
		Bid: keybase1.BlockIdCombo{
			ChargedTo: context.GetWriter(),
			BlockHash: hex.EncodeToString(id[:]),
			Size:      int(context.GetQuotaSize()),
		},
		BlockKey: hex.EncodeToString(serverHalf.ServerHalf[:]),
		Folder:   "", // TODO: convert tlfID to the appropriate type
		Buf:      buf,
	}
	err := b.clt.PutBlock(arg)
	if err != nil {
		libkb.G.Log.Warning("PUT to backend err : %q", err)
	}
	return err
}

// Delete implements the BlockServer interface for BlockServerRemote.
func (b *BlockServerRemote) Delete(id BlockID, context BlockContext) error {
	arg := keybase1.DecBlockReferenceArg{
		Bid: keybase1.BlockIdCombo{
			ChargedTo: context.GetWriter(), //should be the original chargedto
			BlockHash: hex.EncodeToString(id[:]),
			Size:      int(context.GetQuotaSize()),
		},
		Nonce:     "0000",
		Folder:    "",
		ChargedTo: context.GetWriter(), //the actual writer to decrement quota from
	}
	err := b.clt.DecBlockReference(arg)
	if err != nil {
		libkb.G.Log.Warning("PUT to backend err : %q", err)
	}
	return err
}
<commit_msg>pass folder info to backend<commit_after>package libkbfs

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net"
	"sync"
	"time"

	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
)

var (
	// ErrNoActiveConn is an error returned when this component
	// is not yet connected to the block server.
	ErrNoActiveConn = errors.New("Not connected to block server")
	// ErrConnTimeout is an error returned timed out (repeatedly)
	// while trying to connect to the block server.
	ErrConnTimeout = errors.New("Repeatedly failed to connect to block server")
	// BServerTimeout is the timeout for communications with block server.
	BServerTimeout = 60 * time.Second
)

// BlockServerRemote implements the BlockServer interface and
// represents a remote KBFS block server.
type BlockServerRemote struct {
	config Config

	srvAddr  string
	certFile string

	conn      net.Conn
	clt       keybase1.BlockClient
	connected bool

	lastTried time.Time
	retryMu   sync.Mutex
}

// NewBlockServerRemote constructs a new BlockServerRemote for the
// given address.
func NewBlockServerRemote(config Config, blkSrvAddr string) *BlockServerRemote {
	b := &BlockServerRemote{
		config:   config,
		srvAddr:  blkSrvAddr,
		certFile: "./cert.pem",
	}

	if err := b.ConnectOnce(); err != nil {
		go b.Reconnect()
	}

	return b
}

// TLSConnect connects over TLS to the given server, expecting the
// connection to be authenticated with the given certificate.
func TLSConnect(cFile string, Addr string) (conn net.Conn, err error) {
	CAPool := x509.NewCertPool()
	var cacert []byte
	cacert, err = ioutil.ReadFile(cFile)
	if err != nil {
		return
	}
	CAPool.AppendCertsFromPEM(cacert)

	config := tls.Config{RootCAs: CAPool}
	conn, err = tls.Dial("tcp", Addr, &config)
	if err != nil {
		return
	}
	return
}

// Config returns the configuration object
func (b *BlockServerRemote) Config() Config {
	return b.config
}

// ConnectOnce tries once to connect to the remote block server.
func (b *BlockServerRemote) ConnectOnce() error {
	var err error
	if b.conn, err = TLSConnect(b.certFile, b.srvAddr); err != nil {
		return err
	}

	b.clt = keybase1.BlockClient{Cli: rpc2.NewClient(
		rpc2.NewTransport(b.conn, libkb.NewRpcLogFactory(), libkb.WrapError), libkb.UnwrapError)}

	var session *libkb.Session
	if session, err = b.config.KBPKI().GetSession(); err != nil {
		libkb.G.Log.Warning("error getting session, disconnect from backend: %q", err)
		b.conn.Close()
		return err
	}
	/*
		var token []byte
		if token, err = base64.StdEncoding.DecodeString(session.GetToken()); err != nil {
			b.conn.Close()
			return err
		}
	*/
	if err = b.clt.EstablishSession(session.GetToken()); err != nil {
		b.conn.Close()
		return err
	}

	b.connected = true
	return nil
}

// WaitForReconnect waits for the timeout period to reconnect to the
// server.
func (b *BlockServerRemote) WaitForReconnect() error {
	timeout := time.Now().Add(BServerTimeout)

	b.retryMu.Lock()
	defer b.retryMu.Unlock()

	for !b.connected {
		b.retryMu.Unlock()
		if time.Now().After(timeout) {
			return ErrConnTimeout
		}
		time.Sleep(1 * time.Second)
		b.retryMu.Lock()
	}
	return nil
}

// Reconnect reconnects to block server.
func (b *BlockServerRemote) Reconnect() {
	b.retryMu.Lock()
	defer b.retryMu.Unlock()

	for b.ConnectOnce() != nil {
		b.retryMu.Unlock()
		time.Sleep(1 * time.Second)
		b.retryMu.Lock()
	}
	return
}

// Shutdown closes the connection to this remote block server.
func (b *BlockServerRemote) Shutdown() {
	b.conn.Close()
}

// Get implements the BlockServer interface for BlockServerRemote.
func (b *BlockServerRemote) Get(id BlockID, context BlockContext) (
	[]byte, BlockCryptKeyServerHalf, error) {
	if !b.connected {
		if err := b.WaitForReconnect(); err != nil {
			return nil, BlockCryptKeyServerHalf{}, err
		}
	}
	bid := keybase1.BlockIdCombo{
		BlockHash: hex.EncodeToString(id[:]),
		Size:      int(context.GetQuotaSize()),
		ChargedTo: context.GetWriter(),
	}
	//XXX: if fails due to connection problem, should reconnect
	res, err := b.clt.GetBlock(bid)
	if err != nil {
		return nil, BlockCryptKeyServerHalf{}, err
	}

	// TODO: return the server-half of the block key
	bk := BlockCryptKeyServerHalf{}
	if kbuf, err := hex.DecodeString(res.BlockKey); err == nil {
		copy(bk.ServerHalf[:], kbuf)
	}
	return res.Buf, bk, err
}

// Put implements the BlockServer interface for BlockServerRemote.
// TODO: store the server-half of the block key
func (b *BlockServerRemote) Put(id BlockID, tlfID DirID, context BlockContext,
	buf []byte, serverHalf BlockCryptKeyServerHalf) error {
	if !b.connected {
		if err := b.WaitForReconnect(); err != nil {
			return err
		}
	}
	arg := keybase1.PutBlockArg{
		Bid: keybase1.BlockIdCombo{
			ChargedTo: context.GetWriter(),
			BlockHash: hex.EncodeToString(id[:]),
			Size:      int(context.GetQuotaSize()),
		},
		BlockKey: hex.EncodeToString(serverHalf.ServerHalf[:]),
		Folder:   hex.EncodeToString(tlfID[:]),
		Buf:      buf,
	}
	err := b.clt.PutBlock(arg)
	if err != nil {
		libkb.G.Log.Warning("PUT to backend err : %q", err)
	}
	return err
}

// Delete implements the BlockServer interface for BlockServerRemote.
func (b *BlockServerRemote) Delete(id BlockID, context BlockContext) error {
	arg := keybase1.DecBlockReferenceArg{
		Bid: keybase1.BlockIdCombo{
			ChargedTo: context.GetWriter(), //should be the original chargedto
			BlockHash: hex.EncodeToString(id[:]),
			Size:      int(context.GetQuotaSize()),
		},
		Nonce:     "0000",
		Folder:    "",
		ChargedTo: context.GetWriter(), //the actual writer to decrement quota from
	}
	err := b.clt.DecBlockReference(arg)
	if err != nil {
		libkb.G.Log.Warning("PUT to backend err : %q", err)
	}
	return err
}
<|endoftext|>


<commit_before>package app

import (
	"crypto/rand"
	"fmt"
	"io"
	"log"
	randMath "math/rand"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/DanielRenne/GoCore/core/dbServices"
	"github.com/DanielRenne/GoCore/core/fileCache"
	"github.com/DanielRenne/GoCore/core/ginServer"
	"github.com/DanielRenne/GoCore/core/serverSettings"
	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
)

type WebSocketConnection struct {
	sync.Mutex
	Id         string
	Connection *websocket.Conn
}

type WebSocketConnectionCollection struct {
	sync.RWMutex
	Connections []*WebSocketConnection
}

type WebSocketCallbackSync struct {
	sync.RWMutex
	callbacks []WebSocketCallback
}

type WebSocketCallback func(conn *WebSocketConnection, messageType int, data []byte)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

var WebSocketConnections WebSocketConnectionCollection
var WebSocketCallbacks WebSocketCallbackSync

func Initialize(path string) {
	serverSettings.Initialize(path)
	dbServices.Initialize()

	if serverSettings.WebConfig.Application.ReleaseMode == "release" {
		ginServer.Initialize(gin.ReleaseMode)
	} else {
		ginServer.Initialize(gin.DebugMode)
	}

	fileCache.Initialize()

}

func Run() {

	if serverSettings.WebConfig.Application.WebServiceOnly == false {

		loadHTMLTemplates()

		ginServer.Router.Static("/web", serverSettings.APP_LOCATION+"/web")

		ginServer.Router.GET("/ws", func(c *gin.Context) {
			webSocketHandler(c.Writer, c.Request)
		})
	}

	initializeStaticRoutes()

	go ginServer.Router.RunTLS(":"+strconv.Itoa(serverSettings.WebConfig.Application.HttpsPort), serverSettings.APP_LOCATION+"/keys/cert.pem", serverSettings.APP_LOCATION+"/keys/key.pem")

	log.Println("GoCore Application Started")

	ginServer.Router.Run(":" + strconv.Itoa(serverSettings.WebConfig.Application.HttpPort))

	// go ginServer.Router.GET("/", func(c *gin.Context) {
	// 	c.Redirect(http.StatusMovedPermanently, "https://"+serverSettings.WebConfig.Application.Domain+":"+strconv.Itoa(serverSettings.WebConfig.Application.HttpsPort))
	// })

}

func webSocketHandler(w http.ResponseWriter, r *http.Request) {

	log.Println("Web Socket Connection")
	conn, err := upgrader.Upgrade(w, r, nil)

	if err != nil {
		log.Println(err)
		return
	}

	//Start the Reader, listen for Close Message, and Add to the Connection Array.

	wsConn := new(WebSocketConnection)
	wsConn.Connection = conn
	uuid, err := newUUID()
	if err == nil {
		wsConn.Id = uuid
	} else {
		uuid = randomString(20)
		wsConn.Id = uuid
	}

	//Reader
	go func() {
		for {
			messageType, p, err := conn.ReadMessage()
			if err == nil {
				go func() {
					WebSocketCallbacks.RLock()
					log.Println(string(p[:]))
					for _, callback := range WebSocketCallbacks.callbacks {
						callback(wsConn, messageType, p)
					}
					WebSocketCallbacks.RUnlock()
				}()
			} else {
				return
			}
		}
	}()

	//Close Message
	closeHandler := func(code int, text string) error {
		// log.Println("Closing Socket")
		// log.Printf("%+v\n", len(WebSocketConnections.Connections))
		deleteWebSocket(wsConn)
		// log.Printf("%+v\n", len(WebSocketConnections.Connections))
		return nil
	}

	conn.SetCloseHandler(closeHandler)

	WebSocketConnections.Lock()
	WebSocketConnections.Connections = append(WebSocketConnections.Connections, wsConn)
	WebSocketConnections.Unlock()
}

func loadHTMLTemplates() {

	if serverSettings.WebConfig.Application.HtmlTemplates.Enabled {

		levels := "/*"
		dirLevel := ""

		switch serverSettings.WebConfig.Application.HtmlTemplates.DirectoryLevels {
		case 0:
			levels = "/*"
			dirLevel = ""
		case 1:
			levels = "/**/*"
			dirLevel = "root/"
		case 2:
			levels = "/**/**/*"
			dirLevel = "root/root/"
		}

		ginServer.Router.LoadHTMLGlob(serverSettings.APP_LOCATION + "/web/" + serverSettings.WebConfig.Application.HtmlTemplates.Directory + levels)

		ginServer.Router.GET("", func(c *gin.Context) {
			c.HTML(http.StatusOK, dirLevel+"index.tmpl", gin.H{})
		})
	} else {

		if serverSettings.WebConfig.Application.DisableRootIndex {
			return
		}

		ginServer.Router.GET("", func(c *gin.Context) {
			if serverSettings.WebConfig.Application.RootIndexPath == "" {
				ginServer.ReadHTMLFile(serverSettings.APP_LOCATION+"/web/index.htm", c)
			} else {
				ginServer.ReadHTMLFile(serverSettings.APP_LOCATION+"/web/"+serverSettings.WebConfig.Application.RootIndexPath, c)
			}
		})
	}
}

func initializeStaticRoutes() {

	ginServer.Router.GET("/swagger", func(c *gin.Context) {
		// c.Redirect(http.StatusMovedPermanently, "https://"+serverSettings.WebConfig.Application.Domain+":"+strconv.Itoa(serverSettings.WebConfig.Application.HttpsPort)+"/web/swagger/dist/index.html")

		ginServer.ReadHTMLFile(serverSettings.APP_LOCATION+"/web/swagger/dist/index.html", c)
	})
}

func RegisterWebSocketDataCallback(callback WebSocketCallback) {
	WebSocketCallbacks.Lock()
	WebSocketCallbacks.callbacks = append(WebSocketCallbacks.callbacks, callback)
	WebSocketCallbacks.Unlock()
}

func ReplyToWebSocket(conn *WebSocketConnection, data []byte) {

	WebSocketConnections.RLock()
	for _, wsConn := range WebSocketConnections.Connections {
		ws := wsConn
		if ws.Id == conn.Id {
			go func() {
				ws.Lock()
				ws.Connection.WriteMessage(websocket.BinaryMessage, data)
				ws.Unlock()
			}()
			return
		}
	}
	WebSocketConnections.RUnlock()
}

func ReplyToWebSocketJSON(conn *WebSocketConnection, v interface{}) {

	WebSocketConnections.RLock()
	for _, wsConn := range WebSocketConnections.Connections {
		ws := wsConn
		if ws.Id == conn.Id {
			go func() {
				ws.Lock()
				ws.Connection.WriteJSON(v)
				ws.Unlock()
			}()
			return
		}
	}
	WebSocketConnections.RUnlock()
}

func BroadcastWebSocketData(data []byte) {

	WebSocketConnections.RLock()
	for _, wsConn := range WebSocketConnections.Connections {
		ws := wsConn
		go func() {
			ws.Lock()
			ws.Connection.WriteMessage(websocket.BinaryMessage, data)
			ws.Unlock()
		}()
	}
	WebSocketConnections.RUnlock()
}

func BroadcastWebSocketJSON(v interface{}) {
	WebSocketConnections.RLock()
	for _, wsConn := range WebSocketConnections.Connections {
		ws := wsConn
		go func() {
			ws.Lock()
			ws.Connection.WriteJSON(v)
			ws.Unlock()
		}()
	}
	WebSocketConnections.RUnlock()
}

func deleteWebSocket(c *WebSocketConnection) {
	WebSocketConnections.Lock()
	for i, wsConn := range WebSocketConnections.Connections {
		if wsConn.Id == c.Id {
			log.Println("Removing Socket")
			WebSocketConnections.Connections = removeWebSocket(WebSocketConnections.Connections, i)
		}
	}

	WebSocketConnections.Unlock()
}

func removeWebSocket(s []*WebSocketConnection, i int) []*WebSocketConnection {
	s[len(s)-1], s[i] = s[i], s[len(s)-1]
	return s[:len(s)-1]
}

// newUUID generates a random UUID according to RFC 4122
func newUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(rand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}

func randomString(strlen int) string {
	randMath.Seed(time.Now().UTC().UnixNano())
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	result := make([]byte, strlen)
	for i := 0; i < strlen; i++ {
		result[i] = chars[randMath.Intn(len(chars))]
	}
	return string(result)
}
<commit_msg>Web socket updates.<commit_after>package app

import (
	"crypto/rand"
	"fmt"
	"io"
	"log"
	randMath "math/rand"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/DanielRenne/GoCore/core/dbServices"
	"github.com/DanielRenne/GoCore/core/fileCache"
	"github.com/DanielRenne/GoCore/core/ginServer"
	"github.com/DanielRenne/GoCore/core/serverSettings"
	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
)

type WebSocketConnection struct {
	sync.Mutex
	Id         string
	Connection *websocket.Conn
}

type WebSocketConnectionCollection struct {
	sync.RWMutex
	Connections []*WebSocketConnection
}

type WebSocketCallbackSync struct {
	sync.RWMutex
	callbacks []WebSocketCallback
}

type WebSocketCallback func(conn *WebSocketConnection, c *gin.Context, messageType int, data []byte)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

var WebSocketConnections WebSocketConnectionCollection
var WebSocketCallbacks WebSocketCallbackSync

func Initialize(path string) {
	serverSettings.Initialize(path)
	dbServices.Initialize()

	if serverSettings.WebConfig.Application.ReleaseMode == "release" {
		ginServer.Initialize(gin.ReleaseMode)
	} else {
		ginServer.Initialize(gin.DebugMode)
	}

	fileCache.Initialize()

}

func Run() {

	if serverSettings.WebConfig.Application.WebServiceOnly == false {

		loadHTMLTemplates()

		ginServer.Router.Static("/web", serverSettings.APP_LOCATION+"/web")

		ginServer.Router.GET("/ws", func(c *gin.Context) {
			webSocketHandler(c.Writer, c.Request, c)
		})
	}

	initializeStaticRoutes()

	go ginServer.Router.RunTLS(":"+strconv.Itoa(serverSettings.WebConfig.Application.HttpsPort), serverSettings.APP_LOCATION+"/keys/cert.pem", serverSettings.APP_LOCATION+"/keys/key.pem")

	log.Println("GoCore Application Started")

	ginServer.Router.Run(":" + strconv.Itoa(serverSettings.WebConfig.Application.HttpPort))

	// go ginServer.Router.GET("/", func(c *gin.Context) {
	// 	c.Redirect(http.StatusMovedPermanently, "https://"+serverSettings.WebConfig.Application.Domain+":"+strconv.Itoa(serverSettings.WebConfig.Application.HttpsPort))
	// })

}

func webSocketHandler(w http.ResponseWriter, r *http.Request, c *gin.Context) {

	log.Println("Web Socket Connection")
	conn, err := upgrader.Upgrade(w, r, nil)

	if err != nil {
		log.Println(err)
		return
	}

	//Start the Reader, listen for Close Message, and Add to the Connection Array.

	wsConn := new(WebSocketConnection)
	wsConn.Connection = conn
	uuid, err := newUUID()
	if err == nil {
		wsConn.Id = uuid
	} else {
		uuid = randomString(20)
		wsConn.Id = uuid
	}

	//Reader
	go func() {
		for {
			messageType, p, err := conn.ReadMessage()
			if err == nil {
				go func() {
					WebSocketCallbacks.RLock()
					log.Println(string(p[:]))
					for _, callback := range WebSocketCallbacks.callbacks {
						callback(wsConn, c, messageType, p)
					}
					WebSocketCallbacks.RUnlock()
				}()
			} else {
				return
			}
		}
	}()

	//Close Message
	closeHandler := func(code int, text string) error {
		// log.Println("Closing Socket")
		// log.Printf("%+v\n", len(WebSocketConnections.Connections))
		deleteWebSocket(wsConn)
		// log.Printf("%+v\n", len(WebSocketConnections.Connections))
		return nil
	}

	conn.SetCloseHandler(closeHandler)

	WebSocketConnections.Lock()
	WebSocketConnections.Connections = append(WebSocketConnections.Connections, wsConn)
	WebSocketConnections.Unlock()
}

func loadHTMLTemplates() {

	if serverSettings.WebConfig.Application.HtmlTemplates.Enabled {

		levels := "/*"
		dirLevel := ""

		switch serverSettings.WebConfig.Application.HtmlTemplates.DirectoryLevels {
		case 0:
			levels = "/*"
			dirLevel = ""
		case 1:
			levels = "/**/*"
			dirLevel = "root/"
		case 2:
			levels = "/**/**/*"
			dirLevel = "root/root/"
		}

		ginServer.Router.LoadHTMLGlob(serverSettings.APP_LOCATION + "/web/" + serverSettings.WebConfig.Application.HtmlTemplates.Directory + levels)

		ginServer.Router.GET("", func(c *gin.Context) {
			c.HTML(http.StatusOK, dirLevel+"index.tmpl", gin.H{})
		})
	} else {

		if serverSettings.WebConfig.Application.DisableRootIndex {
			return
		}

		ginServer.Router.GET("", func(c *gin.Context) {
			if serverSettings.WebConfig.Application.RootIndexPath == "" {
				ginServer.ReadHTMLFile(serverSettings.APP_LOCATION+"/web/index.htm", c)
			} else {
				ginServer.ReadHTMLFile(serverSettings.APP_LOCATION+"/web/"+serverSettings.WebConfig.Application.RootIndexPath, c)
			}
		})
	}
}

func initializeStaticRoutes() {

	ginServer.Router.GET("/swagger", func(c *gin.Context) {
		// c.Redirect(http.StatusMovedPermanently, "https://"+serverSettings.WebConfig.Application.Domain+":"+strconv.Itoa(serverSettings.WebConfig.Application.HttpsPort)+"/web/swagger/dist/index.html")

		ginServer.ReadHTMLFile(serverSettings.APP_LOCATION+"/web/swagger/dist/index.html", c)
	})
}

func RegisterWebSocketDataCallback(callback WebSocketCallback) {
	WebSocketCallbacks.Lock()
	WebSocketCallbacks.callbacks = append(WebSocketCallbacks.callbacks, callback)
	WebSocketCallbacks.Unlock()
}

func ReplyToWebSocket(conn *WebSocketConnection, data []byte) {

	WebSocketConnections.RLock()
	for _, wsConn := range WebSocketConnections.Connections {
		ws := wsConn
		if ws.Id == conn.Id {
			go func() {
				ws.Lock()
				ws.Connection.WriteMessage(websocket.BinaryMessage, data)
				ws.Unlock()
			}()
			return
		}
	}
	WebSocketConnections.RUnlock()
}

func ReplyToWebSocketJSON(conn *WebSocketConnection, v interface{}) {

	WebSocketConnections.RLock()
	for _, wsConn := range WebSocketConnections.Connections {
		ws := wsConn
		if ws.Id == conn.Id {
			go func() {
				ws.Lock()
				ws.Connection.WriteJSON(v)
				ws.Unlock()
			}()
			return
		}
	}
	WebSocketConnections.RUnlock()
}

func BroadcastWebSocketData(data []byte) {

	WebSocketConnections.RLock()
	for _, wsConn := range WebSocketConnections.Connections {
		ws := wsConn
		go func() {
			ws.Lock()
			ws.Connection.WriteMessage(websocket.BinaryMessage, data)
			ws.Unlock()
		}()
	}
	WebSocketConnections.RUnlock()
}

func BroadcastWebSocketJSON(v interface{}) {
	WebSocketConnections.RLock()
	for _, wsConn := range WebSocketConnections.Connections {
		ws := wsConn
		go func() {
			ws.Lock()
			ws.Connection.WriteJSON(v)
			ws.Unlock()
		}()
	}
	WebSocketConnections.RUnlock()
}

func deleteWebSocket(c *WebSocketConnection) {
	WebSocketConnections.Lock()
	for i, wsConn := range WebSocketConnections.Connections {
		if wsConn.Id == c.Id {
			log.Println("Removing Socket")
			WebSocketConnections.Connections = removeWebSocket(WebSocketConnections.Connections, i)
		}
	}

	WebSocketConnections.Unlock()
}

func removeWebSocket(s []*WebSocketConnection, i int) []*WebSocketConnection {
	s[len(s)-1], s[i] = s[i], s[len(s)-1]
	return s[:len(s)-1]
}

// newUUID generates a random UUID according to RFC 4122
func newUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(rand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}

func randomString(strlen int) string {
	randMath.Seed(time.Now().UTC().UnixNano())
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	result := make([]byte, strlen)
	for i := 0; i < strlen; i++ {
		result[i] = chars[randMath.Intn(len(chars))]
	}
	return string(result)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"reflect"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/google/trillian"
	"github.com/google/trillian-examples/registers/trillian_client"
	"google.golang.org/grpc"
)

var (
	trillianLog = flag.String("trillian_log", "localhost:8090", "address of the Trillian Log RPC server.")
	logID       = flag.Int64("log_id", 0, "Trillian LogID to read.")
	trillianMap = flag.String("trillian_map", "localhost:8095", "address of the Trillian Map RPC server.")
	mapID       = flag.Int64("map_id", 0, "Trillian MapID to write.")
)

type record struct {
	Entry map[string]interface{}
	Items []map[string]interface{}
}

// add only adds if the item is not already present in Items.
func (r *record) add(i map[string]interface{}) {
	for _, ii := range r.Items {
		if reflect.DeepEqual(i, ii) {
			return
		}
	}
	r.Items = append(r.Items, i)
}

type recordCache struct {
	records *lru.Cache
	mapID   int64
	tc      trillian.TrillianMapClient
	ctx     context.Context
}

func newCache(tc trillian.TrillianMapClient, mapID int64, max int, ctx context.Context) *recordCache {
	c := &recordCache{records: lru.New(max), mapID: mapID, tc: tc, ctx: ctx}
	c.records.OnEvicted = func(key lru.Key, value interface{}) { recordEvicted(c, key, value) }
	return c
}

func recordEvicted(c *recordCache, key lru.Key, value interface{}) {
	fmt.Printf("evicting %v -> %v\n", key, value)

	v, err := json.Marshal(value)
	if err != nil {
		log.Fatalf("Marshal() failed: %v", err)
	}

	hash := sha256.Sum256([]byte(key.(string)))
	l := trillian.MapLeaf{
		Index:     hash[:],
		LeafValue: v,
	}

	req := trillian.SetMapLeavesRequest{
		MapId:  c.mapID,
		Leaves: []*trillian.MapLeaf{&l},
	}

	_, err = c.tc.SetLeaves(c.ctx, &req)
	if err != nil {
		log.Fatalf("SetLeaves() failed: %v", err)
	}
}

func (c *recordCache) getLeaf(key string) (*record, error) {
	hash := sha256.Sum256([]byte(key))
	index := [1][]byte{hash[:]}
	req := &trillian.GetMapLeavesRequest{
		MapId: c.mapID,
		Index: index[:],
	}

	resp, err := c.tc.GetLeaves(c.ctx, req)
	if err != nil {
		return nil, err
	}

	l := resp.MapLeafInclusion[0].Leaf.LeafValue
	log.Printf("key=%v leaf=%s", key, l)
	// FIXME: we should be able to detect non-existent vs. empty leaves
	if len(l) == 0 {
		return nil, nil
	}

	var r record
	err = json.Unmarshal(l, &r)
	if err != nil {
		return nil, err
	}

	return &r, nil
}

// Get the current record for the given key, possibly going to Trillian to look it up, possibly flushing the cache if needed.
func (c *recordCache) get(key string) (*record, error) {
	r, ok := c.records.Get(key)
	if !ok {
		rr, err := c.getLeaf(key)
		if err != nil {
			return nil, err
		}
		c.cacheExisting(key, rr)
		r = rr
	}
	if r == nil {
		return nil, nil
	}
	return r.(*record), nil
}

// create creates a new entry or replaces the existing one.
func (c *recordCache) create(key string, entry map[string]interface{}, item map[string]interface{}) {
	c.records.Add(key, &record{Entry: entry, Items: []map[string]interface{}{item}})
}

func (c *recordCache) cacheExisting(key string, r *record) {
	c.records.Add(key, r)
}

func (c *recordCache) flush() {
	c.records.Clear()
}

type logScanner struct {
	cache *recordCache
}

func (s *logScanner) Leaf(leaf *trillian.LogLeaf) error {
	//log.Printf("leaf %d: %s", leaf.LeafIndex, leaf.LeafValue)
	var l map[string]interface{}
	err := json.Unmarshal(leaf.LeafValue, &l)
	if err != nil {
		return err
	}
	//log.Printf("%v", l)

	e := l["Entry"].(map[string]interface{})
	t, err := time.Parse(time.RFC3339, e["entry-timestamp"].(string))
	if err != nil {
		return err
	}

	k := e["key"].(string)
	i := l["Item"].(map[string]interface{})
	log.Printf("k: %s ts: %s", k, t)

	cr, err := s.cache.get(k)
	if err != nil {
		return err
	}
	if cr == nil {
		s.cache.create(k, e, i)
		return nil
	}

	ct, err := time.Parse(time.RFC3339, cr.Entry["entry-timestamp"].(string))
	if err != nil {
		return err
	}

	if t.Before(ct) {
		log.Printf("Skip")
		return nil
	} else if t.After(ct) {
		log.Printf("Replace")
		s.cache.create(k, e, i)
		return nil
	}

	log.Printf("Add")
	cr.add(i)

	return nil
}

func main() {
	flag.Parse()

	tc := trillian_client.New(*trillianLog)
	defer tc.Close()

	g, err := grpc.Dial(*trillianMap, grpc.WithInsecure())
	if err != nil {
		log.Fatalf("Failed to dial Trillian Log: %v", err)
	}
	tmc := trillian.NewTrillianMapClient(g)

	rc := newCache(tmc, *mapID, 3, context.Background())
	defer rc.flush()
	err = tc.Scan(*logID, &logScanner{cache: rc})
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Eliminate cache (partially tested).<commit_after>package main

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"reflect"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/google/trillian"
	"github.com/google/trillian-examples/registers/trillian_client"
	"google.golang.org/grpc"
)

var (
	trillianLog = flag.String("trillian_log", "localhost:8090", "address of the Trillian Log RPC server.")
	logID       = flag.Int64("log_id", 0, "Trillian LogID to read.")
	trillianMap = flag.String("trillian_map", "localhost:8095", "address of the Trillian Map RPC server.")
	mapID       = flag.Int64("map_id", 0, "Trillian MapID to write.")
)

type record struct {
	Entry map[string]interface{}
	Items []map[string]interface{}
}

// add only adds if the item is not already present in Items.
func (r *record) add(i map[string]interface{}) {
	for _, ii := range r.Items {
		if reflect.DeepEqual(i, ii) {
			return
		}
	}
	r.Items = append(r.Items, i)
}

type mapInfo struct {
	mapID int64
	tc    trillian.TrillianMapClient
	ctx   context.Context
}

func newInfo(tc trillian.TrillianMapClient, mapID int64, ctx context.Context) *mapInfo {
	i := &mapInfo{mapID: mapID, tc: tc, ctx: ctx}
	return i
}

func (i *mapInfo) createRecord(key lru.Key, entry map[string]interface{}, item map[string]interface{}) {
	ii := [1]map[string]interface{}{item}
	i.saveRecord(key, &record{Entry: entry, Items: ii[:]})
}

func (i *mapInfo) saveRecord(key lru.Key, value interface{}) {
	fmt.Printf("evicting %v -> %v\n", key, value)

	v, err := json.Marshal(value)
	if err != nil {
		log.Fatalf("Marshal() failed: %v", err)
	}

	hash := sha256.Sum256([]byte(key.(string)))
	l := trillian.MapLeaf{
		Index:     hash[:],
		LeafValue: v,
	}

	req := trillian.SetMapLeavesRequest{
		MapId:  i.mapID,
		Leaves: []*trillian.MapLeaf{&l},
	}

	_, err = i.tc.SetLeaves(i.ctx, &req)
	if err != nil {
		log.Fatalf("SetLeaves() failed: %v", err)
	}
}

func (i *mapInfo) getLeaf(key string) (*record, error) {
	hash := sha256.Sum256([]byte(key))
	index := [1][]byte{hash[:]}
	req := &trillian.GetMapLeavesRequest{
		MapId: i.mapID,
		Index: index[:],
	}

	resp, err := i.tc.GetLeaves(i.ctx, req)
	if err != nil {
		return nil, err
	}

	l := resp.MapLeafInclusion[0].Leaf.LeafValue
	log.Printf("key=%v leaf=%s", key, l)
	// FIXME: we should be able to detect non-existent vs. empty leaves
	if len(l) == 0 {
		return nil, nil
	}

	var r record
	err = json.Unmarshal(l, &r)
	if err != nil {
		return nil, err
	}

	return &r, nil
}

// Get the current record for the given key, possibly going to Trillian to look it up, possibly flushing the cache if needed.
func (i *mapInfo) get(key string) (*record, error) {
	r, err := i.getLeaf(key)
	if err != nil {
		return nil, err
	}
	if r == nil {
		return nil, nil
	}
	return r, nil
}

/*
// create creates a new entry or replaces the existing one.
func (c *recordCache) create(key string, entry map[string]interface{}, item map[string]interface{}) {
	c.records.Add(key, &record{Entry: entry, Items: []map[string]interface{}{item}})
}

func (c *recordCache) cacheExisting(key string, r *record) {
	c.records.Add(key, r)
}

func (c *recordCache) flush() {
	c.records.Clear()
}
*/

type logScanner struct {
	info *mapInfo
}

func (s *logScanner) Leaf(leaf *trillian.LogLeaf) error {
	//log.Printf("leaf %d: %s", leaf.LeafIndex, leaf.LeafValue)
	var l map[string]interface{}
	err := json.Unmarshal(leaf.LeafValue, &l)
	if err != nil {
		return err
	}
	//log.Printf("%v", l)

	e := l["Entry"].(map[string]interface{})
	t, err := time.Parse(time.RFC3339, e["entry-timestamp"].(string))
	if err != nil {
		return err
	}

	k := e["key"].(string)
	i := l["Item"].(map[string]interface{})
	log.Printf("k: %s ts: %s", k, t)

	cr, err := s.info.get(k)
	if err != nil {
		return err
	}
	if cr == nil {
		s.info.createRecord(k, e, i)
		return nil
	}

	ct, err := time.Parse(time.RFC3339, cr.Entry["entry-timestamp"].(string))
	if err != nil {
		return err
	}

	if t.Before(ct) {
		log.Printf("Skip")
		return nil
	} else if t.After(ct) {
		log.Printf("Replace")
		s.info.createRecord(k, e, i)
		return nil
	}

	log.Printf("Add")
	cr.add(i)
	s.info.saveRecord(k, cr)

	return nil
}

func main() {
	flag.Parse()

	tc := trillian_client.New(*trillianLog)
	defer tc.Close()

	g, err := grpc.Dial(*trillianMap, grpc.WithInsecure())
	if err != nil {
		log.Fatalf("Failed to dial Trillian Log: %v", err)
	}
	tmc := trillian.NewTrillianMapClient(g)

	i := newInfo(tmc, *mapID, context.Background())
	err = tc.Scan(*logID, &logScanner{info: i})
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package v2

import (
	"fmt"
	"regexp"
	"strings"
)

// TODO(stevvooe): Move these definitions to the future "reference" package.
// While they are used with v2 definitions, their relevance expands beyond.

const (
	// RepositoryNameTotalLengthMax is the maximum total number of characters in
	// a repository name
	RepositoryNameTotalLengthMax = 255
)

// RepositoryNameComponentRegexp restricts registry path component names to
// start with at least one letter or number, with following parts able to
// be separated by one period, dash or underscore.
var RepositoryNameComponentRegexp = regexp.MustCompile(`[a-z0-9]+(?:[._-][a-z0-9]+)*`)

// RepositoryNameComponentAnchoredRegexp is the version of
// RepositoryNameComponentRegexp which must completely match the content
var RepositoryNameComponentAnchoredRegexp = regexp.MustCompile(`^` + RepositoryNameComponentRegexp.String() + `$`)

// RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow
// multiple path components, separated by a forward slash.
var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/)*` + RepositoryNameComponentRegexp.String())

// TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go.
var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`)

// TagNameAnchoredRegexp matches valid tag names, anchored at the start and
// end of the matched string.
var TagNameAnchoredRegexp = regexp.MustCompile("^" + TagNameRegexp.String() + "$")

var (
	// ErrRepositoryNameEmpty is returned for empty, invalid repository names.
	ErrRepositoryNameEmpty = fmt.Errorf("repository name must have at least one component")

	// ErrRepositoryNameLong is returned when a repository name is longer than
	// RepositoryNameTotalLengthMax
	ErrRepositoryNameLong = fmt.Errorf("repository name must not be more than %v characters", RepositoryNameTotalLengthMax)

	// ErrRepositoryNameComponentInvalid is returned when a repository name does
	// not match RepositoryNameComponentRegexp
	ErrRepositoryNameComponentInvalid = fmt.Errorf("repository name component must match %q", RepositoryNameComponentRegexp.String())
)

// ValidateRepositoryName ensures the repository name is valid for use in the
// registry. This function accepts a superset of what might be accepted by
// docker core or docker hub. If the name does not pass validation, an error,
// describing the conditions, is returned.
//
// Effectively, the name should comply with the following grammar:
//
// 	alpha-numeric := /[a-z0-9]+/
//	separator := /[._-]/
//	component := alpha-numeric [separator alpha-numeric]*
//	namespace := component ['/' component]*
//
// The result of the production, known as the "namespace", should be limited
// to 255 characters.
func ValidateRepositoryName(name string) error {
	if name == "" {
		return ErrRepositoryNameEmpty
	}

	if len(name) > RepositoryNameTotalLengthMax {
		return ErrRepositoryNameLong
	}

	components := strings.Split(name, "/")

	for _, component := range components {
		if !RepositoryNameComponentAnchoredRegexp.MatchString(component) {
			return ErrRepositoryNameComponentInvalid
		}
	}

	return nil
}
<commit_msg>issue 1008 correct repository name restriction<commit_after>package v2

import (
	"fmt"
	"regexp"
	"strings"
)

// TODO(stevvooe): Move these definitions to the future "reference" package.
// While they are used with v2 definitions, their relevance expands beyond.

const (
	// RepositoryNameTotalLengthMax is the maximum total number of characters in
	// a repository name
	RepositoryNameTotalLengthMax = 255
)

// RepositoryNameComponentRegexp restricts registry path component names to
// start with at least one letter or number, with following parts able to
// be separated by one period, dash or underscore, or separated by combination
// of those such as _-_ or __.
var RepositoryNameComponentRegexp = regexp.MustCompile(`[a-z0-9]+(?:[._-]+[a-z0-9]+)*`)

// RepositoryNameComponentAnchoredRegexp is the version of
// RepositoryNameComponentRegexp which must completely match the content
var RepositoryNameComponentAnchoredRegexp = regexp.MustCompile(`^` + RepositoryNameComponentRegexp.String() + `$`)

// RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow
// multiple path components, separated by a forward slash.
var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/)*` + RepositoryNameComponentRegexp.String())

// TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go.
var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`)

// TagNameAnchoredRegexp matches valid tag names, anchored at the start and
// end of the matched string.
var TagNameAnchoredRegexp = regexp.MustCompile("^" + TagNameRegexp.String() + "$")

var (
	// ErrRepositoryNameEmpty is returned for empty, invalid repository names.
	ErrRepositoryNameEmpty = fmt.Errorf("repository name must have at least one component")

	// ErrRepositoryNameLong is returned when a repository name is longer than
	// RepositoryNameTotalLengthMax
	ErrRepositoryNameLong = fmt.Errorf("repository name must not be more than %v characters", RepositoryNameTotalLengthMax)

	// ErrRepositoryNameComponentInvalid is returned when a repository name does
	// not match RepositoryNameComponentRegexp
	ErrRepositoryNameComponentInvalid = fmt.Errorf("repository name component must match %q", RepositoryNameComponentRegexp.String())
)

// ValidateRepositoryName ensures the repository name is valid for use in the
// registry. This function accepts a superset of what might be accepted by
// docker core or docker hub. If the name does not pass validation, an error,
// describing the conditions, is returned.
//
// Effectively, the name should comply with the following grammar:
//
// 	alpha-numeric := /[a-z0-9]+/
//	separator := /[._-]/
//	component := alpha-numeric [separator alpha-numeric]*
//	namespace := component ['/' component]*
//
// The result of the production, known as the "namespace", should be limited
// to 255 characters.
func ValidateRepositoryName(name string) error {
	if name == "" {
		return ErrRepositoryNameEmpty
	}

	if len(name) > RepositoryNameTotalLengthMax {
		return ErrRepositoryNameLong
	}

	components := strings.Split(name, "/")

	for _, component := range components {
		if !RepositoryNameComponentAnchoredRegexp.MatchString(component) {
			return ErrRepositoryNameComponentInvalid
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/ChimeraCoder/anaconda"
	"github.com/gin-gonic/gin"
	"os"
)

func main() {
	consumerKey := os.Getenv("CONSUMER_KEY")
	consumerSecret := os.Getenv("CONSUMER_SECRET")

	anaconda.SetConsumerKey(consumerKey)
	anaconda.SetConsumerSecret(consumerSecret)

	chans := map[string]chan bool{}

	r := gin.Default()
	r.GET("/start", func(c *gin.Context) {
		id := "abc"
		if chans[id] {
			c.JSON(200, gin.H{
				"message": "exists streaming",
			})
		} else {
			chans[id] = make(chan bool)
			go connect(chans[id])
			c.JSON(200, gin.H{
				"message": "start streaming",
			})
		}
	})
	r.GET("/stop", func(c *gin.Context) {
		id := "abc"
		if fin, ok := chans[id]; ok {
			fin <- true
			delete(chans, id)
			c.JSON(200, gin.H{
				"message": "stop streaming",
			})
		} else {
			c.JSON(200, gin.H{
				"message": "missing streaming",
			})
		}
	})
	r.Run()
}

func connect(fin <-chan bool) {
	accessToken := os.Getenv("ACCESS_TOKEN")
	accessTokenSecret := os.Getenv("ACCESS_TOKEN_SECRET")
	api := anaconda.NewTwitterApi(accessToken, accessTokenSecret)
	twitterStream := api.UserStream(nil)
	fmt.Println("connect")
	for {
		select {
		case x := <-twitterStream.C:
			switch data := x.(type) {
			case anaconda.FriendsList:
				// pass
			case anaconda.Tweet:
				fmt.Println(data.Text)
				fmt.Println("-----------")
			case anaconda.StatusDeletionNotice:
				// pass
			case anaconda.EventTweet:
				fmt.Println(data.Event.Event)
				fmt.Println(data.TargetObject.Text)
				fmt.Println("-----------")
				// pass
			default:
				fmt.Printf("unknown type(%T) : %v \n", x, x)
			}
		case <-fin:
			twitterStream.Stop()
			fmt.Println("fin")
			return
		}
	}
}
<commit_msg>Mutex<commit_after>package main

import (
	"fmt"
	"github.com/ChimeraCoder/anaconda"
	"github.com/gin-gonic/gin"
	"os"
	"sync"
)

func main() {
	consumerKey := os.Getenv("CONSUMER_KEY")
	consumerSecret := os.Getenv("CONSUMER_SECRET")

	anaconda.SetConsumerKey(consumerKey)
	anaconda.SetConsumerSecret(consumerSecret)

	m := new(sync.Mutex)
	chans := map[string]chan bool{}

	r := gin.Default()
	r.GET("/start", func(c *gin.Context) {
		m.Lock()
		defer m.Unlock()
		id := "abc"
		_, ok := chans[id]
		if ok {
			c.JSON(200, gin.H{
				"message": "exists streaming",
			})
		} else {
			chans[id] = make(chan bool)
			go connect(chans[id])
			c.JSON(200, gin.H{
				"message": "start streaming",
			})
		}
	})
	r.GET("/stop", func(c *gin.Context) {
		m.Lock()
		defer m.Unlock()
		id := "abc"
		if fin, ok := chans[id]; ok {
			fin <- true
			close(fin)
			delete(chans, id)
			c.JSON(200, gin.H{
				"message": "stop streaming",
			})
		} else {
			c.JSON(200, gin.H{
				"message": "missing streaming",
			})
		}
	})
	r.Run()
}

func connect(fin <-chan bool) {
	accessToken := os.Getenv("ACCESS_TOKEN")
	accessTokenSecret := os.Getenv("ACCESS_TOKEN_SECRET")
	api := anaconda.NewTwitterApi(accessToken, accessTokenSecret)
	twitterStream := api.UserStream(nil)
	fmt.Println("connect")
	for {
		select {
		case x := <-twitterStream.C:
			switch data := x.(type) {
			case anaconda.FriendsList:
				// pass
			case anaconda.Tweet:
				fmt.Println(data.Text)
				fmt.Println("-----------")
			case anaconda.StatusDeletionNotice:
				// pass
			case anaconda.EventTweet:
				fmt.Println(data.Event.Event)
				fmt.Println(data.TargetObject.Text)
				fmt.Println("-----------")
				// pass
			default:
				fmt.Printf("unknown type(%T) : %v \n", x, x)
			}
		case <-fin:
			twitterStream.Stop()
			fmt.Println("fin")
			return
		}
	}
}
<|endoftext|>