<commit_before>// Copyright 2020 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten_test

import (
	"image/color"
	"testing"

	. "github.com/hajimehoshi/ebiten"
)

func TestShaderFill(t *testing.T) {
	const w, h = 16, 16

	dst, _ := NewImage(w, h, FilterDefault)
	s, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(1, 0, 0, 1)
}
`))
	if err != nil {
		t.Fatal(err)
	}

	dst.DrawRectShader(w/2, h/2, s, nil)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			got := dst.At(i, j).(color.RGBA)
			var want color.RGBA
			if i < w/2 && j < h/2 {
				want = color.RGBA{0xff, 0, 0, 0xff}
			}
			if got != want {
				t.Errorf("dst.At(%d, %d): got: %v, want: %v", i, j, got, want)
			}
		}
	}
}

func TestShaderFillWithDrawImage(t *testing.T) {
	const w, h = 16, 16

	dst, _ := NewImage(w, h, FilterDefault)
	s, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(1, 0, 0, 1)
}
`))
	if err != nil {
		t.Fatal(err)
	}

	src, _ := NewImage(w/2, h/2, FilterDefault)
	op := &DrawImageOptions{}
	op.Shader = s
	dst.DrawImage(src, op)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			got := dst.At(i, j).(color.RGBA)
			var want color.RGBA
			if i < w/2 && j < h/2 {
				want = color.RGBA{0xff, 0, 0, 0xff}
			}
			if got != want {
				t.Errorf("dst.At(%d, %d): got: %v, want: %v", i, j, got, want)
			}
		}
	}
}

func TestShaderFunction(t *testing.T) {
	const w, h = 16, 16

	dst, _ := NewImage(w, h, FilterDefault)
	s, err := NewShader([]byte(`package main

func clr(red float) (float, float, float, float) {
	return red, 0, 0, 1
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(clr(1))
}
`))
	if err != nil {
		t.Fatal(err)
	}

	dst.DrawRectShader(w, h, s, nil)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			got := dst.At(i, j).(color.RGBA)
			want := color.RGBA{0xff, 0, 0, 0xff}
			if got != want {
				t.Errorf("dst.At(%d, %d): got: %v, want: %v", i, j, got, want)
			}
		}
	}
}

func TestShaderShadowing(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	var position vec4
	return position
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}

func TestShaderDuplicatedVariables(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	var foo vec4
	var foo vec4
	return foo
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	var foo, foo vec4
	return foo
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	var foo vec4
	foo := vec4(0)
	return foo
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (vec4, vec4) {
	return vec4(0), vec4(0)
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	foo, foo := Foo()
	return foo
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}

func TestShaderNoMain(t *testing.T) {
	if _, err := NewShader([]byte(`package main
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}

func TestShaderNoNewVariables(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	_ := 1
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	_, _ := 1, 1
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (int, int) {
	return 1, 1
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	_, _ := Foo()
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	a, _ := 1, 1
	_ = a
	return vec4(0)
}
`)); err != nil {
		t.Errorf("error must be nil but non-nil: %v", err)
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	_, a := 1, 1
	_ = a
	return vec4(0)
}
`)); err != nil {
		t.Errorf("error must be nil but non-nil: %v", err)
	}
}

func TestShaderWrongReturn(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return 0.0
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (float, float) {
	return 0
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() float {
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}

func TestShaderMultipleValueReturn(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Foo() (float, float) {
	return 0.0
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() float {
	return 0.0, 0.0
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (float, float, float) {
	return 0.0, 0.0
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (float, float) {
	return 0.0, 0.0
}

func Foo2() (float, float, float) {
	return Foo()
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (float, float, float) {
	return 0.0, 0.0, 0.0
}

func Foo2() (float, float, float) {
	return Foo()
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0.0)
}
`)); err != nil {
		t.Error(err)
	}
}

func TestShaderInit(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func init() {
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}
<commit_msg>ebiten: Add shader tests<commit_after>// Copyright 2020 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten_test

import (
	"image/color"
	"testing"

	. "github.com/hajimehoshi/ebiten"
)

func TestShaderFill(t *testing.T) {
	const w, h = 16, 16

	dst, _ := NewImage(w, h, FilterDefault)
	s, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(1, 0, 0, 1)
}
`))
	if err != nil {
		t.Fatal(err)
	}

	dst.DrawRectShader(w/2, h/2, s, nil)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			got := dst.At(i, j).(color.RGBA)
			var want color.RGBA
			if i < w/2 && j < h/2 {
				want = color.RGBA{0xff, 0, 0, 0xff}
			}
			if got != want {
				t.Errorf("dst.At(%d, %d): got: %v, want: %v", i, j, got, want)
			}
		}
	}
}

func TestShaderFillWithDrawImage(t *testing.T) {
	const w, h = 16, 16

	dst, _ := NewImage(w, h, FilterDefault)
	s, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(1, 0, 0, 1)
}
`))
	if err != nil {
		t.Fatal(err)
	}

	src, _ := NewImage(w/2, h/2, FilterDefault)
	op := &DrawImageOptions{}
	op.Shader = s
	dst.DrawImage(src, op)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			got := dst.At(i, j).(color.RGBA)
			var want color.RGBA
			if i < w/2 && j < h/2 {
				want = color.RGBA{0xff, 0, 0, 0xff}
			}
			if got != want {
				t.Errorf("dst.At(%d, %d): got: %v, want: %v", i, j, got, want)
			}
		}
	}
}

func TestShaderFunction(t *testing.T) {
	const w, h = 16, 16

	dst, _ := NewImage(w, h, FilterDefault)
	s, err := NewShader([]byte(`package main

func clr(red float) (float, float, float, float) {
	return red, 0, 0, 1
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(clr(1))
}
`))
	if err != nil {
		t.Fatal(err)
	}

	dst.DrawRectShader(w, h, s, nil)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			got := dst.At(i, j).(color.RGBA)
			want := color.RGBA{0xff, 0, 0, 0xff}
			if got != want {
				t.Errorf("dst.At(%d, %d): got: %v, want: %v", i, j, got, want)
			}
		}
	}
}

func TestShaderShadowing(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	var position vec4
	return position
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}

func TestShaderDuplicatedVariables(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	var foo vec4
	var foo vec4
	return foo
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	var foo, foo vec4
	return foo
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	var foo vec4
	foo := vec4(0)
	return foo
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (vec4, vec4) {
	return vec4(0), vec4(0)
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	foo, foo := Foo()
	return foo
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}

func TestShaderNoMain(t *testing.T) {
	if _, err := NewShader([]byte(`package main
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}

func TestShaderNoNewVariables(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	_ := 1
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	_, _ := 1, 1
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (int, int) {
	return 1, 1
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	_, _ := Foo()
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	a, _ := 1, 1
	_ = a
	return vec4(0)
}
`)); err != nil {
		t.Errorf("error must be nil but non-nil: %v", err)
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	_, a := 1, 1
	_ = a
	return vec4(0)
}
`)); err != nil {
		t.Errorf("error must be nil but non-nil: %v", err)
	}
}

func TestShaderWrongReturn(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return 0.0
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (float, float) {
	return 0
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() float {
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}

func TestShaderMultipleValueReturn(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Foo() (float, float) {
	return 0.0
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() float {
	return 0.0, 0.0
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (float, float, float) {
	return 0.0, 0.0
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (float, float) {
	return 0.0, 0.0
}

func Foo2() (float, float, float) {
	return Foo()
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Foo() (float, float, float) {
	return 0.0, 0.0, 0.0
}

func Foo2() (float, float, float) {
	return Foo()
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0.0)
}
`)); err != nil {
		t.Error(err)
	}
}

func TestShaderInit(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func init() {
}

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}

func TestShaderUnspportedSyntax(t *testing.T) {
	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	x := func() {
	}
	_ = x
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	go func() {
	}()
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	ch := make(chan int)
	_ = ch
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}

	if _, err := NewShader([]byte(`package main

func Fragment(position vec4, texCoord vec2, color vec4) vec4 {
	x := 1i
	_ = x
	return vec4(0)
}
`)); err == nil {
		t.Errorf("error must be non-nil but was nil")
	}
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"

	"io"

	proto "github.com/ncodes/cocoon/core/stubs/golang/proto"
	logging "github.com/op/go-logging"
	context "golang.org/x/net/context"
	"google.golang.org/grpc"
)

var log = logging.MustGetLogger("connector.client")

// Client represents a cocoon code GRPC client
// that interacts with a cocoon code.
type Client struct {
	ccodePort int
	stub      proto.StubClient
	conCtx    context.Context
	conCancel context.CancelFunc
}

// NewClient creates a new cocoon code client
func NewClient(ccodePort int) *Client {
	return &Client{
		ccodePort: ccodePort,
	}
}

// Connect connects to a cocoon code server
// running on a known port
func (c *Client) Connect() error {

	log.Info("Starting cocoon code client")

	conn, err := grpc.Dial(fmt.Sprintf("127.0.0.1:%d", c.ccodePort), grpc.WithInsecure())
	if err != nil {
		return fmt.Errorf("failed to connect to cocoon code server. %s", err)
	}
	defer conn.Close()

	log.Debugf("Now connected to cocoon code at port=%d", c.ccodePort)

	c.stub = proto.NewStubClient(conn)

	if err = c.Do(conn); err != nil {
		return err
	}

	return nil
}

// Do starts a request loop that continously asks the
// server for transactions. When it receives a transaction,
// it processes it and returns the result.
func (c *Client) Do(conn *grpc.ClientConn) error {

	// create a context so we have complete controll of the connection
	c.conCtx, c.conCancel = context.WithCancel(context.Background())

	// connect to the cocoon code
	stream, err := c.stub.Transact(c.conCtx)
	if err != nil {
		return fmt.Errorf("failed to call GetTx of cocoon code stub. %s", err)
	}

	for {

		log.Info("Waiting for transactions")

		in, err := stream.Recv()
		if err == io.EOF {
			return fmt.Errorf("GetTx connection between connector and cocoon code stub has ended")
		}
		if err != nil {
			return fmt.Errorf("Failed to successfully receive message from cocoon code. %s", err)
		}

		log.Infof("New Tx From Cocoon: %", in.String())
	}
}
<commit_msg>Basic implementation of a sub routine to update orderer addresses every 60 seconds.<commit_after>package client

import (
	"fmt"

	"io"

	"os"

	"time"

	proto "github.com/ncodes/cocoon/core/stubs/golang/proto"
	logging "github.com/op/go-logging"
	context "golang.org/x/net/context"
	"google.golang.org/grpc"
)

var log = logging.MustGetLogger("connector.client")

// Client represents a cocoon code GRPC client
// that interacts with a cocoon code.
type Client struct {
	ccodePort        int
	stub             proto.StubClient
	conCtx           context.Context
	conCancel        context.CancelFunc
	orderDiscoTicker *time.Ticker
	orderersAddr     []string
}

// NewClient creates a new cocoon code client
func NewClient(ccodePort int) *Client {
	return &Client{
		ccodePort: ccodePort,
	}
}

// Connect connects to a cocoon code server
// running on a known port
func (c *Client) Connect() error {

	// start a ticker to continously discover orderer addreses
	go func() {
		c.discoverOrderers()
		c.orderDiscoTicker = time.NewTicker(60 * time.Second)
		for _ = range c.orderDiscoTicker.C {
			c.discoverOrderers()
		}
	}()

	log.Info("Starting cocoon code client")

	conn, err := grpc.Dial(fmt.Sprintf("127.0.0.1:%d", c.ccodePort), grpc.WithInsecure())
	if err != nil {
		return fmt.Errorf("failed to connect to cocoon code server. %s", err)
	}
	defer conn.Close()

	log.Debugf("Now connected to cocoon code at port=%d", c.ccodePort)

	c.stub = proto.NewStubClient(conn)

	if err = c.Do(conn); err != nil {
		return err
	}

	return nil
}

// discoverOrderers fetches a list of orderer service addresses
// via consul service discovery API. For development purpose,
// If DEV_ORDERER_ADDR is set, it will fetch the orderer
// address from the env variable.
func (c *Client) discoverOrderers() {
	if len(os.Getenv("DEV_ORDERER_ADDR")) > 0 {
		c.orderersAddr = []string{os.Getenv("DEV_ORDERER_ADDR")}
		log.Infof("Orderer address list updated. Contains %d orderer address(es)", len(c.orderersAddr))
	}
	// Retrieve from consul service API (not implemented)
}

// Do starts a request loop that continously asks the
// server for transactions. When it receives a transaction,
// it processes it and returns the result.
func (c *Client) Do(conn *grpc.ClientConn) error {

	// create a context so we have complete controll of the connection
	c.conCtx, c.conCancel = context.WithCancel(context.Background())

	// connect to the cocoon code
	stream, err := c.stub.Transact(c.conCtx)
	if err != nil {
		return fmt.Errorf("failed to call GetTx of cocoon code stub. %s", err)
	}

	for {

		log.Info("Waiting for transactions")

		in, err := stream.Recv()
		if err == io.EOF {
			return fmt.Errorf("GetTx connection between connector and cocoon code stub has ended")
		}
		if err != nil {
			return fmt.Errorf("Failed to successfully receive message from cocoon code. %s", err)
		}

		log.Infof("New Tx From Cocoon: %", in.String())
	}
}
<|endoftext|>


<commit_before>package sia

import (
	//"errors"
	"net"
	"strconv"
	"time"
)

const (
	timeout   = time.Second * 5
	maxMsgLen = 1 << 16
)

// A NetAddress contains the information needed to contact a peer over TCP.
type NetAddress struct {
	Host string
	Port uint16
}

func (na *NetAddress) String() string {
	return net.JoinHostPort(addr.Host, strconv.Itoa(int(addr.Port)))
}

// TBD
var BootstrapPeers = []NetAddress{}

// A TCPServer sends and receives messages. It also maintains an address book
// of peers to broadcast to and make requests of.
type TCPServer struct {
	net.Listener
	myAddr      NetAddress
	addressbook map[NetAddress]struct{}
}

func NewTCPServer(port uint16) (tcps *TCPServer, err error) {
	tcpServ, err := net.Listen("tcp", ":"+strconv.Itoa(int(port)))
	if err != nil {
		return
	}
	tcps = &TCPServer{
		Listener:    tcpServ,
		addressbook: make(map[NetAddress]struct{}),
	}
	go tcps.listen()
	return
}

// listen runs in the background, accepting incoming connections and serving
// them. listen will return after TCPServer.Close() is called, because the
// Accept() call will fail.
func (tcps *TCPServer) listen() {
	for {
		conn, err := tcps.Accept()
		if err != nil {
			return
		}
		// it is the handler's responsibility to close the connection
		go tcps.handleConn(conn)
	}
}

// handleConn reads header data from a connection, unmarshals the data
// structures it contains, and routes the data to other functions for
// processing.
func (tcps *TCPServer) handleConn(conn net.Conn) {
	defer conn.Close()
	var (
		msgType   = make([]byte, 1)
		msgLenBuf = make([]byte, 4)
		msgData   []byte // length determined by msgLen
	)
	// TODO: make this DRYer?
	if n, err := conn.Read(msgType); err != nil || n != 1 {
		// TODO: log error
		return
	}
	if n, err := conn.Read(msgLenBuf); err != nil || n != 4 {
		// TODO: log error
		return
	}
	msgLen := DecUint64(msgLenBuf)
	if msgLen > maxMsgLen {
		// TODO: log error
		return
	}
	msgData = make([]byte, msgLen)
	if n, err := conn.Read(msgData); err != nil || uint64(n) != msgLen {
		// TODO: log error
		return
	}

	switch msgType[0] {
	// Hostname discovery
	case 'H':
		_, err := conn.Write([]byte(conn.RemoteAddr().String()))
		if err != nil {
			// TODO: log error
			return
		}

	// Block
	case 'B':
		var b Block
		if err := Unmarshal(msgData, &b); err != nil {
			// TODO: log error
			return
		}
		//state.ProcessBlock(b)?

	// Transaction
	case 'T':
		var t Transaction
		if err := Unmarshal(msgData, &t); err != nil {
			// TODO: log error
			return
		}
		//state.ProcessTransaction(t)?

	// Unknown
	default:
		// TODO: log error
	}
	return
}

// Ping returns whether a NetAddress is reachable. It accomplishes this by
// initiating a TCP connection and immediately closes it. This is pretty
// unsophisticated. I'll add a Pong later.
func (tcps *TCPServer) Ping(addr NetAddress) bool {
	conn, err := net.Dial("tcp", addr.String())
	if err != nil {
		return false
	}
	conn.Close()
	return true
}

// send initiates a TCP connection and writes a message to it.
// TODO: add timeout
func (tcps *TCPServer) send(msg []byte, addr NetAddress) (err error) {
	conn, err := net.Dial("tcp", addr.String())
	if err != nil {
		return
	}
	_, err = conn.Write(msg)
	return
}

// learnHostname learns the external IP of the TCPServer.
func (tcps *TCPServer) learnHostname(addr NetAddress) (err error) {
	conn, err := net.Dial("tcp", addr.String())
	if err != nil {
		return
	}
	defer conn.Close()
	// send hostname request
	if _, err = conn.Write([]byte{'H', 0}); err != nil {
		return
	}
	// read response
	buf = make([]byte, 128)
	n, err := conn.Read(buf)
	if err != nil {
		return
	}
	// TODO: try to ping ourselves?
	host, portStr, err := net.SplitHostPort(string(buf[:n]))
	if err != nil {
		return
	}
	port, err := strconv.Atoi(portStr)
	if err != nil {
		return
	}
	tcps.myAddr = NetAddress{host, uint16(port)}
	return
}

// Bootstrap calls Request on a predefined set of peers in order to build up an
// initial peer list. It returns the number of peers added.
func (tcps *TCPServer) Bootstrap() int {
	n := len(tcps.addressbook)
	// for _, host := range BootstrapPeers {

	// }
	return len(tcps.addressbook) - n
}
<commit_msg>add peer discovery/bootstrapping<commit_after>package sia

import (
	//"errors"
	"net"
	"strconv"
	"time"
)

const (
	timeout   = time.Second * 5
	maxMsgLen = 1 << 16
)

// A NetAddress contains the information needed to contact a peer over TCP.
type NetAddress struct {
	Host string
	Port uint16
}

func (na *NetAddress) String() string {
	return net.JoinHostPort(addr.Host, strconv.Itoa(int(addr.Port)))
}

// TODO: add Dial
// takes fn as an arg??

// TBD
var BootstrapPeers = []NetAddress{}

// A TCPServer sends and receives messages. It also maintains an address book
// of peers to broadcast to and make requests of.
type TCPServer struct {
	net.Listener
	myAddr      NetAddress
	addressbook map[NetAddress]struct{}
}

func NewTCPServer(port uint16) (tcps *TCPServer, err error) {
	tcpServ, err := net.Listen("tcp", ":"+strconv.Itoa(int(port)))
	if err != nil {
		return
	}
	tcps = &TCPServer{
		Listener:    tcpServ,
		addressbook: make(map[NetAddress]struct{}),
	}
	go tcps.listen()
	return
}

// listen runs in the background, accepting incoming connections and serving
// them. listen will return after TCPServer.Close() is called, because the
// Accept() call will fail.
func (tcps *TCPServer) listen() {
	for {
		conn, err := tcps.Accept()
		if err != nil {
			return
		}
		// it is the handler's responsibility to close the connection
		go tcps.handleConn(conn)
	}
}

// handleConn reads header data from a connection, unmarshals the data
// structures it contains, and routes the data to other functions for
// processing.
func (tcps *TCPServer) handleConn(conn net.Conn) {
	defer conn.Close()
	var (
		msgType   []byte = make([]byte, 1)
		msgLenBuf []byte = make([]byte, 4)
		msgData   []byte // length determined by msgLen
	)
	// TODO: make this DRYer?
	if n, err := conn.Read(msgType); err != nil || n != 1 {
		// TODO: log error
		return
	}
	if n, err := conn.Read(msgLenBuf); err != nil || n != 4 {
		// TODO: log error
		return
	}
	msgLen := DecUint64(msgLenBuf)
	if msgLen > maxMsgLen {
		// TODO: log error
		return
	}
	msgData = make([]byte, msgLen)
	if n, err := conn.Read(msgData); err != nil || uint64(n) != msgLen {
		// TODO: log error
		return
	}

	switch msgType[0] {
	// Hostname discovery
	case 'H':
		_, err := conn.Write([]byte(conn.RemoteAddr().String()))
		if err != nil {
			// TODO: log error
			return
		}

	// Peer discovery
	case 'P':
		if msgLen != 1 {
			// TODO: log error
			return
		}
		tcps.sharePeers(conn, msgData[0])

	// Block
	case 'B':
		var b Block
		if err := Unmarshal(msgData, &b); err != nil {
			// TODO: log error
			return
		}
		//state.ProcessBlock(b)?

	// Transaction
	case 'T':
		var t Transaction
		if err := Unmarshal(msgData, &t); err != nil {
			// TODO: log error
			return
		}
		//state.ProcessTransaction(t)?

	// Unknown
	default:
		// TODO: log error
	}
	return
}

// Ping returns whether a NetAddress is reachable. It accomplishes this by
// initiating a TCP connection and immediately closes it. This is pretty
// unsophisticated. I'll add a Pong later.
func (tcps *TCPServer) Ping(addr NetAddress) bool {
	conn, err := net.Dial("tcp", addr.String())
	if err != nil {
		return false
	}
	conn.Close()
	return true
}

// send initiates a TCP connection and writes a message to it.
// TODO: add timeout
func (tcps *TCPServer) send(msg []byte, addr NetAddress) (err error) {
	conn, err := net.Dial("tcp", addr.String())
	if err != nil {
		return
	}
	_, err = conn.Write(msg)
	return
}

// learnHostname learns the external IP of the TCPServer.
func (tcps *TCPServer) learnHostname(addr NetAddress) (err error) {
	conn, err := net.Dial("tcp", addr.String())
	if err != nil {
		return
	}
	defer conn.Close()
	// send hostname request
	if _, err = conn.Write([]byte{'H', 0}); err != nil {
		return
	}
	// read response
	buf = make([]byte, 128)
	n, err := conn.Read(buf)
	if err != nil {
		return
	}
	// TODO: try to ping ourselves?
	host, portStr, err := net.SplitHostPort(string(buf[:n]))
	if err != nil {
		return
	}
	port, err := strconv.Atoi(portStr)
	if err != nil {
		return
	}
	tcps.myAddr = NetAddress{host, uint16(port)}
	return
}

// sharePeers transmits at most 'num' peers over the connection.
// TODO: choose random peers?
func (tcps *TCPServer) sharePeers(conn net.Conn, num uint8) {
	var addrs []NetAddress
	for addr := range tcps.addressbook {
		if num == 0 {
			break
		}
		addrs = append(addrs, addr)
		num--
	}
	conn.Write(Marshal(addrs))
	// log error?
}

// requestPeers queries a peer for additional peers, and adds any new peers to
// the address book.
func (tcps *TCPServer) requestPeers(addr NetAddress) (err error) {
	conn, err := net.Dial("tcp", addr.String())
	if err != nil {
		return
	}
	defer conn.Close()
	// request 10 peers
	if _, err = conn.Write([]byte{'P', 1, 10}); err != nil {
		return
	}
	// read response
	buf := make([]byte, 1024)
	n, err := conn.Read(buf)
	if err != nil {
		return
	}
	var addrs []NetAddress
	if err = Unmarshal(buf[:n], &addrs); err != nil {
		return
	}
	// add peers
	// TODO: make sure we don't add ourself
	for _, addr := range addrs {
		if tcps.Ping(addr) {
			tcps.addressbook[addr] = struct{}{}
		}
	}
	return
}

// Bootstrap discovers the external IP of the TCPServer, requests peers from
// the initial peer list, and announces itself to those peers.
func (tcps *TCPServer) Bootstrap() (err error) {
	// populate initial peer list
	for _, addr := range BootstrapPeers {
		if tcps.Ping(addr) {
			tcps.addressbook[addr] = struct{}{}
		}
	}

	// learn hostname
	for addr := range tcps.addressbook {
		if tcps.learnHostname() == nil {
			break
		}
	}
	// request peers
	// TODO: maybe iterate until we have enough new peers?
	for addr := range tcps.addressbook {
		tcps.requestPeers(addr)
	}
	// TODO: announce ourselves to new peers
	return
}
<|endoftext|>


<commit_before>package models

import (
	"fmt"
	"html/template"
	"regexp"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"

	"github.com/kennygrant/sanitize"
	"github.com/russross/blackfriday"
)

type Entry struct {
	Id       int64     `json:"id"`
	Title    string    `json:"title"`                     // optional
	Content  string    `datastore:",noindex" json:"text"` // Markdown
	Datetime time.Time `json:"date"`
	Created  time.Time `json:"created"`
	Modified time.Time `json:"modified"`
	Tags     []string  `json:"tags"`
	Public   bool      `json:"-"`
}

var HashtagRegex *regexp.Regexp = regexp.MustCompile(`(\s)#(\w+)`)
var TwitterHandleRegex *regexp.Regexp = regexp.MustCompile(`(\s)@([_A-Za-z0-9]+)`)

func NewEntry(title string, content string, datetime time.Time, public bool, tags []string) *Entry {
	e := new(Entry)

	// User supplied content
	e.Title = title
	e.Content = content
	e.Datetime = datetime
	e.Tags = tags
	e.Public = public

	// Computer generated content
	e.Created = time.Now()
	e.Modified = time.Now()

	return e
}

func ParseTags(text string) ([]string, error) {
	// http://golang.org/pkg/regexp/#Regexp.FindAllStringSubmatch
	finds := HashtagRegex.FindAllStringSubmatch(text, -1)
	ret := make([]string, 0)
	for _, v := range finds {
		if len(v) > 2 {
			ret = append(ret, strings.ToLower(v[2]))
		}
	}

	return ret, nil
}

func GetEntry(c appengine.Context, id int64) (*Entry, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Filter("Id =", id).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Warningf("Error getting entry %d", id)
		return nil, err
	}

	return &entry, nil
}

func MaxId(c appengine.Context) (int64, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("-Id").Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		return 0, err
	}
	return entry.Id, nil
}

func AllPosts(c appengine.Context) (*[]Entry, error) {
	return Posts(c, -1, true)
}

func Posts(c appengine.Context, limit int, recentFirst bool) (*[]Entry, error) {
	q := datastore.NewQuery("Entry").Filter("Public =", true)

	if recentFirst {
		q = q.Order("-Datetime")
	} else {
		q = q.Order("Datetime")
	}

	if limit > 0 {
		q = q.Limit(limit)
	}

	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	return entries, err
}

func RecentPosts(c appengine.Context) (*[]Entry, error) {
	return Posts(c, 20, true)
}

func (e *Entry) HasId() bool {
	return (e.Id > 0)
}

func (e *Entry) Save(c appengine.Context) error {
	var k *datastore.Key
	if !e.HasId() {
		id, _ := MaxId(c)
		e.Id = id + 1
		k = datastore.NewIncompleteKey(c, "Entry", nil)
	} else {
		// Find the key
		var err error
		q := datastore.NewQuery("Entry").Filter("Id =", e.Id).Limit(1).KeysOnly()
		k, err = q.Run(c).Next(nil)
		if err != nil {
			return err
		}
	}

	// Pull out links
	GetLinksFromContent(c, e.Content)

	// Figure out Tags
	tags, err := ParseTags(e.Content)
	if err != nil {
		return err
	}
	e.Tags = tags

	k2, err := datastore.Put(c, k, e)
	if err == nil {
		c.Infof("Wrote %+v", e)
		c.Infof("Old key: %+v; New Key: %+v", k, k2)
	} else {
		c.Warningf("Error writing entry: %v", e)
	}
	return err
}

func (e *Entry) Url() string {
	return fmt.Sprintf("/post/%d", e.Id)
}

func (e *Entry) EditUrl() string {
	return fmt.Sprintf("/edit/%d", e.Id)
}

func (e *Entry) Html() template.HTML {
	return Markdown(e.Content)
}

func (e *Entry) Summary() string {
	// truncate(strip_tags(m(p.text)), :length => 100).strip
	stripped := sanitize.HTML(string(e.Html()))
	if len(stripped) > 100 {
		return fmt.Sprintf("%s...", stripped[:100])
	} else {
		return stripped
	}
}

func (e *Entry) PrevPost(c appengine.Context) string {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("-Datetime").Filter("Datetime <", e.Datetime).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Infof("Error getting previous post for %d.", e.Id)
		return ""
	}

	return entry.Url()
}

func (e *Entry) NextPost(c appengine.Context) string {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("Datetime").Filter("Datetime >", e.Datetime).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Infof("Error getting next post for %d.", e.Id)
		return ""
	}

	return entry.Url()
}

// TODO(icco): Actually finish this.
func GetLinksFromContent(c appengine.Context, content string) ([]string, error) {
	httpRegex := regexp.MustCompile(`http:\/\/((\w|\.)+)`)
	matches := httpRegex.FindAllString(content, -1)
	if matches == nil {
		return []string{}, nil
	}

	for _, match := range matches {
		c.Infof("%+v", match)
	}

	return []string{}, nil
}

func PostsWithTag(c appengine.Context, tag string) (*map[int]Entry, error) {
	aliases := new([]Alias)
	entries := make(map[int64]Entry, 0)

	q := datastore.NewQuery("Alias").Filter("Tag =", tag)
	_, err := q.GetAll(c, aliases)
	if err != nil {
		return entries, err
	}

	for _, v := range *aliases {
		more_entries := new([]Entry)
		q := datastore.NewQuery("Entry").Order("-Datetime").Filter("Tags =", tag)
		_, err := q.GetAll(c, more_entries)
		for _, e := range *more_entries {
			entries[e.Id] = e
		}
	}

	return &entries, err
}

// Markdown.
func Markdown(args ...interface{}) template.HTML {
	inc := []byte(fmt.Sprintf("%s", args...))
	inc = twitterHandleToMarkdown(inc)
	inc = hashTagsToMarkdown(inc)
	s := blackfriday.MarkdownCommon(inc)
	return template.HTML(s)
}

func twitterHandleToMarkdown(in []byte) []byte {
	return TwitterHandleRegex.ReplaceAll(in, []byte("$1[@$2](http://twitter.com/$2)"))
}

func hashTagsToMarkdown(in []byte) []byte {
	return HashtagRegex.ReplaceAll(in, []byte("$1[#$2](/tags/$2)"))
}
<commit_msg>maybe<commit_after>package models

import (
	"fmt"
	"html/template"
	"regexp"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"

	"github.com/kennygrant/sanitize"
	"github.com/russross/blackfriday"
)

type Entry struct {
	Id       int64     `json:"id"`
	Title    string    `json:"title"`                     // optional
	Content  string    `datastore:",noindex" json:"text"` // Markdown
	Datetime time.Time `json:"date"`
	Created  time.Time `json:"created"`
	Modified time.Time `json:"modified"`
	Tags     []string  `json:"tags"`
	Public   bool      `json:"-"`
}

var HashtagRegex *regexp.Regexp = regexp.MustCompile(`(\s)#(\w+)`)
var TwitterHandleRegex *regexp.Regexp = regexp.MustCompile(`(\s)@([_A-Za-z0-9]+)`)

func NewEntry(title string, content string, datetime time.Time, public bool, tags []string) *Entry {
	e := new(Entry)

	// User supplied content
	e.Title = title
	e.Content = content
	e.Datetime = datetime
	e.Tags = tags
	e.Public = public

	// Computer generated content
	e.Created = time.Now()
	e.Modified = time.Now()

	return e
}

func ParseTags(text string) ([]string, error) {
	// http://golang.org/pkg/regexp/#Regexp.FindAllStringSubmatch
	finds := HashtagRegex.FindAllStringSubmatch(text, -1)
	ret := make([]string, 0)
	for _, v := range finds {
		if len(v) > 2 {
			ret = append(ret, strings.ToLower(v[2]))
		}
	}

	return ret, nil
}

func GetEntry(c appengine.Context, id int64) (*Entry, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Filter("Id =", id).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Warningf("Error getting entry %d", id)
		return nil, err
	}

	return &entry, nil
}

func MaxId(c appengine.Context) (int64, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("-Id").Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		return 0, err
	}
	return entry.Id, nil
}

func AllPosts(c appengine.Context) (*[]Entry, error) {
	return Posts(c, -1, true)
}

func Posts(c appengine.Context, limit int, recentFirst bool) (*[]Entry, error) {
	q := datastore.NewQuery("Entry").Filter("Public =", true)

	if recentFirst {
		q = q.Order("-Datetime")
	} else {
		q = q.Order("Datetime")
	}

	if limit > 0 {
		q = q.Limit(limit)
	}

	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	return entries, err
}

func RecentPosts(c appengine.Context) (*[]Entry, error) {
	return Posts(c, 20, true)
}

func (e *Entry) HasId() bool {
	return (e.Id > 0)
}

func (e *Entry) Save(c appengine.Context) error {
	var k *datastore.Key
	if !e.HasId() {
		id, _ := MaxId(c)
		e.Id = id + 1
		k = datastore.NewIncompleteKey(c, "Entry", nil)
	} else {
		// Find the key
		var err error
		q := datastore.NewQuery("Entry").Filter("Id =", e.Id).Limit(1).KeysOnly()
		k, err = q.Run(c).Next(nil)
		if err != nil {
			return err
		}
	}

	// Pull out links
	GetLinksFromContent(c, e.Content)

	// Figure out Tags
	tags, err := ParseTags(e.Content)
	if err != nil {
		return err
	}
	e.Tags = tags

	k2, err := datastore.Put(c, k, e)
	if err == nil {
		c.Infof("Wrote %+v", e)
		c.Infof("Old key: %+v; New Key: %+v", k, k2)
	} else {
		c.Warningf("Error writing entry: %v", e)
	}
	return err
}

func (e *Entry) Url() string {
	return fmt.Sprintf("/post/%d", e.Id)
}

func (e *Entry) EditUrl() string {
	return fmt.Sprintf("/edit/%d", e.Id)
}

func (e *Entry) Html() template.HTML {
	return Markdown(e.Content)
}

func (e *Entry) Summary() string {
	// truncate(strip_tags(m(p.text)), :length => 100).strip
	stripped := sanitize.HTML(string(e.Html()))
	if len(stripped) > 100 {
		return fmt.Sprintf("%s...", stripped[:100])
	} else {
		return stripped
	}
}

func (e *Entry) PrevPost(c appengine.Context) string {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("-Datetime").Filter("Datetime <", e.Datetime).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Infof("Error getting previous post for %d.", e.Id)
		return ""
	}

	return entry.Url()
}

func (e *Entry) NextPost(c appengine.Context) string {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("Datetime").Filter("Datetime >", e.Datetime).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Infof("Error getting next post for %d.", e.Id)
		return ""
	}

	return entry.Url()
}

// TODO(icco): Actually finish this.
func GetLinksFromContent(c appengine.Context, content string) ([]string, error) {
	httpRegex := regexp.MustCompile(`http:\/\/((\w|\.)+)`)
	matches := httpRegex.FindAllString(content, -1)
	if matches == nil {
		return []string{}, nil
	}

	for _, match := range matches {
		c.Infof("%+v", match)
	}

	return []string{}, nil
}

func PostsWithTag(c appengine.Context, tag string) (*map[int64]Entry, error) {
	aliases := new([]Alias)
	entries := make(map[int64]Entry, 0)

	q := datastore.NewQuery("Alias").Filter("Tag =", tag)
	_, err := q.GetAll(c, aliases)
	if err != nil {
		return nil, err
	}

	for _, v := range *aliases {
		more_entries := new([]Entry)
		q := datastore.NewQuery("Entry").Order("-Datetime").Filter("Tags =", tag)
		_, err := q.GetAll(c, more_entries)
		for _, e := range *more_entries {
			entries[e.Id] = e
		}
	}

	return &entries, err
}

// Markdown.
func Markdown(args ...interface{}) template.HTML {
	inc := []byte(fmt.Sprintf("%s", args...))
	inc = twitterHandleToMarkdown(inc)
	inc = hashTagsToMarkdown(inc)
	s := blackfriday.MarkdownCommon(inc)
	return template.HTML(s)
}

func twitterHandleToMarkdown(in []byte) []byte {
	return TwitterHandleRegex.ReplaceAll(in, []byte("$1[@$2](http://twitter.com/$2)"))
}

func hashTagsToMarkdown(in []byte) []byte {
	return HashtagRegex.ReplaceAll(in, []byte("$1[#$2](/tags/$2)"))
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"
	"net"
	"net/http"
)

func HttpServer(listener net.Listener) {
	log.Printf("HTTP: listening on %s", listener.Addr().String())
	http.HandleFunc("/ping", pingHandler)
	err := http.Serve(listener, nil)
	if err != nil {
		log.Fatal("http.ListenAndServe:", err)
	}
}

func pingHandler(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}
<commit_msg>add /producers HTTP endpoint to nsqlookupd<commit_after>package main

import (
	"../util"
	"encoding/json"
	"io"
	"log"
	"net"
	"net/http"
	"strconv"
)

func HttpServer(listener net.Listener) {
	log.Printf("HTTP: listening on %s", listener.Addr().String())
	http.HandleFunc("/ping", pingHandler)
	http.HandleFunc("/lookup", lookupHandler)
	err := http.Serve(listener, nil)
	if err != nil {
		log.Fatal("http.ListenAndServe:", err)
	}
}

func pingHandler(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Length", "2")
	io.WriteString(w, "OK")
}

func lookupHandler(w http.ResponseWriter, req *http.Request) {
	reqParams, err := util.NewReqParams(req)
	if err != nil {
		log.Printf("ERROR: failed to parse request params - %s", err.Error())
		// TODO: return default response
		return
	}

	topicName, err := reqParams.Query("topic")
	if err != nil {
		// TODO: return default response
		return
	}

	dataInterface, err := sm.Get("topic." + topicName)
	if err != nil {
		// TODO: return default response
		return
	}

	data := dataInterface.(map[string]interface{})
	response, err := json.Marshal(&data)
	if err != nil {
		// TODO: return default response
		return
	}

	w.Header().Set("Content-Length", strconv.Itoa(len(response)))
	w.Write(response)
}
<|endoftext|>


<commit_before>package consensus

import (
	"github.com/golang/glog"
	"github.com/heidi-ann/ios/msgs"
	"reflect"
)

type State struct {
	View        int // local view number
	Log         []msgs.Entry
	CommitIndex int
	MasterID    int
	LastIndex   int
}

func mod(x int, y int) int {
	dif := x - y
	if dif < y {
		return x
	} else {
		return mod(dif, y)
	}
}

// check protocol invariant
func checkInvariant(log []msgs.Entry, index int, nxtEntry msgs.Entry) {
	prevEntry := log[index]

	// if no entry, then no problem
	if !reflect.DeepEqual(prevEntry, msgs.Entry{}) {
		// if committed, request never changes
		if prevEntry.Committed && !reflect.DeepEqual(prevEntry.Requests, nxtEntry.Requests) {
			glog.Fatal("Committed entry is being overwritten at ", prevEntry, nxtEntry, index)
		}
		// each index is allocated once per term
		if prevEntry.View == nxtEntry.View && !reflect.DeepEqual(prevEntry.Requests, nxtEntry.Requests) {
			glog.Fatal("Index has been reallocated at ", prevEntry, nxtEntry, index)
		}
	}
}

// PROTOCOL BODY

func MonitorMaster(s *State, io *msgs.Io, config Config) {
	for {
		failed := <-io.Failure
		if failed == (*s).MasterID {
			nextMaster := mod((*s).View+1, config.N)
			glog.Warningf("Master (ID:%d) failed, next up is ID:%d", (*s).MasterID, nextMaster)
			if nextMaster == config.ID {
				glog.Info("Starting new master at ", config.ID)
				(*s).View++
				// TODO: BUG need to write to disk
				(*s).MasterID = nextMaster
				go RunMaster((*s).View, (*s).CommitIndex, false, io, config)
			}
		}

	}
}

func RunParticipant(state State, io *msgs.Io, config Config) {
	go MonitorMaster(&state, io, config)

	glog.Info("Ready for requests")
	for {

		// get request
		select {

		case req := <-(*io).Incoming.Requests.Prepare:
			glog.Info("Prepare requests recieved at ", config.ID, ": ", req)
			// check view
			if req.View < state.View {
				glog.Warning("Sender is behind")
				reply := msgs.PrepareResponse{config.ID, false}
				(*io).OutgoingUnicast[req.SenderID].Responses.Prepare <- msgs.Prepare{req, reply}
				break

			}

			if req.View > state.View {
				glog.Warning("Participant is behind")
				state.View = req.View
				(*io).ViewPersist <- state.View
				state.MasterID = mod(state.View, config.N)
			}

			// check sender is master
			if req.SenderID != state.MasterID {
				glog.Warningf("Sender (ID %d) is the not master (ID %d)", req.SenderID, state.MasterID)
				reply := msgs.PrepareResponse{config.ID, false}
				(*io).OutgoingUnicast[req.SenderID].Responses.Prepare <- msgs.Prepare{req, reply}
				break
			}

			// add entry
			if req.Index > state.LastIndex {
				state.LastIndex = req.Index
			} else {
				checkInvariant(state.Log, req.Index, req.Entry)
			}
			state.Log[req.Index] = req.Entry
			(*io).LogPersist <- msgs.LogUpdate{req.Index, req.Entry}
			_ = <-(*io).LogPersistFsync

			// reply
			reply := msgs.PrepareResponse{config.ID, true}
			(*(*io).OutgoingUnicast[req.SenderID]).Responses.Prepare <- msgs.Prepare{req, reply}
			glog.Info("Response dispatched: ", reply)

		case req := <-(*io).Incoming.Requests.Commit:
			glog.Info("Commit requests recieved at ", config.ID, ": ", req)
			// check view
			if req.View < state.View {
				glog.Warning("Sender is behind")
				break

			}

			if req.View > state.View {
				glog.Warning("Participant is behind")
				state.View = req.View
				(*io).ViewPersist <- state.View
				state.MasterID = mod(state.View, config.N)
			}

			// check sender is master
			if req.SenderID != state.MasterID {
				glog.Warning("Sender is not master")
				break
			}

			// add entry
			if req.Index > state.LastIndex {
				state.LastIndex = req.Index
			} else {
				checkInvariant(state.Log, req.Index, req.Entry)
			}
			state.Log[req.Index] = req.Entry
			// (*io).LogPersist <- msgs.LogUpdate{req.Index, req.Entry}

			// pass to state machine if ready
			if state.CommitIndex == req.Index-1 {

				for _, request := range req.Entry.Requests {
					(*io).OutgoingRequests <- request
				}
				state.CommitIndex++

				reply := msgs.CommitResponse{config.ID, true, state.CommitIndex}
				(*(*io).OutgoingUnicast[req.SenderID]).Responses.Commit <- msgs.Commit{req, reply}
				glog.Info("Entry Committed")
			} else {

				reply := msgs.CommitResponse{config.ID, false, state.CommitIndex}
				(*(*io).OutgoingUnicast[req.SenderID]).Responses.Commit <- msgs.Commit{req, reply}
				glog.Info("Entry not yet committed")
			}
			glog.Info("Response dispatched")

		case req := <-(*io).Incoming.Requests.NewView:
			glog.Info("New view requests recieved at ", config.ID, ": ", req)

			// check view
			if req.View < state.View {
				glog.Warning("Sender is behind")
				break

			}

			if req.View > state.View {
				glog.Warning("Participant is behind")
				state.View = req.View
				(*io).ViewPersist <- state.View
				state.MasterID = mod(state.View, config.N)
			}

			reply := msgs.NewViewResponse{config.ID, state.View, state.LastIndex}
			(*io).OutgoingUnicast[req.SenderID].Responses.NewView <- msgs.NewView{req, reply}
			glog.Info("Response dispatched")

		case req := <-(*io).Incoming.Requests.Query:
			glog.Info("Query requests recieved at ", config.ID, ": ", req)

			// check view
			if req.View < state.View {
				glog.Warning("Sender is behind")
				break

			}

			if req.View > state.View {
				glog.Warning("Participant is behind")
				state.View = req.View
				(*io).ViewPersist <- state.View
				state.MasterID = mod(state.View, config.N)
			}

			present := state.LastIndex >= req.Index
			reply := msgs.QueryResponse{config.ID, state.View, present, state.Log[req.Index]}
			(*io).OutgoingUnicast[req.SenderID].Responses.Query <- msgs.Query{req, reply}
		}
	}
}
<commit_msg>adding data wait<commit_after>package consensus

import (
	"github.com/golang/glog"
	"github.com/heidi-ann/ios/msgs"
	"reflect"
)

type State struct {
	View        int // local view number
	Log         []msgs.Entry
	CommitIndex int
	MasterID    int
	LastIndex   int
}

func mod(x int, y int) int {
	dif := x - y
	if dif < y {
		return x
	} else {
		return mod(dif, y)
	}
}

// check protocol invariant
func checkInvariant(log []msgs.Entry, index int, nxtEntry msgs.Entry) {
	prevEntry := log[index]

	// if no entry, then no problem
	if !reflect.DeepEqual(prevEntry, msgs.Entry{}) {
		// if committed, request never changes
		if prevEntry.Committed && !reflect.DeepEqual(prevEntry.Requests, nxtEntry.Requests) {
			glog.Fatal("Committed entry is being overwritten at ", prevEntry, nxtEntry, index)
		}
		// each index is allocated once per term
		if prevEntry.View == nxtEntry.View && !reflect.DeepEqual(prevEntry.Requests, nxtEntry.Requests) {
			glog.Fatal("Index has been reallocated at ", prevEntry, nxtEntry, index)
		}
	}
}

// PROTOCOL BODY

func MonitorMaster(s *State, io *msgs.Io, config Config) {
	for {
		failed := <-io.Failure
		if failed == (*s).MasterID {
			nextMaster := mod((*s).View+1, config.N)
			glog.Warningf("Master (ID:%d) failed, next up is ID:%d", (*s).MasterID, nextMaster)
			if nextMaster == config.ID {
				glog.Info("Starting new master at ", config.ID)
				(*s).View++
				// TODO: BUG need to write to disk
				(*s).MasterID = nextMaster
				go RunMaster((*s).View, (*s).CommitIndex, false, io, config)
			}
		}

	}
}

func RunParticipant(state State, io *msgs.Io, config Config) {
	go MonitorMaster(&state, io, config)

	glog.Info("Ready for requests")
	for {

		// get request
		select {

		case req := <-(*io).Incoming.Requests.Prepare:
			glog.Info("Prepare requests recieved at ", config.ID, ": ", req)
			// check view
			if req.View < state.View {
				glog.Warning("Sender is behind")
				reply := msgs.PrepareResponse{config.ID, false}
				(*io).OutgoingUnicast[req.SenderID].Responses.Prepare <- msgs.Prepare{req, reply}
				break

			}

			if req.View > state.View {
				glog.Warning("Participant is behind")
				state.View = req.View
				(*io).ViewPersist <- state.View
				state.MasterID = mod(state.View, config.N)
			}

			// check sender is master
			if req.SenderID != state.MasterID {
				glog.Warningf("Sender (ID %d) is the not master (ID %d)", req.SenderID, state.MasterID)
				reply := msgs.PrepareResponse{config.ID, false}
				(*io).OutgoingUnicast[req.SenderID].Responses.Prepare <- msgs.Prepare{req, reply}
				break
			}

			// add entry
			if req.Index > state.LastIndex {
				state.LastIndex = req.Index
			} else {
				checkInvariant(state.Log, req.Index, req.Entry)
			}
			state.Log[req.Index] = req.Entry
			(*io).LogPersist <- msgs.LogUpdate{req.Index, req.Entry}
			last_written := <-(*io).LogPersistFsync
			for !reflect.DeepEqual(last_written,msgs.LogUpdate{req.Index, req.Entry}) {
				last_written = <-(*io).LogPersistFsync
			}

			// reply
			reply := msgs.PrepareResponse{config.ID, true}
			(*(*io).OutgoingUnicast[req.SenderID]).Responses.Prepare <- msgs.Prepare{req, reply}
			glog.Info("Response dispatched: ", reply)

		case req := <-(*io).Incoming.Requests.Commit:
			glog.Info("Commit requests recieved at ", config.ID, ": ", req)
			// check view
			if req.View < state.View {
				glog.Warning("Sender is behind")
				break

			}

			if req.View > state.View {
				glog.Warning("Participant is behind")
				state.View = req.View
				(*io).ViewPersist <- state.View
				state.MasterID = mod(state.View, config.N)
			}

			// check sender is master
			if req.SenderID != state.MasterID {
				glog.Warning("Sender is not master")
				break
			}

			// add entry
			if req.Index > state.LastIndex {
				state.LastIndex = req.Index
			} else {
				checkInvariant(state.Log, req.Index, req.Entry)
			}
			state.Log[req.Index] = req.Entry
			// (*io).LogPersist <- msgs.LogUpdate{req.Index, req.Entry}

			// pass to state machine if ready
			if state.CommitIndex == req.Index-1 {

				for _, request := range req.Entry.Requests {
					(*io).OutgoingRequests <- request
				}
				state.CommitIndex++

				reply := msgs.CommitResponse{config.ID, true, state.CommitIndex}
				(*(*io).OutgoingUnicast[req.SenderID]).Responses.Commit <- msgs.Commit{req, reply}
				glog.Info("Entry Committed")
			} else {

				reply := msgs.CommitResponse{config.ID, false, state.CommitIndex}
				(*(*io).OutgoingUnicast[req.SenderID]).Responses.Commit <- msgs.Commit{req, reply}
				glog.Info("Entry not yet committed")
			}
			glog.Info("Response dispatched")

		case req := <-(*io).Incoming.Requests.NewView:
			glog.Info("New view requests recieved at ", config.ID, ": ", req)

			// check view
			if req.View < state.View {
				glog.Warning("Sender is behind")
				break

			}

			if req.View > state.View {
				glog.Warning("Participant is behind")
				state.View = req.View
				(*io).ViewPersist <- state.View
				state.MasterID = mod(state.View, config.N)
			}

			reply := msgs.NewViewResponse{config.ID, state.View, state.LastIndex}
			(*io).OutgoingUnicast[req.SenderID].Responses.NewView <- msgs.NewView{req, reply}
			glog.Info("Response dispatched")

		case req := <-(*io).Incoming.Requests.Query:
			glog.Info("Query requests recieved at ", config.ID, ": ", req)

			// check view
			if req.View < state.View {
				glog.Warning("Sender is behind")
				break

			}

			if req.View > state.View {
				glog.Warning("Participant is behind")
				state.View = req.View
				(*io).ViewPersist <- state.View
				state.MasterID = mod(state.View, config.N)
			}

			present := state.LastIndex >= req.Index
			reply := msgs.QueryResponse{config.ID, state.View, present, state.Log[req.Index]}
			(*io).OutgoingUnicast[req.SenderID].Responses.Query <- msgs.Query{req, reply}
		}
	}
}
<|endoftext|>


<commit_before>package corerouting

import (
	"errors"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	core "github.com/jbenet/go-ipfs/core"
	"github.com/jbenet/go-ipfs/p2p/peer"
	routing "github.com/jbenet/go-ipfs/routing"
	grandcentral "github.com/jbenet/go-ipfs/routing/grandcentral"
	gcproxy "github.com/jbenet/go-ipfs/routing/grandcentral/proxy"
)

// NB: DHT option is included in the core to avoid 1) because it's a sane
// default and 2) to avoid a circular dependency (it needs to be referenced in
// the core if it's going to be the default)

var (
	errHostMissing      = errors.New("grandcentral client requires a Host component")
	errIdentityMissing  = errors.New("grandcentral server requires a peer ID identity")
	errPeerstoreMissing = errors.New("grandcentral server requires a peerstore")
	errServersMissing   = errors.New("grandcentral client requires at least 1 server peer")
)

// TODO doc
func GrandCentralServer(recordSource datastore.ThreadSafeDatastore) core.RoutingOption {
	return func(ctx context.Context, node *core.IpfsNode) (routing.IpfsRouting, error) {
		if node.Peerstore == nil {
			return nil, errPeerstoreMissing
		}
		if node.PeerHost == nil {
			return nil, errHostMissing
		}
		if node.Identity == "" {
			return nil, errIdentityMissing
		}
		server, err := grandcentral.NewServer(recordSource, node.Peerstore, node.Identity)
		if err != nil {
			return nil, err
		}
		proxy := &gcproxy.Loopback{
			Handler: server,
			Local:   node.Identity,
		}
		node.PeerHost.SetStreamHandler(gcproxy.ProtocolGCR, proxy.HandleStream)
		return grandcentral.NewClient(proxy, node.PeerHost, node.Peerstore, node.Identity)
	}
}

// TODO doc
func GrandCentralClient(remotes ...peer.PeerInfo) core.RoutingOption {
	return func(ctx context.Context, node *core.IpfsNode) (routing.IpfsRouting, error) {
		if len(remotes) < 1 {
			return nil, errServersMissing
		}
		if node.PeerHost == nil {
			return nil, errHostMissing
		}
		if node.Identity == "" {
			return nil, errIdentityMissing
		}
		if node.Peerstore == nil {
			return nil, errors.New("need peerstore")
		}

		// TODO move to bootstrap method
		for _, info := range remotes {
			if err := node.PeerHost.Connect(ctx, info); err != nil {
				return nil, err // TODO
			}
		}

		// TODO right now, I think this has a hidden dependency on the
		// bootstrap peers provided to the core.Node. Careful...

		var ids []peer.ID
		for _, info := range remotes {
			ids = append(ids, info.ID)
		}
		proxy := gcproxy.Standard(node.PeerHost, ids)
		node.PeerHost.SetStreamHandler(gcproxy.ProtocolGCR, proxy.HandleStream)
		return grandcentral.NewClient(proxy, node.PeerHost, node.Peerstore, node.Identity)
	}
}
<commit_msg>feat(gcr/s) comment<commit_after>package corerouting

import (
	"errors"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	core "github.com/jbenet/go-ipfs/core"
	"github.com/jbenet/go-ipfs/p2p/peer"
	routing "github.com/jbenet/go-ipfs/routing"
	grandcentral "github.com/jbenet/go-ipfs/routing/grandcentral"
	gcproxy "github.com/jbenet/go-ipfs/routing/grandcentral/proxy"
)

// NB: DHT option is included in the core to avoid 1) because it's a sane
// default and 2) to avoid a circular dependency (it needs to be referenced in
// the core if it's going to be the default)

var (
	errHostMissing      = errors.New("grandcentral client requires a Host component")
	errIdentityMissing  = errors.New("grandcentral server requires a peer ID identity")
	errPeerstoreMissing = errors.New("grandcentral server requires a peerstore")
	errServersMissing   = errors.New("grandcentral client requires at least 1 server peer")
)

// GrandCentralServer returns a configuration for a routing server that stores
// routing records to the provided datastore. Only routing records are store in
// the datastore.
func GrandCentralServer(recordSource datastore.ThreadSafeDatastore) core.RoutingOption {
	return func(ctx context.Context, node *core.IpfsNode) (routing.IpfsRouting, error) {
		if node.Peerstore == nil {
			return nil, errPeerstoreMissing
		}
		if node.PeerHost == nil {
			return nil, errHostMissing
		}
		if node.Identity == "" {
			return nil, errIdentityMissing
		}
		server, err := grandcentral.NewServer(recordSource, node.Peerstore, node.Identity)
		if err != nil {
			return nil, err
		}
		proxy := &gcproxy.Loopback{
			Handler: server,
			Local:   node.Identity,
		}
		node.PeerHost.SetStreamHandler(gcproxy.ProtocolGCR, proxy.HandleStream)
		return grandcentral.NewClient(proxy, node.PeerHost, node.Peerstore, node.Identity)
	}
}

// TODO doc
func GrandCentralClient(remotes ...peer.PeerInfo) core.RoutingOption {
	return func(ctx context.Context, node *core.IpfsNode) (routing.IpfsRouting, error) {
		if len(remotes) < 1 {
			return nil, errServersMissing
		}
		if node.PeerHost == nil {
			return nil, errHostMissing
		}
		if node.Identity == "" {
			return nil, errIdentityMissing
		}
		if node.Peerstore == nil {
			return nil, errors.New("need peerstore")
		}

		// TODO move to bootstrap method
		for _, info := range remotes {
			if err := node.PeerHost.Connect(ctx, info); err != nil {
				return nil, err // TODO
			}
		}

		// TODO right now, I think this has a hidden dependency on the
		// bootstrap peers provided to the core.Node. Careful...

		var ids []peer.ID
		for _, info := range remotes {
			ids = append(ids, info.ID)
		}
		proxy := gcproxy.Standard(node.PeerHost, ids)
		node.PeerHost.SetStreamHandler(gcproxy.ProtocolGCR, proxy.HandleStream)
		return grandcentral.NewClient(proxy, node.PeerHost, node.Peerstore, node.Identity)
	}
}
<|endoftext|>


<commit_before>package hoverfly

import (
	"fmt"
	"regexp"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/SpectoLabs/hoverfly/core/cache"
	"github.com/SpectoLabs/hoverfly/core/handlers/v1"
	"github.com/SpectoLabs/hoverfly/core/handlers/v2"
	"github.com/SpectoLabs/hoverfly/core/interfaces"
	"github.com/SpectoLabs/hoverfly/core/metrics"
	"github.com/SpectoLabs/hoverfly/core/models"
)

func (this Hoverfly) GetDestination() string {
	return this.Cfg.Destination
}

// UpdateDestination - updates proxy with new destination regexp
func (hf *Hoverfly) SetDestination(destination string) (err error) {
	_, err = regexp.Compile(destination)
	if err != nil {
		return fmt.Errorf("destination is not a valid regular expression string")
	}

	hf.mu.Lock()
	hf.StopProxy()
	hf.Cfg.Destination = destination
	err = hf.StartProxy()
	hf.mu.Unlock()
	return
}

func (this Hoverfly) GetMode() string {
	return this.Cfg.Mode
}

func (this *Hoverfly) SetMode(mode string) error {
	availableModes := map[string]bool{
		"simulate":   true,
		"capture":    true,
		"modify":     true,
		"synthesize": true,
	}

	if mode == "" || !availableModes[mode] {
		log.Error("Can't change mode to \"%d\"", mode)
		return fmt.Errorf("Not a valid mode")
	}

	if this.Cfg.Webserver {
		log.Error("Can't change state when configured as a webserver ")
		return fmt.Errorf("Can't change state when configured as a webserver")
	}
	this.Cfg.SetMode(mode)
	return nil
}

func (hf Hoverfly) GetMiddleware() string {
	return hf.Cfg.Middleware.FullCommand
}

func (hf Hoverfly) GetMiddlewareV2() (string, string, string) {
	script, _ := hf.Cfg.Middleware.GetScript()
	return hf.Cfg.Middleware.Binary, script, hf.Cfg.Middleware.Remote
}

func (hf Hoverfly) SetMiddleware(middleware string) error {
	if middleware == "" {
		hf.Cfg.Middleware.FullCommand = middleware
		return nil
	}
	originalPair := models.RequestResponsePair{
		Request: models.RequestDetails{
			Path:        "/",
			Method:      "GET",
			Destination: "www.test.com",
			Scheme:      "",
			Query:       "",
			Body:        "",
			Headers:     map[string][]string{"test_header": []string{"true"}},
		},
		Response: models.ResponseDetails{
			Status:  200,
			Body:    "ok",
			Headers: map[string][]string{"test_header": []string{"true"}},
		},
	}

	middlewareObject := &Middleware{
		FullCommand: middleware,
	}

	_, err := middlewareObject.executeMiddlewareLocally(originalPair)
	if err != nil {
		return err
	}

	hf.Cfg.Middleware.FullCommand = middleware
	return nil
}

func (hf *Hoverfly) SetMiddlewareV2(binary, script string) error {
	newMiddleware := Middleware{}

	if binary == "" && script == "" {
		hf.Cfg.Middleware = newMiddleware
		return nil
	} else if binary == "" {
		return fmt.Errorf("Cannot run script with no binary")
	}

	err := newMiddleware.SetBinary(binary)
	if err != nil {
		return err
	}

	err = newMiddleware.SetScript(script)
	if err != nil {
		return nil
	}

	testData := models.RequestResponsePair{
		Request: models.RequestDetails{
			Path:        "/",
			Method:      "GET",
			Destination: "www.test.com",
			Scheme:      "",
			Query:       "",
			Body:        "",
			Headers:     map[string][]string{"test_header": []string{"true"}},
		},
		Response: models.ResponseDetails{
			Status:  200,
			Body:    "ok",
			Headers: map[string][]string{"test_header": []string{"true"}},
		},
	}
	_, err = newMiddleware.executeMiddlewareLocally(testData)
	if err != nil {
		return err
	}

	hf.Cfg.Middleware = newMiddleware
	return nil
}

func (hf Hoverfly) GetRequestCacheCount() (int, error) {
	return hf.RequestCache.RecordsCount()
}

func (this Hoverfly) GetMetadataCache() cache.Cache {
	return this.MetadataCache
}

func (hf Hoverfly) DeleteRequestCache() error {
	return hf.RequestCache.DeleteData()
}

func (this Hoverfly) GetTemplates() v1.RequestTemplateResponsePairPayload {
	return this.RequestMatcher.TemplateStore.GetPayload()
}

func (this *Hoverfly) ImportTemplates(pairPayload v1.RequestTemplateResponsePairPayload) error {
	return this.RequestMatcher.TemplateStore.ImportPayloads(pairPayload)
}

func (this *Hoverfly) DeleteTemplateCache() {
	this.RequestMatcher.TemplateStore.Wipe()
}

func (hf *Hoverfly) GetResponseDelays() v1.ResponseDelayPayloadView {
	return hf.ResponseDelays.ConvertToResponseDelayPayloadView()
}

func (hf *Hoverfly) SetResponseDelays(payloadView v1.ResponseDelayPayloadView) error {
	err := models.ValidateResponseDelayPayload(payloadView)
	if err != nil {
		return err
	}

	var responseDelays models.ResponseDelayList

	for _, responseDelayView := range payloadView.Data {
		responseDelays = append(responseDelays, models.ResponseDelay{
			UrlPattern: responseDelayView.UrlPattern,
			HttpMethod: responseDelayView.HttpMethod,
			Delay:      responseDelayView.Delay,
		})
	}

	hf.ResponseDelays = &responseDelays
	return nil
}

func (hf *Hoverfly) DeleteResponseDelays() {
	hf.ResponseDelays = &models.ResponseDelayList{}
}

func (hf Hoverfly) GetStats() metrics.Stats {
	return hf.Counter.Flush()
}

func (hf Hoverfly) GetRecords() ([]v1.RequestResponsePairView, error) {
	records, err := hf.RequestCache.GetAllEntries()
	if err != nil {
		return nil, err
	}

	var pairViews []v1.RequestResponsePairView

	for _, v := range records {
		if pair, err := models.NewRequestResponsePairFromBytes(v); err == nil {
			pairView := pair.ConvertToV1RequestResponsePairView()
			pairViews = append(pairViews, *pairView)
		} else {
			log.Error(err)
			return nil, err
		}
	}

	for _, v := range hf.RequestMatcher.TemplateStore {
		pairView := v.ConvertToV1RequestResponsePairView()
		pairViews = append(pairViews, pairView)
	}

	return pairViews, nil

}

func (hf Hoverfly) GetSimulation() (v2.SimulationView, error) {
	records, err := hf.RequestCache.GetAllEntries()
	if err != nil {
		return v2.SimulationView{}, err
	}

	pairViews := make([]v2.RequestResponsePairView, 0)

	for _, v := range records {
		if pair, err := models.NewRequestResponsePairFromBytes(v); err == nil {
			pairView := pair.ConvertToRequestResponsePairView()
			pairViews = append(pairViews, pairView)
		} else {
			log.Error(err)
			return v2.SimulationView{}, err
		}
	}

	for _, v := range hf.RequestMatcher.TemplateStore {
		pairViews = append(pairViews, v.ConvertToRequestResponsePairView())
	}

	responseDelays := hf.ResponseDelays.ConvertToResponseDelayPayloadView()

	return v2.SimulationView{
		MetaView: v2.MetaView{
			HoverflyVersion: "v0.9.2",
			SchemaVersion:   "v1",
			TimeExported:    time.Now().Format(time.RFC3339),
		},
		DataView: v2.DataView{
			RequestResponsePairs: pairViews,
			GlobalActions: v2.GlobalActionsView{
				Delays: responseDelays.Data,
			},
		},
	}, nil
}

func (this *Hoverfly) PutSimulation(simulationView v2.SimulationView) error {
	requestResponsePairViews := make([]interfaces.RequestResponsePair, len(simulationView.RequestResponsePairs))
	for i, v := range simulationView.RequestResponsePairs {
		requestResponsePairViews[i] = v
	}

	err := this.ImportRequestResponsePairViews(requestResponsePairViews)
	if err != nil {
		return err
	}

	err = this.SetResponseDelays(v1.ResponseDelayPayloadView{Data: simulationView.GlobalActions.Delays})
	if err != nil {
		return err
	}

	return nil
}

func (this *Hoverfly) DeleteSimulation() {
	this.DeleteTemplateCache()
	this.DeleteResponseDelays()
	this.DeleteRequestCache()
}
<commit_msg>Updated the SetMiddleware() functions to use Execute instead of executeLocalMiddleware()<commit_after>package hoverfly

import (
	"fmt"
	"regexp"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/SpectoLabs/hoverfly/core/cache"
	"github.com/SpectoLabs/hoverfly/core/handlers/v1"
	"github.com/SpectoLabs/hoverfly/core/handlers/v2"
	"github.com/SpectoLabs/hoverfly/core/interfaces"
	"github.com/SpectoLabs/hoverfly/core/metrics"
	"github.com/SpectoLabs/hoverfly/core/models"
)

func (this Hoverfly) GetDestination() string {
	return this.Cfg.Destination
}

// UpdateDestination - updates proxy with new destination regexp
func (hf *Hoverfly) SetDestination(destination string) (err error) {
	_, err = regexp.Compile(destination)
	if err != nil {
		return fmt.Errorf("destination is not a valid regular expression string")
	}

	hf.mu.Lock()
	hf.StopProxy()
	hf.Cfg.Destination = destination
	err = hf.StartProxy()
	hf.mu.Unlock()
	return
}

func (this Hoverfly) GetMode() string {
	return this.Cfg.Mode
}

func (this *Hoverfly) SetMode(mode string) error {
	availableModes := map[string]bool{
		"simulate":   true,
		"capture":    true,
		"modify":     true,
		"synthesize": true,
	}

	if mode == "" || !availableModes[mode] {
		log.Error("Can't change mode to \"%d\"", mode)
		return fmt.Errorf("Not a valid mode")
	}

	if this.Cfg.Webserver {
		log.Error("Can't change state when configured as a webserver ")
		return fmt.Errorf("Can't change state when configured as a webserver")
	}
	this.Cfg.SetMode(mode)
	return nil
}

func (hf Hoverfly) GetMiddleware() string {
	return hf.Cfg.Middleware.FullCommand
}

func (hf Hoverfly) GetMiddlewareV2() (string, string, string) {
	script, _ := hf.Cfg.Middleware.GetScript()
	return hf.Cfg.Middleware.Binary, script, hf.Cfg.Middleware.Remote
}

func (hf Hoverfly) SetMiddleware(middleware string) error {
	if middleware == "" {
		hf.Cfg.Middleware.FullCommand = middleware
		return nil
	}
	originalPair := models.RequestResponsePair{
		Request: models.RequestDetails{
			Path:        "/",
			Method:      "GET",
			Destination: "www.test.com",
			Scheme:      "",
			Query:       "",
			Body:        "",
			Headers:     map[string][]string{"test_header": []string{"true"}},
		},
		Response: models.ResponseDetails{
			Status:  200,
			Body:    "ok",
			Headers: map[string][]string{"test_header": []string{"true"}},
		},
	}

	middlewareObject := &Middleware{
		FullCommand: middleware,
	}

	_, err := middlewareObject.Execute(originalPair)
	if err != nil {
		return err
	}

	hf.Cfg.Middleware.FullCommand = middleware
	return nil
}

func (hf *Hoverfly) SetMiddlewareV2(binary, script string) error {
	newMiddleware := Middleware{}

	if binary == "" && script == "" {
		hf.Cfg.Middleware = newMiddleware
		return nil
	} else if binary == "" {
		return fmt.Errorf("Cannot run script with no binary")
	}

	err := newMiddleware.SetBinary(binary)
	if err != nil {
		return err
	}

	err = newMiddleware.SetScript(script)
	if err != nil {
		return nil
	}

	testData := models.RequestResponsePair{
		Request: models.RequestDetails{
			Path:        "/",
			Method:      "GET",
			Destination: "www.test.com",
			Scheme:      "",
			Query:       "",
			Body:        "",
			Headers:     map[string][]string{"test_header": []string{"true"}},
		},
		Response: models.ResponseDetails{
			Status:  200,
			Body:    "ok",
			Headers: map[string][]string{"test_header": []string{"true"}},
		},
	}
	_, err = newMiddleware.Execute(testData)
	if err != nil {
		return err
	}

	hf.Cfg.Middleware = newMiddleware
	return nil
}

func (hf Hoverfly) GetRequestCacheCount() (int, error) {
	return hf.RequestCache.RecordsCount()
}

func (this Hoverfly) GetMetadataCache() cache.Cache {
	return this.MetadataCache
}

func (hf Hoverfly) DeleteRequestCache() error {
	return hf.RequestCache.DeleteData()
}

func (this Hoverfly) GetTemplates() v1.RequestTemplateResponsePairPayload {
	return this.RequestMatcher.TemplateStore.GetPayload()
}

func (this *Hoverfly) ImportTemplates(pairPayload v1.RequestTemplateResponsePairPayload) error {
	return this.RequestMatcher.TemplateStore.ImportPayloads(pairPayload)
}

func (this *Hoverfly) DeleteTemplateCache() {
	this.RequestMatcher.TemplateStore.Wipe()
}

func (hf *Hoverfly) GetResponseDelays() v1.ResponseDelayPayloadView {
	return hf.ResponseDelays.ConvertToResponseDelayPayloadView()
}

func (hf *Hoverfly) SetResponseDelays(payloadView v1.ResponseDelayPayloadView) error {
	err := models.ValidateResponseDelayPayload(payloadView)
	if err != nil {
		return err
	}

	var responseDelays models.ResponseDelayList

	for _, responseDelayView := range payloadView.Data {
		responseDelays = append(responseDelays, models.ResponseDelay{
			UrlPattern: responseDelayView.UrlPattern,
			HttpMethod: responseDelayView.HttpMethod,
			Delay:      responseDelayView.Delay,
		})
	}

	hf.ResponseDelays = &responseDelays
	return nil
}

func (hf *Hoverfly) DeleteResponseDelays() {
	hf.ResponseDelays = &models.ResponseDelayList{}
}

func (hf Hoverfly) GetStats() metrics.Stats {
	return hf.Counter.Flush()
}

func (hf Hoverfly) GetRecords() ([]v1.RequestResponsePairView, error) {
	records, err := hf.RequestCache.GetAllEntries()
	if err != nil {
		return nil, err
	}

	var pairViews []v1.RequestResponsePairView

	for _, v := range records {
		if pair, err := models.NewRequestResponsePairFromBytes(v); err == nil {
			pairView := pair.ConvertToV1RequestResponsePairView()
			pairViews = append(pairViews, *pairView)
		} else {
			log.Error(err)
			return nil, err
		}
	}

	for _, v := range hf.RequestMatcher.TemplateStore {
		pairView := v.ConvertToV1RequestResponsePairView()
		pairViews = append(pairViews, pairView)
	}

	return pairViews, nil

}

func (hf Hoverfly) GetSimulation() (v2.SimulationView, error) {
	records, err := hf.RequestCache.GetAllEntries()
	if err != nil {
		return v2.SimulationView{}, err
	}

	pairViews := make([]v2.RequestResponsePairView, 0)

	for _, v := range records {
		if pair, err := models.NewRequestResponsePairFromBytes(v); err == nil {
			pairView := pair.ConvertToRequestResponsePairView()
			pairViews = append(pairViews, pairView)
		} else {
			log.Error(err)
			return v2.SimulationView{}, err
		}
	}

	for _, v := range hf.RequestMatcher.TemplateStore {
		pairViews = append(pairViews, v.ConvertToRequestResponsePairView())
	}

	responseDelays := hf.ResponseDelays.ConvertToResponseDelayPayloadView()

	return v2.SimulationView{
		MetaView: v2.MetaView{
			HoverflyVersion: "v0.9.2",
			SchemaVersion:   "v1",
			TimeExported:    time.Now().Format(time.RFC3339),
		},
		DataView: v2.DataView{
			RequestResponsePairs: pairViews,
			GlobalActions: v2.GlobalActionsView{
				Delays: responseDelays.Data,
			},
		},
	}, nil
}

func (this *Hoverfly) PutSimulation(simulationView v2.SimulationView) error {
	requestResponsePairViews := make([]interfaces.RequestResponsePair, len(simulationView.RequestResponsePairs))
	for i, v := range simulationView.RequestResponsePairs {
		requestResponsePairViews[i] = v
	}

	err := this.ImportRequestResponsePairViews(requestResponsePairViews)
	if err != nil {
		return err
	}

	err = this.SetResponseDelays(v1.ResponseDelayPayloadView{Data: simulationView.GlobalActions.Delays})
	if err != nil {
		return err
	}

	return nil
}

func (this *Hoverfly) DeleteSimulation() {
	this.DeleteTemplateCache()
	this.DeleteResponseDelays()
	this.DeleteRequestCache()
}
<|endoftext|>


<commit_before>package core

import (
	"strconv"
	"time"

	"github.com/af83/edwig/logger"
	"github.com/af83/edwig/model"
)

type PartnersGuardian struct {
	model.ClockConsumer

	stop        chan struct{}
	referential *Referential
}

func NewPartnersGuardian(referential *Referential) *PartnersGuardian {
	return &PartnersGuardian{referential: referential}
}

func (guardian *PartnersGuardian) Start() {
	logger.Log.Debugf("Start partners guardian")

	guardian.stop = make(chan struct{})
	go guardian.Run()
}

func (guardian *PartnersGuardian) Stop() {
	if guardian.stop != nil {
		close(guardian.stop)
	}
}

func (guardian *PartnersGuardian) Run() {
	for {
		select {
		case <-guardian.stop:
			logger.Log.Debugf("Stop Partners Guardian")
			return
		case <-guardian.Clock().After(30 * time.Second):
			logger.Log.Debugf("Check partners status")
			for _, partner := range guardian.referential.Partners().FindAll() {
				go guardian.routineWork(partner)
			}
		}
	}
}

func (guardian *PartnersGuardian) routineWork(partner *Partner) {
	s := guardian.checkPartnerStatus(partner)
	if !s {
		return
	}

	guardian.checkSubscriptionsTerminatedTime(partner)
}

func (guardian *PartnersGuardian) checkPartnerStatus(partner *Partner) bool {
	defer func() {
		if r := recover(); r != nil {
			logger.Log.Printf("Recovered error %v in checkPartnerStatus for partner %#v", r, partner)
		}
	}()

	partnerStatus, _ := partner.CheckStatus()

	if partnerStatus.OperationnalStatus != partner.PartnerStatus.OperationnalStatus && partnerStatus.OperationnalStatus != OPERATIONNAL_STATUS_UP {
		logger.Log.Debugf("Partner status changed after a CheckStatus: was %v, now is %v", partner.PartnerStatus.OperationnalStatus, partnerStatus.OperationnalStatus)
		guardian.referential.CollectManager().HandlePartnerStatusChange(string(partner.Slug()), false)
	}

	if partnerStatus.OperationnalStatus == OPERATIONNAL_STATUS_UNKNOWN || partnerStatus.OperationnalStatus == OPERATIONNAL_STATUS_DOWN || partnerStatus.ServiceStartedAt != partner.PartnerStatus.ServiceStartedAt {
		partner.PartnerStatus = partnerStatus

		if b, _ := strconv.ParseBool(partner.Setting("subscriptions.persistent")); b {
			partner.Subscriptions().CancelCollectSubscriptions()
			return false
		}

		partner.CancelSubscriptions()

		return false
	}

	partner.PartnerStatus = partnerStatus
	return true
}

func (guardian *PartnersGuardian) checkSubscriptionsTerminatedTime(partner *Partner) {
	if partner.Subscriptions() == nil {
		return
	}

	for _, sub := range partner.Subscriptions().FindAll() {
		for key, value := range sub.ResourcesByObjectIDCopy() {
			if !value.SubscribedUntil.Before(guardian.Clock().Now()) || value.SubscribedAt.IsZero() {
				continue
			}
			sub.DeleteResource(key)
			logger.Log.Printf("Deleting ressource %v from subscription with id %v", key, sub.Id())
		}
		if sub.ResourcesLen() == 0 {
			sub.Delete()
		}
	}
}
<commit_msg>small chenge for log in partner guardian<commit_after>package core

import (
	"strconv"
	"time"

	"github.com/af83/edwig/logger"
	"github.com/af83/edwig/model"
)

type PartnersGuardian struct {
	model.ClockConsumer

	stop        chan struct{}
	referential *Referential
}

func NewPartnersGuardian(referential *Referential) *PartnersGuardian {
	return &PartnersGuardian{referential: referential}
}

func (guardian *PartnersGuardian) Start() {
	logger.Log.Debugf("Start partners guardian")

	guardian.stop = make(chan struct{})
	go guardian.Run()
}

func (guardian *PartnersGuardian) Stop() {
	if guardian.stop != nil {
		close(guardian.stop)
	}
}

func (guardian *PartnersGuardian) Run() {
	for {
		select {
		case <-guardian.stop:
			logger.Log.Debugf("Stop Partners Guardian")
			return
		case <-guardian.Clock().After(30 * time.Second):
			logger.Log.Debugf("Check partners status")
			for _, partner := range guardian.referential.Partners().FindAll() {
				go guardian.routineWork(partner)
			}
		}
	}
}

func (guardian *PartnersGuardian) routineWork(partner *Partner) {
	s := guardian.checkPartnerStatus(partner)
	if !s {
		return
	}

	guardian.checkSubscriptionsTerminatedTime(partner)
}

func (guardian *PartnersGuardian) checkPartnerStatus(partner *Partner) bool {
	defer func() {
		if r := recover(); r != nil {
			logger.Log.Printf("Recovered error %v in checkPartnerStatus for partner %#v", r, partner)
		}
	}()

	partnerStatus, _ := partner.CheckStatus()

	if partnerStatus.OperationnalStatus != partner.PartnerStatus.OperationnalStatus && partnerStatus.OperationnalStatus != OPERATIONNAL_STATUS_UP {
		logger.Log.Debugf("Partner %v status changed after a CheckStatus: was %v, now is %v", partner.Slug(), partner.PartnerStatus.OperationnalStatus, partnerStatus.OperationnalStatus)
		guardian.referential.CollectManager().HandlePartnerStatusChange(string(partner.Slug()), false)
	}

	if partnerStatus.OperationnalStatus == OPERATIONNAL_STATUS_UNKNOWN || partnerStatus.OperationnalStatus == OPERATIONNAL_STATUS_DOWN || partnerStatus.ServiceStartedAt != partner.PartnerStatus.ServiceStartedAt {
		partner.PartnerStatus = partnerStatus

		if b, _ := strconv.ParseBool(partner.Setting("subscriptions.persistent")); b {
			partner.Subscriptions().CancelCollectSubscriptions()
			return false
		}

		partner.CancelSubscriptions()

		return false
	}

	partner.PartnerStatus = partnerStatus
	return true
}

func (guardian *PartnersGuardian) checkSubscriptionsTerminatedTime(partner *Partner) {
	if partner.Subscriptions() == nil {
		return
	}

	for _, sub := range partner.Subscriptions().FindAll() {
		for key, value := range sub.ResourcesByObjectIDCopy() {
			if !value.SubscribedUntil.Before(guardian.Clock().Now()) || value.SubscribedAt.IsZero() {
				continue
			}
			sub.DeleteResource(key)
			logger.Log.Printf("Deleting ressource %v from subscription with id %v", key, sub.Id())
		}
		if sub.ResourcesLen() == 0 {
			sub.Delete()
		}
	}
}
<|endoftext|>


<commit_before>package sentry

import (
	"errors"
	"log"
	"strings"

	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
)

func resourceSentryTeamImport(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	addrID := d.Id()

	log.Printf("[DEBUG] Importing key using ADDR ID %s", addrID)

	parts := strings.Split(addrID, "/")

	if len(parts) != 2 {
		return nil, errors.New("Project import requires an ADDR ID of the following schema org-slug/project-slug")
	}

	d.Set("organization", parts[0])
	d.SetId(parts[1])

	return []*schema.ResourceData{d}, nil
}
<commit_msg>Fix wrong wording in error msg on import team<commit_after>package sentry

import (
	"errors"
	"log"
	"strings"

	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
)

func resourceSentryTeamImport(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	addrID := d.Id()

	log.Printf("[DEBUG] Importing key using ADDR ID %s", addrID)

	parts := strings.Split(addrID, "/")

	if len(parts) != 2 {
		return nil, errors.New("Project import requires an ADDR ID of the following schema org-slug/team-slug")
	}

	d.Set("organization", parts[0])
	d.SetId(parts[1])

	return []*schema.ResourceData{d}, nil
}
<|endoftext|>


<commit_before>package controllers

import (
	"net/http"
	"strconv"

	"github.com/l2x/wolffy/server/config"
	"github.com/l2x/wolffy/server/models"
	"github.com/l2x/wolffy/utils/git"
	"github.com/martini-contrib/render"
)

type Deploy struct{}

func (c Deploy) Push(r render.Render, req *http.Request) {
	res := NewRes()

	pid := req.URL.Query().Get("pid")
	commit := req.URL.Query().Get("commit")
	pidint, err := strconv.Atoi(pid)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	deploy, err := models.DeployModel.Add(pidint, commit)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	project, err := models.ProjectModel.GetOne(pidint)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	repo := git.NewRepository(config.RepoPath, project.Path)
	err = repo.Archive(commit, repo.Path)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	//TODO push code to agent

	//finish
	err = models.DeployModel.UpdateStatus(deploy.Id, 2)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, deploy)
}

func (c Deploy) History(r render.Render, req *http.Request) {
	res := NewRes()

	pid := req.URL.Query().Get("pid")
	pidint, err := strconv.Atoi(pid)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	deploys, err := models.DeployModel.GetAll(pidint)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, deploys)
}
<commit_msg>update<commit_after>package controllers

import (
	"net/http"
	"strconv"

	"github.com/l2x/wolffy/server/config"
	"github.com/l2x/wolffy/server/models"
	"github.com/l2x/wolffy/utils/git"
	"github.com/martini-contrib/render"
)

type Deploy struct{}

func (c Deploy) Push(r render.Render, req *http.Request) {
	res := NewRes()

	pid := req.URL.Query().Get("pid")
	commit := req.URL.Query().Get("commit")
	pidint, err := strconv.Atoi(pid)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	deploy, err := models.DeployModel.Add(pidint, commit)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	project, err := models.ProjectModel.GetOne(pidint)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	repo := git.NewRepository(config.RepoPath, project.Path)
	err = repo.Archive(commit, repo.Path)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	//TODO push code to agent

	//finish
	err = models.DeployModel.UpdateStatus(deploy.Id, 2)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, deploy)
}

func (c Deploy) Get(r render.Render, req *http.Request) {
	res := NewRes()

	id := req.URL.Query().Get("id")
	idint, _ := strconv.Atoi(id)

	deploy, err := models.DeployModel.GetOne(idint)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, deploy)
}

func (c Deploy) History(r render.Render, req *http.Request) {
	res := NewRes()

	pid := req.URL.Query().Get("pid")
	pidint, err := strconv.Atoi(pid)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	deploys, err := models.DeployModel.GetAll(pidint)
	if err = RenderError(r, res, err); err != nil {
		return
	}

	RenderRes(r, res, deploys)
}
<|endoftext|>


<commit_before>package process

import (
	_ "fmt"
	"github.com/whosonfirst/go-whosonfirst-log"
	"github.com/whosonfirst/go-whosonfirst-s3"
	"github.com/whosonfirst/go-whosonfirst-updated"
	"github.com/whosonfirst/go-whosonfirst-updated/queue"
	"github.com/whosonfirst/go-whosonfirst-updated/utils"
	"os"
	_ "os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

type S3Process struct {
	Process
	queue     *queue.Queue
	data_root string
	flushing  bool
	mu        *sync.Mutex
	files     map[string][]string
	s3_bucket string
	s3_prefix string
	logger    *log.WOFLogger
}

func NewS3Process(data_root string, s3_bucket string, s3_prefix string, logger *log.WOFLogger) (*S3Process, error) {

	data_root, err := filepath.Abs(data_root)

	if err != nil {
		return nil, err
	}

	_, err = os.Stat(data_root)

	if os.IsNotExist(err) {
		return nil, err
	}

	q, err := queue.NewQueue()

	if err != nil {
		return nil, err
	}

	files := make(map[string][]string)

	mu := new(sync.Mutex)

	pr := S3Process{
		queue:     q,
		data_root: data_root,
		flushing:  false,
		mu:        mu,
		files:     files,
		s3_bucket: s3_bucket,
		s3_prefix: s3_prefix,
		logger:    logger,
	}

	return &pr, nil
}

func (pr *S3Process) Flush() error {

	pr.mu.Lock()

	if pr.flushing {
		pr.mu.Unlock()
		return nil
	}

	pr.flushing = true
	pr.mu.Unlock()

	for _, repo := range pr.queue.Pending() {
		go pr.ProcessRepo(repo)
	}

	pr.mu.Lock()

	pr.flushing = false
	pr.mu.Unlock()

	return nil
}

func (pr *S3Process) Name() string {
	return "s3"
}

func (pr *S3Process) ProcessTask(task updated.UpdateTask) error {

	repo := task.Repo

	pr.mu.Lock()

	files, ok := pr.files[repo]

	if !ok {
		files = make([]string, 0)
	}

	for _, path := range task.Commits {

		if strings.HasSuffix(path, ".geojson") {
			files = append(files, path)
		}
	}

	pr.files[repo] = files
	pr.mu.Unlock()

	return pr.ProcessRepo(repo)
}

func (pr *S3Process) ProcessRepo(repo string) error {

	if pr.queue.IsProcessing(repo) {
		return pr.queue.Schedule(repo)
	}

	err := pr.queue.Lock(repo)

	if err != nil {
		return err
	}

	if len(pr.files[repo]) > 0 {
		err = pr._process(repo)

		if err != nil {
			return err
		}
	}

	err = pr.queue.Release(repo)

	if err != nil {
		return err
	}

	return nil
}

func (pr *S3Process) _process(repo string) error {

	t1 := time.Now()

	defer func() {
		t2 := time.Since(t1)
		pr.logger.Status("Time to process (%s) %s: %v", pr.Name(), repo, t2)
	}()

	root := filepath.Join(pr.data_root, repo)

	_, err := os.Stat(root)

	if os.IsNotExist(err) {
		pr.logger.Error("Can't find repo", root)
		return err
	}

	/* sudo wrap all of this in a single function somewhere... */

	pr.mu.Lock()
	files := pr.files[repo]

	delete(pr.files, repo)
	pr.mu.Unlock()

	tmpfile, err := utils.FilesToFileList(files, root)

	if err != nil {

		pr.mu.Lock()

		_, ok := pr.files[repo]

		if ok {

			for _, path := range files {
				pr.files[repo] = append(pr.files[repo], path)
			}

		} else {
			pr.files[repo] = files
		}

		pr.mu.Unlock()

		return err
	}

	/* end of sudo wrap all of this in a single function somewhere... */

	debug := false
	procs := 10

	pr.logger.Debug("Process (S3) file list %s", tmpfile.Name())

	sink := s3.WOFSync(pr.s3_bucket, pr.s3_prefix, procs, debug, pr.logger)

	err = sink.SyncFileList(tmpfile.Name(), root)

	if err != nil {
		pr.logger.Error("Failed to process (S3) file list because %s (%s)", err, tmpfile.Name())
		return err
	}

	pr.logger.Debug("Successfully processed (S3) file list %s", tmpfile.Name())
	os.Remove(tmpfile.Name())

	return nil
}
<commit_msg>use go-wof-uri to determine is a file is a WOF file; ensure file is present on disk (for S3 transfers)<commit_after>package process

import (
	"fmt"
	"github.com/whosonfirst/go-whosonfirst-log"
	"github.com/whosonfirst/go-whosonfirst-s3"
	"github.com/whosonfirst/go-whosonfirst-updated"
	"github.com/whosonfirst/go-whosonfirst-updated/queue"
	"github.com/whosonfirst/go-whosonfirst-updated/utils"
	"github.com/whosonfirst/go-whosonfirst-uri"
	"os"
	_ "os/exec"
	"path/filepath"
	"sync"
	"time"
)

type S3Process struct {
	Process
	queue     *queue.Queue
	data_root string
	flushing  bool
	mu        *sync.Mutex
	files     map[string][]string
	s3_bucket string
	s3_prefix string
	logger    *log.WOFLogger
}

func NewS3Process(data_root string, s3_bucket string, s3_prefix string, logger *log.WOFLogger) (*S3Process, error) {

	data_root, err := filepath.Abs(data_root)

	if err != nil {
		return nil, err
	}

	_, err = os.Stat(data_root)

	if os.IsNotExist(err) {
		return nil, err
	}

	q, err := queue.NewQueue()

	if err != nil {
		return nil, err
	}

	files := make(map[string][]string)

	mu := new(sync.Mutex)

	pr := S3Process{
		queue:     q,
		data_root: data_root,
		flushing:  false,
		mu:        mu,
		files:     files,
		s3_bucket: s3_bucket,
		s3_prefix: s3_prefix,
		logger:    logger,
	}

	return &pr, nil
}

func (pr *S3Process) Flush() error {

	pr.mu.Lock()

	if pr.flushing {
		pr.mu.Unlock()
		return nil
	}

	pr.flushing = true
	pr.mu.Unlock()

	for _, repo := range pr.queue.Pending() {
		go pr.ProcessRepo(repo)
	}

	pr.mu.Lock()

	pr.flushing = false
	pr.mu.Unlock()

	return nil
}

func (pr *S3Process) Name() string {
	return "s3"
}

func (pr *S3Process) ProcessTask(task updated.UpdateTask) error {

	repo := task.Repo

	pr.mu.Lock()

	files, ok := pr.files[repo]

	if !ok {
		files = make([]string, 0)
	}

	for _, path := range task.Commits {

		repo_path := filepath.Join(pr.data_root, repo)
		abs_path := filepath.Join(repo_path, path)

		wof, err := uri.IsWOFFile(abs_path)

		if err != nil {
			pr.logger.Warning(fmt.Sprintf("Failed to determine if %s is a WOF file, because %s", abs_path, err))
			continue
		}

		if !wof {
			continue
		}

		_, err = os.Stat(abs_path)

		if os.IsNotExist(err) {
			pr.logger.Warning(fmt.Sprintf("Failed to clone %s, because it doesn't exist", abs_path))
			continue
		}

		files = append(files, path)
	}

	pr.files[repo] = files
	pr.mu.Unlock()

	return pr.ProcessRepo(repo)
}

func (pr *S3Process) ProcessRepo(repo string) error {

	if pr.queue.IsProcessing(repo) {
		return pr.queue.Schedule(repo)
	}

	err := pr.queue.Lock(repo)

	if err != nil {
		return err
	}

	if len(pr.files[repo]) > 0 {
		err = pr._process(repo)

		if err != nil {
			return err
		}
	}

	err = pr.queue.Release(repo)

	if err != nil {
		return err
	}

	return nil
}

func (pr *S3Process) _process(repo string) error {

	t1 := time.Now()

	defer func() {
		t2 := time.Since(t1)
		pr.logger.Status("Time to process (%s) %s: %v", pr.Name(), repo, t2)
	}()

	root := filepath.Join(pr.data_root, repo)

	_, err := os.Stat(root)

	if os.IsNotExist(err) {
		pr.logger.Error("Can't find repo", root)
		return err
	}

	/* sudo wrap all of this in a single function somewhere... */

	pr.mu.Lock()
	files := pr.files[repo]

	delete(pr.files, repo)
	pr.mu.Unlock()

	tmpfile, err := utils.FilesToFileList(files, root)

	if err != nil {

		pr.mu.Lock()

		_, ok := pr.files[repo]

		if ok {

			for _, path := range files {
				pr.files[repo] = append(pr.files[repo], path)
			}

		} else {
			pr.files[repo] = files
		}

		pr.mu.Unlock()

		return err
	}

	/* end of sudo wrap all of this in a single function somewhere... */

	debug := false
	procs := 10

	pr.logger.Debug("Process (S3) file list %s", tmpfile.Name())

	sink := s3.WOFSync(pr.s3_bucket, pr.s3_prefix, procs, debug, pr.logger)

	err = sink.SyncFileList(tmpfile.Name(), root)

	if err != nil {
		pr.logger.Error("Failed to process (S3) file list because %s (%s)", err, tmpfile.Name())
		return err
	}

	pr.logger.Debug("Successfully processed (S3) file list %s", tmpfile.Name())
	os.Remove(tmpfile.Name())

	return nil
}
<|endoftext|>