<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	json2 "encoding/json"
	"flag"

	log "github.com/Sirupsen/logrus"
	"github.com/uber-common/bark"
	"github.com/uber/ringpop-go"
	"github.com/uber/ringpop-go/swim"
	"github.com/uber/tchannel-go"
	"github.com/uber/tchannel-go/json"
	"golang.org/x/net/context"
)

var (
	hostport = flag.String("listen", "127.0.0.1:3000", "hostport to start ringpop on")
	hostfile = flag.String("hosts", "./hosts.json", "path to hosts file")
)

type worker struct {
	ringpop *ringpop.Ringpop
	channel *tchannel.Channel
	logger  *log.Logger
}

// Ping is a request
type Ping struct {
	Key string `json:"key"`
}

// Bytes returns the byets for a ping
func (p Ping) Bytes() []byte {
	data, _ := json2.Marshal(p)
	return data
}

// Pong is a ping response
type Pong struct {
	Message string `json:"message"`
	From    string `json:"from"`
}

func (w *worker) RegisterPong() error {
	hmap := map[string]interface{}{"/ping": w.PingHandler}

	return json.Register(w.channel, hmap, func(ctx context.Context, err error) {
		w.logger.Debug("error occured: %v", err)
	})
}

func (w *worker) PingHandler(ctx json.Context, ping *Ping) (*Pong, error) {
	var pong Pong
	var res []byte

	handle, err := w.ringpop.HandleOrForward(ping.Key, ping.Bytes(), &res, "ping", "/ping", tchannel.JSON, nil)
	if handle {
		return &Pong{"Hello, world!", w.ringpop.WhoAmI()}, nil
	}

	if err := json2.Unmarshal(res, &pong); err != nil {
		return nil, err
	}

	// else request was forwarded
	return &pong, err
}

func main() {
	flag.Parse()

	ch, err := tchannel.NewChannel("ping", nil)
	if err != nil {
		log.Fatalf("channel did not create successfully: %v", err)
	}

	logger := log.StandardLogger()

	rp, err := ringpop.New("ping-app",
		ringpop.Channel(ch),
		ringpop.Identity(*hostport),
		ringpop.Logger(bark.NewLoggerFromLogrus(logger)),
	)
	if err != nil {
		log.Fatalf("Unable to create Ringpop: %v", err)
	}

	worker := &worker{
		channel: ch,
		ringpop: rp,
		logger:  logger,
	}

	if err := worker.RegisterPong(); err != nil {
		log.Fatalf("could not register pong handler: %v", err)
	}

	if err := worker.channel.ListenAndServe(*hostport); err != nil {
		log.Fatalf("could not listen on given hostport: %v", err)
	}

	opts := new(swim.BootstrapOptions)
	opts.File = *hostfile

	if _, err := worker.ringpop.Bootstrap(opts); err != nil {
		log.Fatalf("ringpop bootstrap failed: %v", err)
	}

	select {}
}
<commit_msg>Fix bug in pong example<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	json2 "encoding/json"
	"flag"

	log "github.com/Sirupsen/logrus"
	"github.com/uber-common/bark"
	"github.com/uber/ringpop-go"
	"github.com/uber/ringpop-go/swim"
	"github.com/uber/tchannel-go"
	"github.com/uber/tchannel-go/json"
	"golang.org/x/net/context"
)

var (
	hostport = flag.String("listen", "127.0.0.1:3000", "hostport to start ringpop on")
	hostfile = flag.String("hosts", "./hosts.json", "path to hosts file")
)

type worker struct {
	ringpop *ringpop.Ringpop
	channel *tchannel.Channel
	logger  *log.Logger
}

// Ping is a request
type Ping struct {
	Key string `json:"key"`
}

// Bytes returns the byets for a ping
func (p Ping) Bytes() []byte {
	data, _ := json2.Marshal(p)
	return data
}

// Pong is a ping response
type Pong struct {
	Message string `json:"message"`
	From    string `json:"from"`
}

func (w *worker) RegisterPong() error {
	hmap := map[string]interface{}{"/ping": w.PingHandler}

	return json.Register(w.channel, hmap, func(ctx context.Context, err error) {
		w.logger.Debug("error occured: %v", err)
	})
}

func (w *worker) PingHandler(ctx json.Context, ping *Ping) (*Pong, error) {
	var pong Pong
	var res []byte

	handle, err := w.ringpop.HandleOrForward(ping.Key, ping.Bytes(), &res, "ping", "/ping", tchannel.JSON, nil)
	if handle {
		identity, err := w.ringpop.WhoAmI()
		if err != nil {
			return nil, err
		}
		return &Pong{"Hello, world!", identity}, nil
	}

	if err := json2.Unmarshal(res, &pong); err != nil {
		return nil, err
	}

	// else request was forwarded
	return &pong, err
}

func main() {
	flag.Parse()

	ch, err := tchannel.NewChannel("ping", nil)
	if err != nil {
		log.Fatalf("channel did not create successfully: %v", err)
	}

	logger := log.StandardLogger()

	rp, err := ringpop.New("ping-app",
		ringpop.Channel(ch),
		ringpop.Identity(*hostport),
		ringpop.Logger(bark.NewLoggerFromLogrus(logger)),
	)
	if err != nil {
		log.Fatalf("Unable to create Ringpop: %v", err)
	}

	worker := &worker{
		channel: ch,
		ringpop: rp,
		logger:  logger,
	}

	if err := worker.RegisterPong(); err != nil {
		log.Fatalf("could not register pong handler: %v", err)
	}

	if err := worker.channel.ListenAndServe(*hostport); err != nil {
		log.Fatalf("could not listen on given hostport: %v", err)
	}

	opts := new(swim.BootstrapOptions)
	opts.File = *hostfile

	if _, err := worker.ringpop.Bootstrap(opts); err != nil {
		log.Fatalf("ringpop bootstrap failed: %v", err)
	}

	select {}
}
<|endoftext|>


<commit_before>package config

import (
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestParseServerConfig(t *testing.T) {
	ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/delegated/server.conf")
	ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/fpaxos/server.conf")
	ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/simple/server.conf")
}

// TestParseServerConfig calls ParseServerConfig for the two example configuration files
func TestParseSingleServerConfig(t *testing.T) {
	conf := ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/simple/server.conf")

	assert.Equal(t, 100000, conf.Performance.Length)
	assert.Equal(t, 0, conf.Performance.BatchInterval)
	assert.Equal(t, 10, conf.Performance.MaxBatch)
	assert.Equal(t, 0, conf.Algorithm.DelegateReplication)
	assert.Equal(t, 1, conf.Performance.WindowSize)
	assert.Equal(t, 1000, conf.Performance.SnapshotInterval)
	assert.Equal(t, "strict majority", conf.Algorithm.QuorumSystem)
	assert.Equal(t, false, conf.Algorithm.IndexExclusivity)
	assert.Equal(t, "kv-store", conf.Application.Name)

	unsafe := conf.Unsafe
	assert.False(t, unsafe.DumpPersistentStorage)
	assert.Equal(t, "fsync", unsafe.PersistenceMode)

}
<commit_msg>splitting server config file into subsections<commit_after>package config

import (
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestParseServerConfig(t *testing.T) {
	ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/delegated/server.conf")
	ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/fpaxos/server.conf")
	ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/simple/server.conf")
}

// TestParseServerConfig calls ParseServerConfig for the two example configuration files
func TestParseSingleServerConfig(t *testing.T) {
	conf := ParseServerConfig(os.Getenv("GOPATH") + "/src/github.com/heidi-ann/ios/configfiles/simple/server.conf")

	assert.Equal(t, 100000, conf.Performance.Length)
	assert.Equal(t, 0, conf.Performance.BatchInterval)
	assert.Equal(t, 10, conf.Performance.MaxBatch)
	assert.Equal(t, 0, conf.Algorithm.DelegateReplication)
	assert.Equal(t, 1, conf.Performance.WindowSize)
	assert.Equal(t, 1000, conf.Performance.SnapshotInterval)
	assert.Equal(t, "strict majority", conf.Algorithm.QuorumSystem)
	assert.Equal(t, false, conf.Algorithm.IndexExclusivity)
	assert.Equal(t, "kv-store", conf.Application.Name)

	assert.False(t, conf.Unsafe.DumpPersistentStorage)
	assert.Equal(t, "fsync", conf.Unsafe.PersistenceMode)

}
<|endoftext|>


<commit_before>package extractor

import (
	"fmt"
	"golang.org/x/mod/modfile"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/github/codeql-go/extractor/dbscheme"
	"github.com/github/codeql-go/extractor/srcarchive"
	"github.com/github/codeql-go/extractor/trap"
)

func extractGoMod(path string) error {
	if normPath, err := filepath.EvalSymlinks(path); err == nil {
		path = normPath
	}

	tw, err := trap.NewWriter(path, nil)
	if err != nil {
		return err
	}
	defer tw.Close()

	err = srcarchive.Add(path)
	if err != nil {
		return err
	}

	extractFileInfo(tw, path)

	file, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("failed to open go.mod file %s: %s", path, err.Error())
	}
	data, err := ioutil.ReadAll(file)
	if err != nil {
		return fmt.Errorf("failed to read go.mod file %s: %s", path, err.Error())
	}

	modfile, err := modfile.Parse(path, data, nil)
	if err != nil {
		return fmt.Errorf("failed to parse go.mod file %s: %s", path, err.Error())
	}

	extractGoModFile(tw, modfile.Syntax)

	return nil
}

type commentGroupIdxAllocator struct {
	nextIdx int
}

func (cgIdxAlloc *commentGroupIdxAllocator) nextCgIdx() int {
	ret := cgIdxAlloc.nextIdx
	cgIdxAlloc.nextIdx++
	return ret
}

func extractGoModFile(tw *trap.Writer, file *modfile.FileSyntax) {
	cgIdxAlloc := commentGroupIdxAllocator{0}

	for idx, stmt := range file.Stmt {
		extractGoModExpr(tw, stmt, tw.Labeler.FileLabel(), idx, &cgIdxAlloc)
	}

	extractGoModComments(tw, file, tw.Labeler.FileLabel(), &cgIdxAlloc)
}

func extractGoModExpr(tw *trap.Writer, expr modfile.Expr, parent trap.Label, idx int, cgIdxAlloc *commentGroupIdxAllocator) {
	lbl := tw.Labeler.LocalID(expr)

	var kind int
	switch expr := expr.(type) {
	case *modfile.CommentBlock:
		kind = dbscheme.ModCommentBlockType.Index()
	case *modfile.LParen:
		kind = dbscheme.ModLParenType.Index()
	case *modfile.RParen:
		kind = dbscheme.ModRParenType.Index()
	case *modfile.Line:
		kind = dbscheme.ModLineType.Index()
		for idx, tok := range expr.Token {
			dbscheme.ModTokensTable.Emit(tw, tok, lbl, idx)
		}
	case *modfile.LineBlock:
		kind = dbscheme.ModLineBlockType.Index()
		for idx, tok := range expr.Token {
			dbscheme.ModTokensTable.Emit(tw, tok, lbl, idx)
		}
		extractGoModExpr(tw, &expr.LParen, lbl, 0, cgIdxAlloc)
		for idx, line := range expr.Line {
			extractGoModExpr(tw, line, lbl, idx+1, cgIdxAlloc)
		}
		extractGoModExpr(tw, &expr.RParen, lbl, len(expr.Line)+1, cgIdxAlloc)
	default:
		log.Fatalf("unknown go.mod expression of type %T", expr)
	}

	dbscheme.ModExprsTable.Emit(tw, lbl, kind, parent, idx)

	extractGoModComments(tw, expr, lbl, cgIdxAlloc)

	start, end := expr.Span()
	extractLocation(tw, lbl, start.Line, start.LineRune, end.Line, end.LineRune)
}

type GoModExprCommentWrapper struct {
	expr modfile.Expr
}

func minInt(a int, b int) int {
	if a < b {
		return a
	}
	return b
}

func maxInt(a int, b int) int {
	if a > b {
		return a
	}
	return b
}

func lexMin(a1 int, a2 int, b1 int, b2 int) (int, int) {
	if a1 < b1 {
		return a1, a2
	} else if a1 > b1 {
		return b1, b2
	} else {
		return a1, minInt(b1, b2)
	}
}

func lexMax(a1 int, a2 int, b1 int, b2 int) (int, int) {
	if a1 < b1 {
		return b1, b2
	} else if a1 > b1 {
		return a1, a2
	} else {
		return a1, maxInt(b1, b2)
	}
}

func extractGoModComments(tw *trap.Writer, expr modfile.Expr, exprlbl trap.Label, cgIdxAlloc *commentGroupIdxAllocator) {
	comments := expr.Comment()

	if len(comments.Before) == 0 && len(comments.Suffix) == 0 && len(comments.After) == 0 {
		return
	}

	// extract a pseudo `@commentgroup` for each expr that contains their associated comments
	grouplbl := tw.Labeler.LocalID(GoModExprCommentWrapper{expr})
	dbscheme.CommentGroupsTable.Emit(tw, grouplbl, tw.Labeler.FileLabel(), cgIdxAlloc.nextCgIdx())
	dbscheme.DocCommentsTable.Emit(tw, exprlbl, grouplbl)

	var allComments []modfile.Comment
	allComments = append(allComments, comments.Before...)
	allComments = append(allComments, comments.Suffix...)
	allComments = append(allComments, comments.After...)

	var startLine, startCol, endLine, endCol int = 0, 0, 0, 0
	var first bool = true
	idx := 0
	for _, comment := range allComments {
		commentToken := strings.TrimSuffix(comment.Token, "\r")
		extractGoModComment(tw, comment, commentToken, grouplbl, idx)
		idx++
		commentEndCol := comment.Start.LineRune + (len(commentToken) - 1)
		if first {
			startLine, startCol, endLine, endCol = comment.Start.Line, comment.Start.LineRune, comment.Start.Line, commentEndCol
			first = false
		} else {
			startLine, startCol = lexMin(comment.Start.Line, comment.Start.LineRune, startLine, startCol)
			endLine, endCol = lexMax(comment.Start.Line, commentEndCol, endLine, endCol)
		}
	}

	extractLocation(tw, grouplbl, startLine, startCol, endLine, endCol)
}

func extractGoModComment(tw *trap.Writer, comment modfile.Comment, commentToken string, grouplbl trap.Label, idx int) {
	lbl := tw.Labeler.LocalID(comment)
	dbscheme.CommentsTable.Emit(tw, lbl, dbscheme.SlashSlashComment.Index(), grouplbl, idx, commentToken)

	extractLocation(tw, lbl, comment.Start.Line, comment.Start.LineRune, comment.Start.Line, comment.Start.LineRune+(len(commentToken)-1))
}
<commit_msg>Go.mod comments: trim newlines<commit_after>package extractor

import (
	"fmt"
	"golang.org/x/mod/modfile"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/github/codeql-go/extractor/dbscheme"
	"github.com/github/codeql-go/extractor/srcarchive"
	"github.com/github/codeql-go/extractor/trap"
)

func extractGoMod(path string) error {
	if normPath, err := filepath.EvalSymlinks(path); err == nil {
		path = normPath
	}

	tw, err := trap.NewWriter(path, nil)
	if err != nil {
		return err
	}
	defer tw.Close()

	err = srcarchive.Add(path)
	if err != nil {
		return err
	}

	extractFileInfo(tw, path)

	file, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("failed to open go.mod file %s: %s", path, err.Error())
	}
	data, err := ioutil.ReadAll(file)
	if err != nil {
		return fmt.Errorf("failed to read go.mod file %s: %s", path, err.Error())
	}

	modfile, err := modfile.Parse(path, data, nil)
	if err != nil {
		return fmt.Errorf("failed to parse go.mod file %s: %s", path, err.Error())
	}

	extractGoModFile(tw, modfile.Syntax)

	return nil
}

type commentGroupIdxAllocator struct {
	nextIdx int
}

func (cgIdxAlloc *commentGroupIdxAllocator) nextCgIdx() int {
	ret := cgIdxAlloc.nextIdx
	cgIdxAlloc.nextIdx++
	return ret
}

func extractGoModFile(tw *trap.Writer, file *modfile.FileSyntax) {
	cgIdxAlloc := commentGroupIdxAllocator{0}

	for idx, stmt := range file.Stmt {
		extractGoModExpr(tw, stmt, tw.Labeler.FileLabel(), idx, &cgIdxAlloc)
	}

	extractGoModComments(tw, file, tw.Labeler.FileLabel(), &cgIdxAlloc)
}

func extractGoModExpr(tw *trap.Writer, expr modfile.Expr, parent trap.Label, idx int, cgIdxAlloc *commentGroupIdxAllocator) {
	lbl := tw.Labeler.LocalID(expr)

	var kind int
	switch expr := expr.(type) {
	case *modfile.CommentBlock:
		kind = dbscheme.ModCommentBlockType.Index()
	case *modfile.LParen:
		kind = dbscheme.ModLParenType.Index()
	case *modfile.RParen:
		kind = dbscheme.ModRParenType.Index()
	case *modfile.Line:
		kind = dbscheme.ModLineType.Index()
		for idx, tok := range expr.Token {
			dbscheme.ModTokensTable.Emit(tw, tok, lbl, idx)
		}
	case *modfile.LineBlock:
		kind = dbscheme.ModLineBlockType.Index()
		for idx, tok := range expr.Token {
			dbscheme.ModTokensTable.Emit(tw, tok, lbl, idx)
		}
		extractGoModExpr(tw, &expr.LParen, lbl, 0, cgIdxAlloc)
		for idx, line := range expr.Line {
			extractGoModExpr(tw, line, lbl, idx+1, cgIdxAlloc)
		}
		extractGoModExpr(tw, &expr.RParen, lbl, len(expr.Line)+1, cgIdxAlloc)
	default:
		log.Fatalf("unknown go.mod expression of type %T", expr)
	}

	dbscheme.ModExprsTable.Emit(tw, lbl, kind, parent, idx)

	extractGoModComments(tw, expr, lbl, cgIdxAlloc)

	start, end := expr.Span()
	extractLocation(tw, lbl, start.Line, start.LineRune, end.Line, end.LineRune)
}

type GoModExprCommentWrapper struct {
	expr modfile.Expr
}

func minInt(a int, b int) int {
	if a < b {
		return a
	}
	return b
}

func maxInt(a int, b int) int {
	if a > b {
		return a
	}
	return b
}

func lexMin(a1 int, a2 int, b1 int, b2 int) (int, int) {
	if a1 < b1 {
		return a1, a2
	} else if a1 > b1 {
		return b1, b2
	} else {
		return a1, minInt(b1, b2)
	}
}

func lexMax(a1 int, a2 int, b1 int, b2 int) (int, int) {
	if a1 < b1 {
		return b1, b2
	} else if a1 > b1 {
		return a1, a2
	} else {
		return a1, maxInt(b1, b2)
	}
}

func extractGoModComments(tw *trap.Writer, expr modfile.Expr, exprlbl trap.Label, cgIdxAlloc *commentGroupIdxAllocator) {
	comments := expr.Comment()

	if len(comments.Before) == 0 && len(comments.Suffix) == 0 && len(comments.After) == 0 {
		return
	}

	// extract a pseudo `@commentgroup` for each expr that contains their associated comments
	grouplbl := tw.Labeler.LocalID(GoModExprCommentWrapper{expr})
	dbscheme.CommentGroupsTable.Emit(tw, grouplbl, tw.Labeler.FileLabel(), cgIdxAlloc.nextCgIdx())
	dbscheme.DocCommentsTable.Emit(tw, exprlbl, grouplbl)

	var allComments []modfile.Comment
	allComments = append(allComments, comments.Before...)
	allComments = append(allComments, comments.Suffix...)
	allComments = append(allComments, comments.After...)

	var startLine, startCol, endLine, endCol int = 0, 0, 0, 0
	var first bool = true
	idx := 0
	for _, comment := range allComments {
		commentToken := strings.TrimSuffix(strings.TrimSuffix(comment.Token, "\n"), "\r")
		extractGoModComment(tw, comment, commentToken, grouplbl, idx)
		idx++
		commentEndCol := comment.Start.LineRune + (len(commentToken) - 1)
		if first {
			startLine, startCol, endLine, endCol = comment.Start.Line, comment.Start.LineRune, comment.Start.Line, commentEndCol
			first = false
		} else {
			startLine, startCol = lexMin(comment.Start.Line, comment.Start.LineRune, startLine, startCol)
			endLine, endCol = lexMax(comment.Start.Line, commentEndCol, endLine, endCol)
		}
	}

	extractLocation(tw, grouplbl, startLine, startCol, endLine, endCol)
}

func extractGoModComment(tw *trap.Writer, comment modfile.Comment, commentToken string, grouplbl trap.Label, idx int) {
	lbl := tw.Labeler.LocalID(comment)
	dbscheme.CommentsTable.Emit(tw, lbl, dbscheme.SlashSlashComment.Index(), grouplbl, idx, commentToken)

	extractLocation(tw, lbl, comment.Start.Line, comment.Start.LineRune, comment.Start.Line, comment.Start.LineRune+(len(commentToken)-1))
}
<|endoftext|>


<commit_before>// Copyright 2017 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2eutil

import (
	"bytes"
	"fmt"
	"testing"
	"time"

	"github.com/coreos/etcd-operator/pkg/spec"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/coreos/etcd-operator/pkg/util/retryutil"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
)

type acceptFunc func(*v1.Pod) bool

func WaitUntilSizeReached(t *testing.T, kubeClient kubernetes.Interface, size int, timeout time.Duration, cl *spec.Cluster) ([]string, error) {
	return waitSizeReachedWithAccept(t, kubeClient, size, timeout, cl)
}

func WaitSizeAndVersionReached(t *testing.T, kubeClient kubernetes.Interface, version string, size int, timeout time.Duration, cl *spec.Cluster) error {
	_, err := waitSizeReachedWithAccept(t, kubeClient, size, timeout, cl, func(pod *v1.Pod) bool {
		return k8sutil.GetEtcdVersion(pod) == version
	})
	return err
}

func waitSizeReachedWithAccept(t *testing.T, kubeClient kubernetes.Interface, size int, timeout time.Duration, cl *spec.Cluster, accepts ...acceptFunc) ([]string, error) {
	var names []string
	err := retryutil.Retry(10*time.Second, int(timeout/(10*time.Second)), func() (done bool, err error) {
		podList, err := kubeClient.Core().Pods(cl.Metadata.Namespace).List(k8sutil.ClusterListOpt(cl.Metadata.Name))
		if err != nil {
			return false, err
		}
		names = nil
		var nodeNames []string
		for i := range podList.Items {
			pod := &podList.Items[i]
			if pod.Status.Phase != v1.PodRunning {
				continue
			}
			accepted := true
			for _, acceptPod := range accepts {
				if !acceptPod(pod) {
					accepted = false
					break
				}
			}
			if !accepted {
				continue
			}
			names = append(names, pod.Name)
			nodeNames = append(nodeNames, pod.Spec.NodeName)
		}
		LogfWithTimestamp(t, "waiting size (%d), etcd pods: names (%v), nodes (%v)", size, names, nodeNames)
		// TODO: Change this to check for ready members and not just cluster pods
		if len(names) != size {
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return nil, err
	}
	return names, nil
}

func waitResourcesDeleted(t *testing.T, kubeClient kubernetes.Interface, cl *spec.Cluster) error {
	err := retryutil.Retry(5*time.Second, 5, func() (done bool, err error) {
		list, err := kubeClient.CoreV1().Pods(cl.Metadata.Namespace).List(k8sutil.ClusterListOpt(cl.Metadata.Name))
		if err != nil {
			return false, err
		}
		if len(list.Items) > 0 {
			p := list.Items[0]
			LogfWithTimestamp(t, "waiting pod (%s) to be deleted.", p.Name)

			buf := bytes.NewBuffer(nil)
			buf.WriteString("init container status:\n")
			printContainerStatus(buf, p.Status.InitContainerStatuses)
			buf.WriteString("container status:\n")
			printContainerStatus(buf, p.Status.ContainerStatuses)
			t.Logf("pod (%s) status.phase is (%s): %v", p.Name, p.Status.Phase, buf.String())

			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return fmt.Errorf("fail to wait pods deleted: %v", err)
	}

	err = retryutil.Retry(5*time.Second, 5, func() (done bool, err error) {
		list, err := kubeClient.CoreV1().Services(cl.Metadata.Namespace).List(k8sutil.ClusterListOpt(cl.Metadata.Name))
		if err != nil {
			return false, err
		}
		if len(list.Items) > 0 {
			LogfWithTimestamp(t, "waiting service (%s) to be deleted", list.Items[0].Name)
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return fmt.Errorf("fail to wait services deleted: %v", err)
	}
	return nil
}

func waitBackupDeleted(kubeClient kubernetes.Interface, cl *spec.Cluster, storageCheckerOptions *StorageCheckerOptions) error {
	err := retryutil.Retry(5*time.Second, 5, func() (bool, error) {
		_, err := kubeClient.AppsV1beta1().Deployments(cl.Metadata.Namespace).Get(k8sutil.BackupSidecarName(cl.Metadata.Name), metav1.GetOptions{})
		if err == nil {
			return false, nil
		}
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		return false, err
	})
	if err != nil {
		return fmt.Errorf("failed to wait backup Deployment deleted: %v", err)
	}
	err = retryutil.Retry(5*time.Second, 2, func() (done bool, err error) {
		ls := labels.SelectorFromSet(map[string]string{
			"app":          k8sutil.BackupPodSelectorAppField,
			"etcd_cluster": cl.Metadata.Name,
		}).String()
		pl, err := kubeClient.CoreV1().Pods(cl.Metadata.Namespace).List(metav1.ListOptions{
			LabelSelector: ls,
		})
		if err != nil {
			return false, err
		}
		if len(pl.Items) == 0 {
			return true, nil
		}
		if pl.Items[0].DeletionTimestamp != nil {
			return true, nil
		}
		return false, nil
	})
	if err != nil {
		return fmt.Errorf("failed to wait backup pod terminated: %v", err)
	}
	// The rest is to track backup storage, e.g. PV or S3 "dir" deleted.
	// If CleanupBackupsOnClusterDelete=false, we don't delete them and thus don't check them.
	if !cl.Spec.Backup.CleanupBackupsOnClusterDelete {
		return nil
	}
	err = retryutil.Retry(5*time.Second, 5, func() (done bool, err error) {
		switch cl.Spec.Backup.StorageType {
		case spec.BackupStorageTypePersistentVolume, spec.BackupStorageTypeDefault:
			pl, err := kubeClient.CoreV1().PersistentVolumeClaims(cl.Metadata.Namespace).List(k8sutil.ClusterListOpt(cl.Metadata.Name))
			if err != nil {
				return false, err
			}
			if len(pl.Items) > 0 {
				return false, nil
			}
		case spec.BackupStorageTypeS3:
			resp, err := storageCheckerOptions.S3Cli.ListObjects(&s3.ListObjectsInput{
				Bucket: aws.String(storageCheckerOptions.S3Bucket),
				Prefix: aws.String(cl.Metadata.Name + "/"),
			})
			if err != nil {
				return false, err
			}
			if len(resp.Contents) > 0 {
				return false, nil
			}
		}
		return true, nil
	})
	if err != nil {
		return fmt.Errorf("failed to wait storage (%s) to be deleted: %v", cl.Spec.Backup.StorageType, err)
	}
	return nil
}
<commit_msg>e2e: fix wait s3 delete check path<commit_after>// Copyright 2017 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2eutil

import (
	"bytes"
	"fmt"
	"path"
	"testing"
	"time"

	"github.com/coreos/etcd-operator/pkg/spec"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/coreos/etcd-operator/pkg/util/retryutil"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
)

type acceptFunc func(*v1.Pod) bool

func WaitUntilSizeReached(t *testing.T, kubeClient kubernetes.Interface, size int, timeout time.Duration, cl *spec.Cluster) ([]string, error) {
	return waitSizeReachedWithAccept(t, kubeClient, size, timeout, cl)
}

func WaitSizeAndVersionReached(t *testing.T, kubeClient kubernetes.Interface, version string, size int, timeout time.Duration, cl *spec.Cluster) error {
	_, err := waitSizeReachedWithAccept(t, kubeClient, size, timeout, cl, func(pod *v1.Pod) bool {
		return k8sutil.GetEtcdVersion(pod) == version
	})
	return err
}

func waitSizeReachedWithAccept(t *testing.T, kubeClient kubernetes.Interface, size int, timeout time.Duration, cl *spec.Cluster, accepts ...acceptFunc) ([]string, error) {
	var names []string
	err := retryutil.Retry(10*time.Second, int(timeout/(10*time.Second)), func() (done bool, err error) {
		podList, err := kubeClient.Core().Pods(cl.Metadata.Namespace).List(k8sutil.ClusterListOpt(cl.Metadata.Name))
		if err != nil {
			return false, err
		}
		names = nil
		var nodeNames []string
		for i := range podList.Items {
			pod := &podList.Items[i]
			if pod.Status.Phase != v1.PodRunning {
				continue
			}
			accepted := true
			for _, acceptPod := range accepts {
				if !acceptPod(pod) {
					accepted = false
					break
				}
			}
			if !accepted {
				continue
			}
			names = append(names, pod.Name)
			nodeNames = append(nodeNames, pod.Spec.NodeName)
		}
		LogfWithTimestamp(t, "waiting size (%d), etcd pods: names (%v), nodes (%v)", size, names, nodeNames)
		// TODO: Change this to check for ready members and not just cluster pods
		if len(names) != size {
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return nil, err
	}
	return names, nil
}

func waitResourcesDeleted(t *testing.T, kubeClient kubernetes.Interface, cl *spec.Cluster) error {
	err := retryutil.Retry(5*time.Second, 5, func() (done bool, err error) {
		list, err := kubeClient.CoreV1().Pods(cl.Metadata.Namespace).List(k8sutil.ClusterListOpt(cl.Metadata.Name))
		if err != nil {
			return false, err
		}
		if len(list.Items) > 0 {
			p := list.Items[0]
			LogfWithTimestamp(t, "waiting pod (%s) to be deleted.", p.Name)

			buf := bytes.NewBuffer(nil)
			buf.WriteString("init container status:\n")
			printContainerStatus(buf, p.Status.InitContainerStatuses)
			buf.WriteString("container status:\n")
			printContainerStatus(buf, p.Status.ContainerStatuses)
			t.Logf("pod (%s) status.phase is (%s): %v", p.Name, p.Status.Phase, buf.String())

			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return fmt.Errorf("fail to wait pods deleted: %v", err)
	}

	err = retryutil.Retry(5*time.Second, 5, func() (done bool, err error) {
		list, err := kubeClient.CoreV1().Services(cl.Metadata.Namespace).List(k8sutil.ClusterListOpt(cl.Metadata.Name))
		if err != nil {
			return false, err
		}
		if len(list.Items) > 0 {
			LogfWithTimestamp(t, "waiting service (%s) to be deleted", list.Items[0].Name)
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return fmt.Errorf("fail to wait services deleted: %v", err)
	}
	return nil
}

func waitBackupDeleted(kubeClient kubernetes.Interface, cl *spec.Cluster, storageCheckerOptions *StorageCheckerOptions) error {
	err := retryutil.Retry(5*time.Second, 5, func() (bool, error) {
		_, err := kubeClient.AppsV1beta1().Deployments(cl.Metadata.Namespace).Get(k8sutil.BackupSidecarName(cl.Metadata.Name), metav1.GetOptions{})
		if err == nil {
			return false, nil
		}
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		return false, err
	})
	if err != nil {
		return fmt.Errorf("failed to wait backup Deployment deleted: %v", err)
	}
	err = retryutil.Retry(5*time.Second, 2, func() (done bool, err error) {
		ls := labels.SelectorFromSet(map[string]string{
			"app":          k8sutil.BackupPodSelectorAppField,
			"etcd_cluster": cl.Metadata.Name,
		}).String()
		pl, err := kubeClient.CoreV1().Pods(cl.Metadata.Namespace).List(metav1.ListOptions{
			LabelSelector: ls,
		})
		if err != nil {
			return false, err
		}
		if len(pl.Items) == 0 {
			return true, nil
		}
		if pl.Items[0].DeletionTimestamp != nil {
			return true, nil
		}
		return false, nil
	})
	if err != nil {
		return fmt.Errorf("failed to wait backup pod terminated: %v", err)
	}
	// The rest is to track backup storage, e.g. PV or S3 "dir" deleted.
	// If CleanupBackupsOnClusterDelete=false, we don't delete them and thus don't check them.
	if !cl.Spec.Backup.CleanupBackupsOnClusterDelete {
		return nil
	}
	err = retryutil.Retry(5*time.Second, 5, func() (done bool, err error) {
		switch cl.Spec.Backup.StorageType {
		case spec.BackupStorageTypePersistentVolume, spec.BackupStorageTypeDefault:
			pl, err := kubeClient.CoreV1().PersistentVolumeClaims(cl.Metadata.Namespace).List(k8sutil.ClusterListOpt(cl.Metadata.Name))
			if err != nil {
				return false, err
			}
			if len(pl.Items) > 0 {
				return false, nil
			}
		case spec.BackupStorageTypeS3:
			resp, err := storageCheckerOptions.S3Cli.ListObjects(&s3.ListObjectsInput{
				Bucket: aws.String(storageCheckerOptions.S3Bucket),
				Prefix: aws.String(path.Join("v1", cl.Metadata.Namespace, cl.Metadata.Name) + "/"),
			})
			if err != nil {
				return false, err
			}
			if len(resp.Contents) > 0 {
				return false, nil
			}
		}
		return true, nil
	})
	if err != nil {
		return fmt.Errorf("failed to wait storage (%s) to be deleted: %v", cl.Spec.Backup.StorageType, err)
	}
	return nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package base

import (
	"fmt"
	"regexp"

	"github.com/couchbaselabs/go-couchbase"
	"github.com/couchbaselabs/walrus"
	"github.com/dustin/gomemcached/client"
)

type Bucket walrus.Bucket
type TapArguments walrus.TapArguments
type TapFeed walrus.TapFeed
type AuthHandler couchbase.AuthHandler

// Full specification of how to connect to a bucket
type BucketSpec struct {
	Server, PoolName, BucketName string
	Auth                         AuthHandler
}

// Implementation of walrus.Bucket that talks to a Couchbase server
type couchbaseBucket struct {
	*couchbase.Bucket
}

type couchbaseFeedImpl struct {
	*couchbase.TapFeed
	events <-chan walrus.TapEvent
}

func (feed *couchbaseFeedImpl) Events() <-chan walrus.TapEvent {
	return feed.events
}

func (bucket couchbaseBucket) GetName() string {
	return bucket.Name
}

func (bucket couchbaseBucket) Write(k string, flags int, exp int, v interface{}, opt walrus.WriteOptions) (err error) {
	return bucket.Bucket.Write(k, flags, exp, v, couchbase.WriteOptions(opt))
}

func (bucket couchbaseBucket) Update(k string, exp int, callback walrus.UpdateFunc) error {
	return bucket.Bucket.Update(k, exp, couchbase.UpdateFunc(callback))
}

func (bucket couchbaseBucket) WriteUpdate(k string, exp int, callback walrus.WriteUpdateFunc) error {
	cbCallback := func(current []byte) (updated []byte, opt couchbase.WriteOptions, err error) {
		updated, walrusOpt, err := callback(current)
		opt = couchbase.WriteOptions(walrusOpt)
		return
	}
	return bucket.Bucket.WriteUpdate(k, exp, cbCallback)
}

func (bucket couchbaseBucket) View(ddoc, name string, params map[string]interface{}) (walrus.ViewResult, error) {
	vres := walrus.ViewResult{}
	return vres, bucket.Bucket.ViewCustom(ddoc, name, params, &vres)
}

func (bucket couchbaseBucket) StartTapFeed(args walrus.TapArguments) (walrus.TapFeed, error) {
	cbArgs := memcached.TapArguments{
		Backfill: args.Backfill,
		Dump:     args.Dump,
		KeysOnly: args.KeysOnly,
	}
	cbFeed, err := bucket.Bucket.StartTapFeed(&cbArgs)
	if err != nil {
		return nil, err
	}

	// Create a bridge from the Couchbase tap feed to a Walrus tap feed:
	events := make(chan walrus.TapEvent)
	impl := couchbaseFeedImpl{cbFeed, events}
	go func() {
		for cbEvent := range cbFeed.C {
			events <- walrus.TapEvent{
				Opcode: walrus.TapOpcode(cbEvent.Opcode),
				Expiry: cbEvent.Expiry,
				Flags:  cbEvent.Flags,
				Key:    cbEvent.Key,
				Value:  cbEvent.Value,
			}
		}
	}()
	return &impl, nil
}

func (bucket couchbaseBucket) Dump() {
	Warn("Dump not implemented for couchbaseBucket")
}

// Creates a Bucket that talks to a real live Couchbase server.
func GetCouchbaseBucket(spec BucketSpec) (bucket Bucket, err error) {
	client, err := couchbase.ConnectWithAuth(spec.Server, spec.Auth)
	if err != nil {
		return
	}
	poolName := spec.PoolName
	if poolName == "" {
		poolName = "default"
	}
	pool, err := client.GetPool(poolName)
	if err != nil {
		return
	}
	cbbucket, err := pool.GetBucket(spec.BucketName)
	if err == nil {
		bucket = couchbaseBucket{cbbucket}
	}
	return
}

func GetBucket(spec BucketSpec) (bucket Bucket, err error) {
	if isWalrus, _ := regexp.MatchString(`^(walrus:|file:|/|\.)`, spec.Server); isWalrus {
		Log("Opening Walrus database %s on <%s>", spec.BucketName, spec.Server)
		walrus.Logging = LogKeys["Walrus"]
		return walrus.GetBucket(spec.Server, spec.PoolName, spec.BucketName)
	}
	suffix := ""
	if spec.Auth != nil {
		username, _ := spec.Auth.GetCredentials()
		suffix = fmt.Sprintf(" as user %q", username)
	}
	Log("Opening Couchbase database %s on <%s>%s", spec.BucketName, spec.Server, suffix)
	return GetCouchbaseBucket(spec)
}
<commit_msg>Increase max memcached request size to 10M bytes<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package base

import (
	"fmt"
	"regexp"

	"github.com/couchbaselabs/go-couchbase"
	"github.com/couchbaselabs/walrus"
	"github.com/dustin/gomemcached"
	"github.com/dustin/gomemcached/client"
)

func init() {
	// Increase max memcached request size to 10M bytes, to support large docs (attachments!)
	// arriving in a tap feed. (see issue #210.)
	gomemcached.MaxBodyLen = int(10.0e6)
}

type Bucket walrus.Bucket
type TapArguments walrus.TapArguments
type TapFeed walrus.TapFeed
type AuthHandler couchbase.AuthHandler

// Full specification of how to connect to a bucket
type BucketSpec struct {
	Server, PoolName, BucketName string
	Auth                         AuthHandler
}

// Implementation of walrus.Bucket that talks to a Couchbase server
type couchbaseBucket struct {
	*couchbase.Bucket
}

type couchbaseFeedImpl struct {
	*couchbase.TapFeed
	events <-chan walrus.TapEvent
}

func (feed *couchbaseFeedImpl) Events() <-chan walrus.TapEvent {
	return feed.events
}

func (bucket couchbaseBucket) GetName() string {
	return bucket.Name
}

func (bucket couchbaseBucket) Write(k string, flags int, exp int, v interface{}, opt walrus.WriteOptions) (err error) {
	return bucket.Bucket.Write(k, flags, exp, v, couchbase.WriteOptions(opt))
}

func (bucket couchbaseBucket) Update(k string, exp int, callback walrus.UpdateFunc) error {
	return bucket.Bucket.Update(k, exp, couchbase.UpdateFunc(callback))
}

func (bucket couchbaseBucket) WriteUpdate(k string, exp int, callback walrus.WriteUpdateFunc) error {
	cbCallback := func(current []byte) (updated []byte, opt couchbase.WriteOptions, err error) {
		updated, walrusOpt, err := callback(current)
		opt = couchbase.WriteOptions(walrusOpt)
		return
	}
	return bucket.Bucket.WriteUpdate(k, exp, cbCallback)
}

func (bucket couchbaseBucket) View(ddoc, name string, params map[string]interface{}) (walrus.ViewResult, error) {
	vres := walrus.ViewResult{}
	return vres, bucket.Bucket.ViewCustom(ddoc, name, params, &vres)
}

func (bucket couchbaseBucket) StartTapFeed(args walrus.TapArguments) (walrus.TapFeed, error) {
	cbArgs := memcached.TapArguments{
		Backfill: args.Backfill,
		Dump:     args.Dump,
		KeysOnly: args.KeysOnly,
	}
	cbFeed, err := bucket.Bucket.StartTapFeed(&cbArgs)
	if err != nil {
		return nil, err
	}

	// Create a bridge from the Couchbase tap feed to a Walrus tap feed:
	events := make(chan walrus.TapEvent)
	impl := couchbaseFeedImpl{cbFeed, events}
	go func() {
		for cbEvent := range cbFeed.C {
			events <- walrus.TapEvent{
				Opcode: walrus.TapOpcode(cbEvent.Opcode),
				Expiry: cbEvent.Expiry,
				Flags:  cbEvent.Flags,
				Key:    cbEvent.Key,
				Value:  cbEvent.Value,
			}
		}
	}()
	return &impl, nil
}

func (bucket couchbaseBucket) Dump() {
	Warn("Dump not implemented for couchbaseBucket")
}

// Creates a Bucket that talks to a real live Couchbase server.
func GetCouchbaseBucket(spec BucketSpec) (bucket Bucket, err error) {
	client, err := couchbase.ConnectWithAuth(spec.Server, spec.Auth)
	if err != nil {
		return
	}
	poolName := spec.PoolName
	if poolName == "" {
		poolName = "default"
	}
	pool, err := client.GetPool(poolName)
	if err != nil {
		return
	}
	cbbucket, err := pool.GetBucket(spec.BucketName)
	if err == nil {
		bucket = couchbaseBucket{cbbucket}
	}
	return
}

func GetBucket(spec BucketSpec) (bucket Bucket, err error) {
	if isWalrus, _ := regexp.MatchString(`^(walrus:|file:|/|\.)`, spec.Server); isWalrus {
		Log("Opening Walrus database %s on <%s>", spec.BucketName, spec.Server)
		walrus.Logging = LogKeys["Walrus"]
		return walrus.GetBucket(spec.Server, spec.PoolName, spec.BucketName)
	}
	suffix := ""
	if spec.Auth != nil {
		username, _ := spec.Auth.GetCredentials()
		suffix = fmt.Sprintf(" as user %q", username)
	}
	Log("Opening Couchbase database %s on <%s>%s", spec.BucketName, spec.Server, suffix)
	return GetCouchbaseBucket(spec)
}
<|endoftext|>


<commit_before>package openflow

import (
	"errors"
	"fmt"

	"github.com/kandoo/beehive/Godeps/_workspace/src/github.com/golang/glog"
	bh "github.com/kandoo/beehive"
	"github.com/kandoo/beehive-netctrl/nom"
	"github.com/kandoo/beehive-netctrl/openflow/of"
	"github.com/kandoo/beehive-netctrl/openflow/of10"
	"github.com/kandoo/beehive-netctrl/openflow/of12"
)

type ofDriver interface {
	handshake(conn *ofConn) error
	handlePkt(pkt of.Header, conn *ofConn) error
	handleMsg(msg bh.Msg, conn *ofConn) error
	handleConnClose(conn *ofConn)
}

type of10Driver struct {
	ofPorts  map[uint16]*nom.Port
	nomPorts map[nom.UID]uint16
}

type of12Driver struct {
	ofPorts  map[uint32]*nom.Port
	nomPorts map[nom.UID]uint32
}

func (d *of10Driver) handlePkt(pkt of.Header, c *ofConn) error {
	pkt10, err := of10.ToHeader10(pkt)
	if err != nil {
		return err
	}

	switch {
	case of10.IsEchoRequest(pkt10):
		return d.handleEchoRequest(of10.NewEchoRequestWithBuf(pkt10.Buf), c)
	case of10.IsFeaturesReply(pkt10):
		return d.handleFeaturesReply(of10.NewFeaturesReplyWithBuf(pkt10.Buf), c)
	case of10.IsPacketIn(pkt10):
		return d.handlePacketIn(of10.NewPacketInWithBuf(pkt10.Buf), c)
	case of10.IsErrorMsg(pkt10):
		return d.handleErrorMsg(of10.NewErrorMsgWithBuf(pkt10.Buf), c)
	default:
		return fmt.Errorf("Received unsupported packet: %v", pkt.Type())
	}
}

func (d *of12Driver) handlePkt(pkt of.Header, c *ofConn) error {
	pkt12, err := of12.ToHeader12(pkt)
	if err != nil {
		return err
	}

	switch {
	case of12.IsEchoRequest(pkt12):
		return d.handleEchoRequest(of12.NewEchoRequestWithBuf(pkt12.Buf), c)
	case of12.IsFeaturesReply(pkt12):
		return d.handleFeaturesReply(of12.NewFeaturesReplyWithBuf(pkt12.Buf), c)
	case of12.IsPacketIn(pkt12):
		return d.handlePacketIn(of12.NewPacketInWithBuf(pkt12.Buf), c)
	case of12.IsErrorMsg(pkt12):
		return d.handleErrorMsg(of12.NewErrorMsgWithBuf(pkt12.Buf), c)
	default:
		return fmt.Errorf("Received unsupported packet: %v", pkt.Type())
	}
}

func (d *of10Driver) handleMsg(msg bh.Msg, c *ofConn) error {
	ofh, err := d.convToOF(msg)
	if err != nil {
		return err
	}

	if err := c.WriteHeader(ofh); err != nil {
		glog.Errorf("ofconn: Cannot write packet: %v", err)
		return err
	}

	return nil
}

func (d *of12Driver) handleMsg(msg bh.Msg, c *ofConn) error {
	ofh, err := d.convToOF(msg)
	if err != nil {
		return err
	}

	if err := c.WriteHeader(ofh); err != nil {
		glog.Errorf("ofconn: Cannot write packet: %v", err)
		return err
	}

	return nil
}

func (d *of10Driver) handleConnClose(c *ofConn) {
	emitNodeDisconnected(c)
}

func (d *of12Driver) handleConnClose(c *ofConn) {
	emitNodeDisconnected(c)
}

func emitNodeDisconnected(c *ofConn) {
	c.ctx.Emit(nom.NodeDisconnected{
		Node: c.node,
		Driver: nom.Driver{
			BeeID: c.ctx.ID(),
		},
	})
}

func (d *of10Driver) convToOF(msg bh.Msg) (of.Header, error) {
	switch data := msg.Data().(type) {
	case nom.PacketOut:
		buf := make([]byte, 256)
		out := of10.NewPacketOutWithBuf(buf)
		out.Init()
		out.SetBufferId(uint32(data.BufferID))

		ofPort, ok := d.nomPorts[data.InPort]
		if ok {
			out.SetInPort(ofPort)
		}

		// FIXME(soheil): When actions are added after data, the packet becomes
		// corrupted.
		for _, a := range data.Actions {
			ofa, err := d.convAction(a)
			if err != nil {
				return of.Header{},
					fmt.Errorf("of10Driver: Invalid action %v", err)
			}
			out.AddActions(ofa)
		}

		if data.BufferID == 0xFFFFFFFF {
			for _, d := range data.Packet {
				out.AddData(d)
			}
		}

		return out.Header, nil

	default:
		return of.Header{}, fmt.Errorf("of10Driver: Unsupported message %+v", data)
	}
}

func (d *of12Driver) convToOF(msg bh.Msg) (of.Header, error) {
	return of.Header{}, fmt.Errorf("of12Driver: Message not supported %+v",
		msg.Data())
}

func (d *of10Driver) convAction(a nom.Action) (of10.ActionHeader, error) {
	switch action := a.(type) {
	case nom.ActionFlood:
		flood := of10.NewActionOutput()
		flood.SetPort(uint16(of10.PP_FLOOD))
		return flood.ActionHeader, nil

	case nom.ActionForward:
		if len(action.Ports) != 1 {
			return of10.ActionHeader{},
				errors.New("of10Driver: can forward to only one port")
		}
		p, ok := d.nomPorts[action.Ports[0]]
		if !ok {
			return of10.ActionHeader{},
				fmt.Errorf("of10Driver: port %v no found", action.Ports[0])
		}
		out := of10.NewActionOutput()
		out.SetPort(uint16(p))
		return out.ActionHeader, nil

	default:
		return of10.ActionHeader{},
			fmt.Errorf("of10Driver: Action not supported %v", action)
	}
}
<commit_msg>Add support for FlowMod in the of10 driver<commit_after>package openflow

import (
	"errors"
	"fmt"

	bh "github.com/kandoo/beehive"
	"github.com/kandoo/beehive-netctrl/nom"
	"github.com/kandoo/beehive-netctrl/openflow/of"
	"github.com/kandoo/beehive-netctrl/openflow/of10"
	"github.com/kandoo/beehive-netctrl/openflow/of12"
	"github.com/kandoo/beehive/Godeps/_workspace/src/github.com/golang/glog"
)

type ofDriver interface {
	handshake(conn *ofConn) error
	handlePkt(pkt of.Header, conn *ofConn) error
	handleMsg(msg bh.Msg, conn *ofConn) error
	handleConnClose(conn *ofConn)
}

type of10Driver struct {
	ofPorts  map[uint16]*nom.Port
	nomPorts map[nom.UID]uint16
}

type of12Driver struct {
	ofPorts  map[uint32]*nom.Port
	nomPorts map[nom.UID]uint32
}

func (d *of10Driver) handlePkt(pkt of.Header, c *ofConn) error {
	pkt10, err := of10.ToHeader10(pkt)
	if err != nil {
		return err
	}

	switch {
	case of10.IsEchoRequest(pkt10):
		return d.handleEchoRequest(of10.NewEchoRequestWithBuf(pkt10.Buf), c)
	case of10.IsFeaturesReply(pkt10):
		return d.handleFeaturesReply(of10.NewFeaturesReplyWithBuf(pkt10.Buf), c)
	case of10.IsPacketIn(pkt10):
		return d.handlePacketIn(of10.NewPacketInWithBuf(pkt10.Buf), c)
	case of10.IsErrorMsg(pkt10):
		return d.handleErrorMsg(of10.NewErrorMsgWithBuf(pkt10.Buf), c)
	default:
		return fmt.Errorf("Received unsupported packet: %v", pkt.Type())
	}
}

func (d *of12Driver) handlePkt(pkt of.Header, c *ofConn) error {
	pkt12, err := of12.ToHeader12(pkt)
	if err != nil {
		return err
	}

	switch {
	case of12.IsEchoRequest(pkt12):
		return d.handleEchoRequest(of12.NewEchoRequestWithBuf(pkt12.Buf), c)
	case of12.IsFeaturesReply(pkt12):
		return d.handleFeaturesReply(of12.NewFeaturesReplyWithBuf(pkt12.Buf), c)
	case of12.IsPacketIn(pkt12):
		return d.handlePacketIn(of12.NewPacketInWithBuf(pkt12.Buf), c)
	case of12.IsErrorMsg(pkt12):
		return d.handleErrorMsg(of12.NewErrorMsgWithBuf(pkt12.Buf), c)
	default:
		return fmt.Errorf("received unsupported packet: %v", pkt.Type())
	}
}

func (d *of10Driver) handleMsg(msg bh.Msg, c *ofConn) error {
	ofh, err := d.convToOF(msg)
	if err != nil {
		return err
	}

	if err := c.WriteHeader(ofh); err != nil {
		glog.Errorf("ofconn: cannot write packet: %v", err)
		return err
	}

	return nil
}

func (d *of12Driver) handleMsg(msg bh.Msg, c *ofConn) error {
	ofh, err := d.convToOF(msg)
	if err != nil {
		return err
	}

	if err := c.WriteHeader(ofh); err != nil {
		glog.Errorf("ofconn: cannot write packet: %v", err)
		return err
	}

	return nil
}

func (d *of10Driver) handleConnClose(c *ofConn) {
	emitNodeDisconnected(c)
}

func (d *of12Driver) handleConnClose(c *ofConn) {
	emitNodeDisconnected(c)
}

func emitNodeDisconnected(c *ofConn) {
	c.ctx.Emit(nom.NodeDisconnected{
		Node: c.node,
		Driver: nom.Driver{
			BeeID: c.ctx.ID(),
		},
	})
}

func (d *of10Driver) convToOF(msg bh.Msg) (of.Header, error) {
	switch data := msg.Data().(type) {
	case nom.PacketOut:
		buf := make([]byte, 256)
		out := of10.NewPacketOutWithBuf(buf)
		out.Init()
		out.SetBufferId(uint32(data.BufferID))

		ofPort, ok := d.nomPorts[data.InPort]
		if ok {
			out.SetInPort(ofPort)
		}

		// FIXME(soheil): when actions are added after data, the packet becomes
		// corrupted.
		for _, a := range data.Actions {
			ofa, err := d.convAction(a)
			if err != nil {
				return of.Header{},
					fmt.Errorf("of10Driver: invalid action %v", err)
			}
			out.AddActions(ofa)
		}

		if data.BufferID == 0xFFFFFFFF {
			for _, d := range data.Packet {
				out.AddData(d)
			}
		}

		return out.Header, nil

	case nom.AddFlowEntry:
		mod := of10.NewFlowMod()
		mod.SetCommand(uint16(of10.PFC_ADD))
		mod.SetPriority(uint16(data.Flow.Priority))
		mod.SetIdleTimeout(uint16(data.Flow.IdleTimeout))
		mod.SetHardTimeout(uint16(data.Flow.HardTimeout))
		mod.SetBufferId(0xFFFFFFFF)
		match, err := d.convMatch(data.Flow.Match)
		if err != nil {
			return of.Header{}, fmt.Errorf("of10Driver: invalid match %v", err)
		}
		mod.SetMatch(match)
		for _, a := range data.Flow.Actions {
			ofa, err := d.convAction(a)
			if err != nil {
				return of.Header{},
					fmt.Errorf("of10Driver: invalid action %v", err)
			}
			mod.AddActions(ofa)
		}
		return mod.Header, nil

	case nom.DelFlowEntry:
		mod := of10.NewFlowMod()
		if data.Exact {
			mod.SetCommand(uint16(of10.PFC_DELETE))
		} else {
			mod.SetCommand(uint16(of10.PFC_DELETE_STRICT))
		}
		match, err := d.convMatch(data.Match)
		if err != nil {
			return of.Header{}, fmt.Errorf("of10Driver: invalid match %v", err)
		}
		mod.SetMatch(match)
		return mod.Header, nil

	default:
		return of.Header{}, fmt.Errorf("of10Driver: unsupported message %+v", data)
	}
}

func (d *of12Driver) convToOF(msg bh.Msg) (of.Header, error) {
	return of.Header{}, fmt.Errorf("of12Driver: message not supported %+v",
		msg.Data())
}

func (d *of10Driver) convAction(a nom.Action) (of10.ActionHeader, error) {
	switch action := a.(type) {
	case nom.ActionFlood:
		flood := of10.NewActionOutput()
		flood.SetPort(uint16(of10.PP_FLOOD))
		return flood.ActionHeader, nil

	case nom.ActionForward:
		if len(action.Ports) != 1 {
			return of10.ActionHeader{},
				errors.New("of10Driver: can forward to only one port")
		}
		p, ok := d.nomPorts[action.Ports[0]]
		if !ok {
			return of10.ActionHeader{},
				fmt.Errorf("of10Driver: port %v no found", action.Ports[0])
		}
		out := of10.NewActionOutput()
		out.SetPort(uint16(p))
		return out.ActionHeader, nil

	default:
		return of10.ActionHeader{},
			fmt.Errorf("of10Driver: action not supported %v", action)
	}
}

func (d *of10Driver) convMatch(m nom.Match) (of10.Match, error) {
	ofm := of10.NewMatch()
	w := of10.PFW_ALL
	for _, f := range m.Fields {
		switch f := f.(type) {
		case nom.InPort:
			p, ok := d.nomPorts[nom.UID(f)]
			if !ok {
				return of10.Match{}, fmt.Errorf("of10Driver: nom port not found %v", f)
			}
			ofm.SetInPort(p)
			w &= ^of10.PFW_IN_PORT

		case nom.EthDst:
			if f.Mask != [6]byte{} {
				return of10.Match{},
					fmt.Errorf("of10Driver: masked ethernet address is not supported")
			}
			ofm.SetDlDst([6]byte(f.Addr))
			w &= ^of10.PFW_DL_DST

		case nom.EthSrc:
			if f.Mask != [6]byte{} {
				return of10.Match{},
					fmt.Errorf("of10Driver: masked ethernet address is not supported")
			}
			ofm.SetDlSrc([6]byte(f.Addr))
			w &= ^of10.PFW_DL_SRC

		case nom.EthType:
			ofm.SetDlType(uint16(f))
			w &= ^of10.PFW_DL_TYPE

		case nom.IPV4Src:
			ofm.SetNwSrc(f.Addr.Uint32())
			mask := f.Mask.Uint32()
			w &= ^of10.PFW_NW_SRC_ALL
			for i := uint(0); i < 32; i++ {
				if mask&(1<<i) != 0 {
					w |= of10.FlowWildcards(i << uint(of10.PFW_NW_SRC_SHIFT))
					break
				}
			}

		case nom.IPV4Dst:
			ofm.SetNwDst(f.Addr.Uint32())
			mask := f.Mask.Uint32()
			w &= ^of10.PFW_NW_DST_ALL
			for i := uint(0); i < 32; i++ {
				if mask&(1<<i) != 0 {
					w |= of10.FlowWildcards(i << uint(of10.PFW_NW_DST_SHIFT))
					break
				}
			}

		case nom.TransportPortSrc:
			ofm.SetTpSrc(uint16(f))
			w &= ^of10.PFW_TP_SRC

		case nom.TransportPortDst:
			ofm.SetTpDst(uint16(f))
			w &= ^of10.PFW_TP_DST

		case nom.IPV6Src, nom.IPV6Dst:
			return of10.Match{}, fmt.Errorf("of10Driver: IPv6 not supported")
		}
	}
	ofm.SetWildcards(uint32(w))
	return ofm, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Jamie Hall. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package spdy

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/SlyMarbo/spdy/common"
)

// A Transport is an HTTP/SPDY http.RoundTripper.
type Transport struct {
	m sync.Mutex

	// Proxy specifies a function to return a proxy for a given
	// Request. If the function returns a non-nil error, the
	// request is aborted with the provided error.
	// If Proxy is nil or returns a nil *URL, no proxy is used.
	Proxy func(*http.Request) (*url.URL, error)

	// Dial specifies the dial function for creating TCP
	// connections.
	// If Dial is nil, net.Dial is used.
	Dial func(network, addr string) (net.Conn, error) // TODO: use

	// TLSClientConfig specifies the TLS configuration to use with
	// tls.Client. If nil, the default configuration is used.
	TLSClientConfig *tls.Config

	// DisableKeepAlives, if true, prevents re-use of TCP connections
	// between different HTTP requests.
	DisableKeepAlives bool

	// DisableCompression, if true, prevents the Transport from
	// requesting compression with an "Accept-Encoding: gzip"
	// request header when the Request contains no existing
	// Accept-Encoding value. If the Transport requests gzip on
	// its own and gets a gzipped response, it's transparently
	// decoded in the Response.Body. However, if the user
	// explicitly requested gzip it is not automatically
	// uncompressed.
	DisableCompression bool

	// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
	// (keep-alive) to keep per-host.  If zero,
	// DefaultMaxIdleConnsPerHost is used.
	MaxIdleConnsPerHost int

	// ResponseHeaderTimeout, if non-zero, specifies the amount of
	// time to wait for a server's response headers after fully
	// writing the request (including its body, if any). This
	// time does not include the time to read the response body.
	ResponseHeaderTimeout time.Duration

	spdyConns map[string]common.Conn   // SPDY connections mapped to host:port.
	tcpConns  map[string]chan net.Conn // Non-SPDY connections mapped to host:port.
	connLimit map[string]chan struct{} // Used to enforce the TCP conn limit.

	// Priority is used to determine the request priority of SPDY
	// requests. If nil, spdy.DefaultPriority is used.
	Priority func(*url.URL) common.Priority

	// Receiver is used to receive the server's response. If left
	// nil, the default Receiver will parse and create a normal
	// Response.
	Receiver common.Receiver

	// PushReceiver is used to receive server pushes. If left nil,
	// pushes will be refused. The provided Request will be that
	// sent with the server push. See Receiver for more detail on
	// its methods.
	PushReceiver common.Receiver
}

// NewTransport gives a simple initialised Transport.
func NewTransport(insecureSkipVerify bool) *Transport {
	return &Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: insecureSkipVerify,
			NextProtos:         npn(),
		},
	}
}

// dial makes the connection to an endpoint.
func (t *Transport) dial(u *url.URL) (conn net.Conn, err error) {

	if t.TLSClientConfig == nil {
		t.TLSClientConfig = &tls.Config{
			NextProtos: npn(),
		}
	} else if t.TLSClientConfig.NextProtos == nil {
		t.TLSClientConfig.NextProtos = npn()
	}

	// Wait for a connection slot to become available.
	<-t.connLimit[u.Host]

	switch u.Scheme {
	case "http":
		conn, err = net.Dial("tcp", u.Host)
	case "https":
		conn, err = tls.Dial("tcp", u.Host, t.TLSClientConfig)
	default:
		err = errors.New(fmt.Sprintf("Error: URL has invalid scheme %q.", u.Scheme))
	}

	if err != nil {
		// The connection never happened, which frees up a slot.
		t.connLimit[u.Host] <- struct{}{}
	}

	return conn, err
}

// doHTTP is used to process an HTTP(S) request, using the TCP connection pool.
func (t *Transport) doHTTP(conn net.Conn, req *http.Request) (*http.Response, error) {
	debug.Printf("Requesting %q over HTTP.\n", req.URL.String())

	// Create the HTTP ClientConn, which handles the
	// HTTP details.
	httpConn := httputil.NewClientConn(conn, nil)
	res, err := httpConn.Do(req)
	if err != nil {
		return nil, err
	}

	if !res.Close {
		t.tcpConns[req.URL.Host] <- conn
	} else {
		// This connection is closing, so another can be used.
		t.connLimit[req.URL.Host] <- struct{}{}
		err = httpConn.Close()
		if err != nil {
			return nil, err
		}
	}

	return res, nil
}

// RoundTrip handles the actual request; ensuring a connection is
// made, determining which protocol to use, and performing the
// request.
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	u := req.URL

	// Make sure the URL host contains the port.
	if !strings.Contains(u.Host, ":") {
		switch u.Scheme {
		case "http":
			u.Host += ":80"

		case "https":
			u.Host += ":443"
		}
	}

	conn, tcpConn, err := t.process(req)
	if err != nil {
		return nil, err
	}
	if tcpConn != nil {
		return t.doHTTP(tcpConn, req)
	}

	// The connection has now been established.

	debug.Printf("Requesting %q over SPDY.\n", u.String())

	// Determine the request priority.
	var priority common.Priority
	if t.Priority != nil {
		priority = t.Priority(req.URL)
	} else {
		priority = common.DefaultPriority(req.URL)
	}

	return conn.RequestResponse(req, t.Receiver, priority)
}

func (t *Transport) process(req *http.Request) (common.Conn, net.Conn, error) {
	t.m.Lock()
	defer t.m.Unlock()

	u := req.URL

	// Initialise structures if necessary.
	if t.spdyConns == nil {
		t.spdyConns = make(map[string]common.Conn)
	}
	if t.tcpConns == nil {
		t.tcpConns = make(map[string]chan net.Conn)
	}
	if t.connLimit == nil {
		t.connLimit = make(map[string]chan struct{})
	}
	if t.MaxIdleConnsPerHost == 0 {
		t.MaxIdleConnsPerHost = http.DefaultMaxIdleConnsPerHost
	}
	if _, ok := t.connLimit[u.Host]; !ok {
		limitChan := make(chan struct{}, t.MaxIdleConnsPerHost)
		t.connLimit[u.Host] = limitChan
		for i := 0; i < t.MaxIdleConnsPerHost; i++ {
			limitChan <- struct{}{}
		}
	}

	// Check the non-SPDY connection pool.
	if connChan, ok := t.tcpConns[u.Host]; ok {
		select {
		case tcpConn := <-connChan:
			// Use a connection from the pool.
			return nil, tcpConn, nil
		default:
		}
	} else {
		t.tcpConns[u.Host] = make(chan net.Conn, t.MaxIdleConnsPerHost)
	}

	// Check the SPDY connection pool.
	conn, ok := t.spdyConns[u.Host]
	if !ok || u.Scheme == "http" || (conn != nil && conn.Closed()) {
		tcpConn, err := t.dial(req.URL)
		if err != nil {
			return nil, nil, err
		}

		if tlsConn, ok := tcpConn.(*tls.Conn); !ok {
			// Handle HTTP requests.
			return nil, tcpConn, nil
		} else {
			// Handle HTTPS/SPDY requests.
			state := tlsConn.ConnectionState()

			// Complete handshake if necessary.
			if !state.HandshakeComplete {
				err = tlsConn.Handshake()
				if err != nil {
					return nil, nil, err
				}
			}

			// Verify hostname, unless requested not to.
			if !t.TLSClientConfig.InsecureSkipVerify {
				err = tlsConn.VerifyHostname(req.URL.Host)
				if err != nil {
					// Also try verifying the hostname with/without a port number.
					i := strings.Index(req.URL.Host, ":")
					err = tlsConn.VerifyHostname(req.URL.Host[:i])
					if err != nil {
						return nil, nil, err
					}
				}
			}

			// If a protocol could not be negotiated, assume HTTPS.
			if !state.NegotiatedProtocolIsMutual {
				return nil, tcpConn, nil
			}

			// Scan the list of supported NPN strings.
			supported := false
			for _, proto := range npn() {
				if state.NegotiatedProtocol == proto {
					supported = true
					break
				}
			}

			// Ensure the negotiated protocol is supported.
			if !supported && state.NegotiatedProtocol != "" {
				msg := fmt.Sprintf("Error: Unsupported negotiated protocol %q.", state.NegotiatedProtocol)
				return nil, nil, errors.New(msg)
			}

			// Handle the protocol.
			switch state.NegotiatedProtocol {
			case "http/1.1", "":
				return nil, tcpConn, nil

			case "spdy/3.1":
				newConn, err := NewClientConn(tlsConn, t.PushReceiver, 3, 1)
				if err != nil {
					return nil, nil, err
				}
				go newConn.Run()
				t.spdyConns[u.Host] = newConn
				conn = newConn

			case "spdy/3":
				newConn, err := NewClientConn(tlsConn, t.PushReceiver, 3, 0)
				if err != nil {
					return nil, nil, err
				}
				go newConn.Run()
				t.spdyConns[u.Host] = newConn
				conn = newConn

			case "spdy/2":
				newConn, err := NewClientConn(tlsConn, t.PushReceiver, 2, 0)
				if err != nil {
					return nil, nil, err
				}
				go newConn.Run()
				t.spdyConns[u.Host] = newConn
				conn = newConn
			}
		}
	}

	return conn, nil, nil
}
<commit_msg>Added connection cleanup<commit_after>// Copyright 2013 Jamie Hall. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package spdy

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/SlyMarbo/spdy/common"
)

// A Transport is an HTTP/SPDY http.RoundTripper.
type Transport struct {
	m sync.Mutex

	// Proxy specifies a function to return a proxy for a given
	// Request. If the function returns a non-nil error, the
	// request is aborted with the provided error.
	// If Proxy is nil or returns a nil *URL, no proxy is used.
	Proxy func(*http.Request) (*url.URL, error)

	// Dial specifies the dial function for creating TCP
	// connections.
	// If Dial is nil, net.Dial is used.
	Dial func(network, addr string) (net.Conn, error) // TODO: use

	// TLSClientConfig specifies the TLS configuration to use with
	// tls.Client. If nil, the default configuration is used.
	TLSClientConfig *tls.Config

	// DisableKeepAlives, if true, prevents re-use of TCP connections
	// between different HTTP requests.
	DisableKeepAlives bool

	// DisableCompression, if true, prevents the Transport from
	// requesting compression with an "Accept-Encoding: gzip"
	// request header when the Request contains no existing
	// Accept-Encoding value. If the Transport requests gzip on
	// its own and gets a gzipped response, it's transparently
	// decoded in the Response.Body. However, if the user
	// explicitly requested gzip it is not automatically
	// uncompressed.
	DisableCompression bool

	// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
	// (keep-alive) to keep per-host.  If zero,
	// DefaultMaxIdleConnsPerHost is used.
	MaxIdleConnsPerHost int

	// ResponseHeaderTimeout, if non-zero, specifies the amount of
	// time to wait for a server's response headers after fully
	// writing the request (including its body, if any). This
	// time does not include the time to read the response body.
	ResponseHeaderTimeout time.Duration

	spdyConns map[string]common.Conn   // SPDY connections mapped to host:port.
	tcpConns  map[string]chan net.Conn // Non-SPDY connections mapped to host:port.
	connLimit map[string]chan struct{} // Used to enforce the TCP conn limit.

	// Priority is used to determine the request priority of SPDY
	// requests. If nil, spdy.DefaultPriority is used.
	Priority func(*url.URL) common.Priority

	// Receiver is used to receive the server's response. If left
	// nil, the default Receiver will parse and create a normal
	// Response.
	Receiver common.Receiver

	// PushReceiver is used to receive server pushes. If left nil,
	// pushes will be refused. The provided Request will be that
	// sent with the server push. See Receiver for more detail on
	// its methods.
	PushReceiver common.Receiver
}

// NewTransport gives a simple initialised Transport.
func NewTransport(insecureSkipVerify bool) *Transport {
	return &Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: insecureSkipVerify,
			NextProtos:         npn(),
		},
	}
}

// dial makes the connection to an endpoint.
func (t *Transport) dial(u *url.URL) (conn net.Conn, err error) {

	if t.TLSClientConfig == nil {
		t.TLSClientConfig = &tls.Config{
			NextProtos: npn(),
		}
	} else if t.TLSClientConfig.NextProtos == nil {
		t.TLSClientConfig.NextProtos = npn()
	}

	// Wait for a connection slot to become available.
	<-t.connLimit[u.Host]

	switch u.Scheme {
	case "http":
		conn, err = net.Dial("tcp", u.Host)
	case "https":
		conn, err = tls.Dial("tcp", u.Host, t.TLSClientConfig)
	default:
		err = errors.New(fmt.Sprintf("Error: URL has invalid scheme %q.", u.Scheme))
	}

	if err != nil {
		// The connection never happened, which frees up a slot.
		t.connLimit[u.Host] <- struct{}{}
	}

	return conn, err
}

// doHTTP is used to process an HTTP(S) request, using the TCP connection pool.
func (t *Transport) doHTTP(conn net.Conn, req *http.Request) (*http.Response, error) {
	debug.Printf("Requesting %q over HTTP.\n", req.URL.String())

	// Create the HTTP ClientConn, which handles the
	// HTTP details.
	httpConn := httputil.NewClientConn(conn, nil)
	res, err := httpConn.Do(req)
	if err != nil {
		return nil, err
	}

	if !res.Close {
		t.tcpConns[req.URL.Host] <- conn
	} else {
		// This connection is closing, so another can be used.
		t.connLimit[req.URL.Host] <- struct{}{}
		err = httpConn.Close()
		if err != nil {
			return nil, err
		}
	}

	return res, nil
}

// RoundTrip handles the actual request; ensuring a connection is
// made, determining which protocol to use, and performing the
// request.
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	u := req.URL

	// Make sure the URL host contains the port.
	if !strings.Contains(u.Host, ":") {
		switch u.Scheme {
		case "http":
			u.Host += ":80"

		case "https":
			u.Host += ":443"
		}
	}

	conn, tcpConn, err := t.process(req)
	if err != nil {
		return nil, err
	}
	if tcpConn != nil {
		return t.doHTTP(tcpConn, req)
	}

	// The connection has now been established.

	debug.Printf("Requesting %q over SPDY.\n", u.String())

	// Determine the request priority.
	var priority common.Priority
	if t.Priority != nil {
		priority = t.Priority(req.URL)
	} else {
		priority = common.DefaultPriority(req.URL)
	}

	res, err := conn.RequestResponse(req, t.Receiver, priority)
	if conn.Closed() {
		t.connLimit[u.Host] <- struct{}{}
	}
	if err != nil {
		return nil, err
	}

	return res, nil
}

func (t *Transport) process(req *http.Request) (common.Conn, net.Conn, error) {
	t.m.Lock()
	defer t.m.Unlock()

	u := req.URL

	// Initialise structures if necessary.
	if t.spdyConns == nil {
		t.spdyConns = make(map[string]common.Conn)
	}
	if t.tcpConns == nil {
		t.tcpConns = make(map[string]chan net.Conn)
	}
	if t.connLimit == nil {
		t.connLimit = make(map[string]chan struct{})
	}
	if t.MaxIdleConnsPerHost == 0 {
		t.MaxIdleConnsPerHost = http.DefaultMaxIdleConnsPerHost
	}
	if _, ok := t.connLimit[u.Host]; !ok {
		limitChan := make(chan struct{}, t.MaxIdleConnsPerHost)
		t.connLimit[u.Host] = limitChan
		for i := 0; i < t.MaxIdleConnsPerHost; i++ {
			limitChan <- struct{}{}
		}
	}

	// Check the non-SPDY connection pool.
	if connChan, ok := t.tcpConns[u.Host]; ok {
		select {
		case tcpConn := <-connChan:
			// Use a connection from the pool.
			return nil, tcpConn, nil
		default:
		}
	} else {
		t.tcpConns[u.Host] = make(chan net.Conn, t.MaxIdleConnsPerHost)
	}

	// Check the SPDY connection pool.
	conn, ok := t.spdyConns[u.Host]
	if !ok || u.Scheme == "http" || (conn != nil && conn.Closed()) {
		tcpConn, err := t.dial(req.URL)
		if err != nil {
			return nil, nil, err
		}

		if tlsConn, ok := tcpConn.(*tls.Conn); !ok {
			// Handle HTTP requests.
			return nil, tcpConn, nil
		} else {
			// Handle HTTPS/SPDY requests.
			state := tlsConn.ConnectionState()

			// Complete handshake if necessary.
			if !state.HandshakeComplete {
				err = tlsConn.Handshake()
				if err != nil {
					return nil, nil, err
				}
			}

			// Verify hostname, unless requested not to.
			if !t.TLSClientConfig.InsecureSkipVerify {
				err = tlsConn.VerifyHostname(req.URL.Host)
				if err != nil {
					// Also try verifying the hostname with/without a port number.
					i := strings.Index(req.URL.Host, ":")
					err = tlsConn.VerifyHostname(req.URL.Host[:i])
					if err != nil {
						return nil, nil, err
					}
				}
			}

			// If a protocol could not be negotiated, assume HTTPS.
			if !state.NegotiatedProtocolIsMutual {
				return nil, tcpConn, nil
			}

			// Scan the list of supported NPN strings.
			supported := false
			for _, proto := range npn() {
				if state.NegotiatedProtocol == proto {
					supported = true
					break
				}
			}

			// Ensure the negotiated protocol is supported.
			if !supported && state.NegotiatedProtocol != "" {
				msg := fmt.Sprintf("Error: Unsupported negotiated protocol %q.", state.NegotiatedProtocol)
				return nil, nil, errors.New(msg)
			}

			// Handle the protocol.
			switch state.NegotiatedProtocol {
			case "http/1.1", "":
				return nil, tcpConn, nil

			case "spdy/3.1":
				newConn, err := NewClientConn(tlsConn, t.PushReceiver, 3, 1)
				if err != nil {
					return nil, nil, err
				}
				go newConn.Run()
				t.spdyConns[u.Host] = newConn
				conn = newConn

			case "spdy/3":
				newConn, err := NewClientConn(tlsConn, t.PushReceiver, 3, 0)
				if err != nil {
					return nil, nil, err
				}
				go newConn.Run()
				t.spdyConns[u.Host] = newConn
				conn = newConn

			case "spdy/2":
				newConn, err := NewClientConn(tlsConn, t.PushReceiver, 2, 0)
				if err != nil {
					return nil, nil, err
				}
				go newConn.Run()
				t.spdyConns[u.Host] = newConn
				conn = newConn
			}
		}
	}

	return conn, nil, nil
}
<|endoftext|>


<commit_before><commit_msg>Don't use go routines for subscribe and unsubscribe<commit_after><|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	calendar "google.golang.org/api/calendar/v3"

	"github.com/grafana/grafana_plugin_model/go/datasource"
	plugin "github.com/hashicorp/go-plugin"
)

type GoogleCalendarDatasource struct {
	plugin.NetRPCUnsupportedPlugin
}

type EventsListRequest struct {
	CalendarId             string
	TimeMin                string
	TimeMax                string
	OrderBy                string
	Q                      string
	SharedExtendedProperty string
	ShowDeleted            bool
	SingleEvents           bool
	MaxResults             int64
}

func (t *GoogleCalendarDatasource) Query(ctx context.Context, tsdbReq *datasource.DatasourceRequest) (*datasource.DatasourceResponse, error) {
	keyFilePath, ok := tsdbReq.Datasource.DecryptedSecureJsonData["serviceAccountKeyFilePath"]
	if !ok {
		return nil, fmt.Errorf("Unable to get service account key file path")
	}

	keyFileData, err := ioutil.ReadFile(keyFilePath)
	if err != nil {
		return nil, fmt.Errorf("Unable to get service account key: %s", keyFilePath)
	}

	conf, err := google.JWTConfigFromJSON(keyFileData, "https://www.googleapis.com/auth/calendar.readonly")
	if err != nil {
		return nil, fmt.Errorf("Unable to get service account JWT")
	}

	client := conf.Client(oauth2.NoContext)
	client.Get("...")

	calendarService, err := calendar.New(client)
	if err != nil {
		return nil, fmt.Errorf("Unable to retrieve Calendar client: %v", err)
	}

	return t.handleRawQuery(calendarService, tsdbReq)
}

func (t *GoogleCalendarDatasource) handleRawQuery(calendarService *calendar.Service, tsdbReq *datasource.DatasourceRequest) (*datasource.DatasourceResponse, error) {
	response := &datasource.DatasourceResponse{}

	var req EventsListRequest
	if err := json.Unmarshal([]byte(tsdbReq.Queries[0].ModelJson), &req); err != nil {
		return nil, err
	}

	eventsListCall := calendarService.Events.List(req.CalendarId).
		TimeMin(req.TimeMin).TimeMax(req.TimeMax).ShowDeleted(req.ShowDeleted).
		SingleEvents(req.SingleEvents).MaxResults(req.MaxResults).OrderBy(req.OrderBy)
	if req.Q != "" {
		eventsListCall = eventsListCall.Q(req.Q)
	}
	if req.SharedExtendedProperty != "" {
		eventsListCall = eventsListCall.SharedExtendedProperty(req.SharedExtendedProperty)
	}
	events, err := eventsListCall.Do()
	if err != nil {
		return nil, fmt.Errorf("Unable to retrieve events: %v", err)
	}

	resultJson, err := json.Marshal(events)
	if err != nil {
		return nil, err
	}

	response.Results = []*datasource.QueryResult{
		&datasource.QueryResult{
			MetaJson: string(resultJson),
		},
	}

	return response, nil
}
<commit_msg>improve error message<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	calendar "google.golang.org/api/calendar/v3"

	"github.com/grafana/grafana_plugin_model/go/datasource"
	plugin "github.com/hashicorp/go-plugin"
)

type GoogleCalendarDatasource struct {
	plugin.NetRPCUnsupportedPlugin
}

type EventsListRequest struct {
	CalendarId             string
	TimeMin                string
	TimeMax                string
	OrderBy                string
	Q                      string
	SharedExtendedProperty string
	ShowDeleted            bool
	SingleEvents           bool
	MaxResults             int64
}

func (t *GoogleCalendarDatasource) Query(ctx context.Context, tsdbReq *datasource.DatasourceRequest) (*datasource.DatasourceResponse, error) {
	keyFilePath, ok := tsdbReq.Datasource.DecryptedSecureJsonData["serviceAccountKeyFilePath"]
	if !ok {
		return nil, fmt.Errorf("Unable to get service account key file path")
	}

	keyFileData, err := ioutil.ReadFile(keyFilePath)
	if err != nil {
		return nil, fmt.Errorf("Unable to get service account key: %s", keyFilePath)
	}

	conf, err := google.JWTConfigFromJSON(keyFileData, "https://www.googleapis.com/auth/calendar.readonly")
	if err != nil {
		return nil, fmt.Errorf("Unable to get service account JWT")
	}

	client := conf.Client(oauth2.NoContext)
	client.Get("...")

	calendarService, err := calendar.New(client)
	if err != nil {
		return nil, fmt.Errorf("Unable to retrieve Calendar client: %v", err)
	}

	response, err := t.handleRawQuery(calendarService, tsdbReq)
	if err != nil {
		return &datasource.DatasourceResponse{
			Results: []*datasource.QueryResult{
				&datasource.QueryResult{
					Error: err.Error(),
				},
			},
		}, nil
	}
	return response, nil
}

func (t *GoogleCalendarDatasource) handleRawQuery(calendarService *calendar.Service, tsdbReq *datasource.DatasourceRequest) (*datasource.DatasourceResponse, error) {
	response := &datasource.DatasourceResponse{}

	var req EventsListRequest
	if err := json.Unmarshal([]byte(tsdbReq.Queries[0].ModelJson), &req); err != nil {
		return nil, err
	}

	eventsListCall := calendarService.Events.List(req.CalendarId).
		TimeMin(req.TimeMin).TimeMax(req.TimeMax).ShowDeleted(req.ShowDeleted).
		SingleEvents(req.SingleEvents).MaxResults(req.MaxResults).OrderBy(req.OrderBy)
	if req.Q != "" {
		eventsListCall = eventsListCall.Q(req.Q)
	}
	if req.SharedExtendedProperty != "" {
		eventsListCall = eventsListCall.SharedExtendedProperty(req.SharedExtendedProperty)
	}
	events, err := eventsListCall.Do()
	if err != nil {
		return nil, fmt.Errorf("Unable to retrieve events: %v", err)
	}

	resultJson, err := json.Marshal(events)
	if err != nil {
		return nil, err
	}

	response.Results = []*datasource.QueryResult{
		&datasource.QueryResult{
			MetaJson: string(resultJson),
		},
	}

	return response, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Jesse Meek <https://github.com/waigani>
// This program is Free Software see LICENSE file for details.

package diffparser

import (
	"regexp"
	"strconv"
	"strings"

	"errors"
)

// FileMode represents the file status in a diff
type FileMode int

const (
	// DELETED if the file is deleted
	DELETED FileMode = iota
	// MODIFIED if the file is modified
	MODIFIED
	// NEW if the file is created and there is no diff
	NEW
)

type diffRange struct {

	// starting line number
	Start int

	// the number of lines the change diffHunk applies to
	Length int

	// Each line of the hunk range.
	Lines []*DiffLine
}

// DiffLineMode tells the line if added, removed or unchanged
type DiffLineMode rune

const (
	// ADDED if the line is added (shown green in diff)
	ADDED DiffLineMode = iota
	// REMOVED if the line is deleted (shown red in diff)
	REMOVED
	// UNCHANGED if the line is unchanged (not colored in diff)
	UNCHANGED
)

// DiffLine is the least part of an actual diff
type DiffLine struct {
	Mode     DiffLineMode
	Number   int
	Content  string
	Position int // the line in the diff
}

// DiffHunk is a group of difflines
type DiffHunk struct {
	HunkHeader string
	OrigRange  diffRange
	NewRange   diffRange
	WholeRange diffRange
}

// DiffFile is the sum of diffhunks and holds the changes of the file features
type DiffFile struct {
	DiffHeader string
	Mode       FileMode
	OrigName   string
	NewName    string
	Hunks      []*DiffHunk
}

//Diff is the collection of DiffFiles
type Diff struct {
	Files []*DiffFile
	Raw   string `sql:"type:text"`

	PullID uint `sql:"index"`
}

func (d *Diff) addFile(file *DiffFile) {
	d.Files = append(d.Files, file)
}

// Changed returns a map of filename to lines changed in that file. Deleted
// files are ignored.
func (d *Diff) Changed() map[string][]int {
	dFiles := make(map[string][]int)

	for _, f := range d.Files {
		if f.Mode == DELETED {
			continue
		}

		for _, h := range f.Hunks {
			for _, dl := range h.NewRange.Lines {
				if dl.Mode == ADDED { // TODO(waigani) return removed
					dFiles[f.NewName] = append(dFiles[f.NewName], dl.Number)
				}
			}
		}
	}

	return dFiles
}

func regFind(s string, reg string, group int) string {
	re := regexp.MustCompile(reg)
	return re.FindStringSubmatch(s)[group]
}

func lineMode(line string) (*DiffLineMode, error) {
	var m DiffLineMode
	switch line[:1] {
	case " ":
		m = UNCHANGED
	case "+":
		m = ADDED
	case "-":
		m = REMOVED
	default:
		return nil, errors.New("could not parse line mode for line: \"" + line + "\"")
	}
	return &m, nil
}

// Parse takes a diff, such as produced by "git diff", and parses it into a
// Diff struct.
func Parse(diffString string) (*Diff, error) {
	var diff Diff
	diff.Raw = diffString
	lines := strings.Split(diffString, "\n")

	var file *DiffFile
	var hunk *DiffHunk
	var ADDEDCount int
	var REMOVEDCount int
	var inHunk bool
	oldFilePrefix := "--- a/"
	newFilePrefix := "+++ b/"

	var diffPosCount int
	var firstHunkInFile bool
	// Parse each line of diff.
	for idx, l := range lines {
		diffPosCount++
		switch {
		case strings.HasPrefix(l, "diff "):
			inHunk = false

			// Start a new file.
			file = &DiffFile{}
			header := l
			if len(lines) > idx+3 {
				rein := regexp.MustCompile(`^index .+$`)
				remp := regexp.MustCompile(`^(-|\+){3} .+$`)
				index := lines[idx+1]
				if rein.MatchString(index) {
					header = header + "\n" + index
				}
				mp1 := lines[idx+2]
				mp2 := lines[idx+3]
				if remp.MatchString(mp1) && remp.MatchString(mp2) {
					header = header + "\n" + mp1 + "\n" + mp2
				}
			}
			file.DiffHeader = header
			diff.Files = append(diff.Files, file)
			firstHunkInFile = true

			// File mode.
			file.Mode = MODIFIED
		case l == "+++ /dev/null":
			file.Mode = DELETED
		case l == "--- /dev/null":
			file.Mode = NEW
		case strings.HasPrefix(l, oldFilePrefix):
			file.OrigName = strings.TrimPrefix(l, oldFilePrefix)
		case strings.HasPrefix(l, newFilePrefix):
			file.NewName = strings.TrimPrefix(l, newFilePrefix)
		case strings.HasPrefix(l, "@@ "):
			if firstHunkInFile {
				diffPosCount = 0
				firstHunkInFile = false
			}

			inHunk = true
			// Start new hunk.
			hunk = &DiffHunk{}
			file.Hunks = append(file.Hunks, hunk)

			// Parse hunk heading for ranges
			re := regexp.MustCompile(`@@ \-(\d+),?(\d+)? \+(\d+),?(\d+)? @@ ?(.+)?`)
			m := re.FindStringSubmatch(l)
			if len(m) < 5 {
				return nil, errors.New("Error parsing line: " + l)
			}
			a, err := strconv.Atoi(m[1])
			if err != nil {
				return nil, err
			}
			b := a
			if len(m[2]) > 0 {
				b, err = strconv.Atoi(m[2])
				if err != nil {
					return nil, err
				}
			}
			c, err := strconv.Atoi(m[3])
			if err != nil {
				return nil, err
			}
			d := c
			if len(m[4]) > 0 {
				d, err = strconv.Atoi(m[4])
				if err != nil {
					return nil, err
				}
			}
			if len(m[5]) > 0 {
				hunk.HunkHeader = m[5]
			}

			// hunk orig range.
			hunk.OrigRange = diffRange{
				Start:  a,
				Length: b,
			}

			// hunk new range.
			hunk.NewRange = diffRange{
				Start:  c,
				Length: d,
			}

			// (re)set line counts
			ADDEDCount = hunk.NewRange.Start
			REMOVEDCount = hunk.OrigRange.Start
		case inHunk && isSourceLine(l):
			m, err := lineMode(l)
			if err != nil {
				return nil, err
			}
			line := DiffLine{
				Mode:     *m,
				Content:  l[1:],
				Position: diffPosCount,
			}
			newLine := line
			origLine := line

			// add lines to ranges
			switch *m {
			case ADDED:
				newLine.Number = ADDEDCount
				hunk.NewRange.Lines = append(hunk.NewRange.Lines, &newLine)
				hunk.WholeRange.Lines = append(hunk.WholeRange.Lines, &newLine)
				ADDEDCount++

			case REMOVED:
				origLine.Number = REMOVEDCount
				hunk.OrigRange.Lines = append(hunk.OrigRange.Lines, &origLine)
				hunk.WholeRange.Lines = append(hunk.WholeRange.Lines, &origLine)
				REMOVEDCount++

			case UNCHANGED:
				newLine.Number = ADDEDCount
				hunk.NewRange.Lines = append(hunk.NewRange.Lines, &newLine)
				hunk.WholeRange.Lines = append(hunk.WholeRange.Lines, &newLine)
				origLine.Number = REMOVEDCount
				hunk.OrigRange.Lines = append(hunk.OrigRange.Lines, &origLine)
				ADDEDCount++
				REMOVEDCount++
			}
		}
	}

	return &diff, nil
}

func isSourceLine(line string) bool {
	if line == `\ No newline at end of file` {
		return false
	}
	if l := len(line); l == 0 || (l >= 3 && (line[:3] == "---" || line[:3] == "+++")) {
		return false
	}
	return true
}

// Length returns the hunks line length
func (hunk *DiffHunk) Length() int {
	return len(hunk.WholeRange.Lines) + 1
}
<commit_msg>Make diffRange public<commit_after>// Copyright (c) 2015 Jesse Meek <https://github.com/waigani>
// This program is Free Software see LICENSE file for details.

package diffparser

import (
	"regexp"
	"strconv"
	"strings"

	"errors"
)

// FileMode represents the file status in a diff
type FileMode int

const (
	// DELETED if the file is deleted
	DELETED FileMode = iota
	// MODIFIED if the file is modified
	MODIFIED
	// NEW if the file is created and there is no diff
	NEW
)

type DiffRange struct {

	// starting line number
	Start int

	// the number of lines the change diffHunk applies to
	Length int

	// Each line of the hunk range.
	Lines []*DiffLine
}

// DiffLineMode tells the line if added, removed or unchanged
type DiffLineMode rune

const (
	// ADDED if the line is added (shown green in diff)
	ADDED DiffLineMode = iota
	// REMOVED if the line is deleted (shown red in diff)
	REMOVED
	// UNCHANGED if the line is unchanged (not colored in diff)
	UNCHANGED
)

// DiffLine is the least part of an actual diff
type DiffLine struct {
	Mode     DiffLineMode
	Number   int
	Content  string
	Position int // the line in the diff
}

// DiffHunk is a group of difflines
type DiffHunk struct {
	HunkHeader string
	OrigRange  DiffRange
	NewRange   DiffRange
	WholeRange DiffRange
}

// DiffFile is the sum of diffhunks and holds the changes of the file features
type DiffFile struct {
	DiffHeader string
	Mode       FileMode
	OrigName   string
	NewName    string
	Hunks      []*DiffHunk
}

//Diff is the collection of DiffFiles
type Diff struct {
	Files []*DiffFile
	Raw   string `sql:"type:text"`

	PullID uint `sql:"index"`
}

func (d *Diff) addFile(file *DiffFile) {
	d.Files = append(d.Files, file)
}

// Changed returns a map of filename to lines changed in that file. Deleted
// files are ignored.
func (d *Diff) Changed() map[string][]int {
	dFiles := make(map[string][]int)

	for _, f := range d.Files {
		if f.Mode == DELETED {
			continue
		}

		for _, h := range f.Hunks {
			for _, dl := range h.NewRange.Lines {
				if dl.Mode == ADDED { // TODO(waigani) return removed
					dFiles[f.NewName] = append(dFiles[f.NewName], dl.Number)
				}
			}
		}
	}

	return dFiles
}

func regFind(s string, reg string, group int) string {
	re := regexp.MustCompile(reg)
	return re.FindStringSubmatch(s)[group]
}

func lineMode(line string) (*DiffLineMode, error) {
	var m DiffLineMode
	switch line[:1] {
	case " ":
		m = UNCHANGED
	case "+":
		m = ADDED
	case "-":
		m = REMOVED
	default:
		return nil, errors.New("could not parse line mode for line: \"" + line + "\"")
	}
	return &m, nil
}

// Parse takes a diff, such as produced by "git diff", and parses it into a
// Diff struct.
func Parse(diffString string) (*Diff, error) {
	var diff Diff
	diff.Raw = diffString
	lines := strings.Split(diffString, "\n")

	var file *DiffFile
	var hunk *DiffHunk
	var ADDEDCount int
	var REMOVEDCount int
	var inHunk bool
	oldFilePrefix := "--- a/"
	newFilePrefix := "+++ b/"

	var diffPosCount int
	var firstHunkInFile bool
	// Parse each line of diff.
	for idx, l := range lines {
		diffPosCount++
		switch {
		case strings.HasPrefix(l, "diff "):
			inHunk = false

			// Start a new file.
			file = &DiffFile{}
			header := l
			if len(lines) > idx+3 {
				rein := regexp.MustCompile(`^index .+$`)
				remp := regexp.MustCompile(`^(-|\+){3} .+$`)
				index := lines[idx+1]
				if rein.MatchString(index) {
					header = header + "\n" + index
				}
				mp1 := lines[idx+2]
				mp2 := lines[idx+3]
				if remp.MatchString(mp1) && remp.MatchString(mp2) {
					header = header + "\n" + mp1 + "\n" + mp2
				}
			}
			file.DiffHeader = header
			diff.Files = append(diff.Files, file)
			firstHunkInFile = true

			// File mode.
			file.Mode = MODIFIED
		case l == "+++ /dev/null":
			file.Mode = DELETED
		case l == "--- /dev/null":
			file.Mode = NEW
		case strings.HasPrefix(l, oldFilePrefix):
			file.OrigName = strings.TrimPrefix(l, oldFilePrefix)
		case strings.HasPrefix(l, newFilePrefix):
			file.NewName = strings.TrimPrefix(l, newFilePrefix)
		case strings.HasPrefix(l, "@@ "):
			if firstHunkInFile {
				diffPosCount = 0
				firstHunkInFile = false
			}

			inHunk = true
			// Start new hunk.
			hunk = &DiffHunk{}
			file.Hunks = append(file.Hunks, hunk)

			// Parse hunk heading for ranges
			re := regexp.MustCompile(`@@ \-(\d+),?(\d+)? \+(\d+),?(\d+)? @@ ?(.+)?`)
			m := re.FindStringSubmatch(l)
			if len(m) < 5 {
				return nil, errors.New("Error parsing line: " + l)
			}
			a, err := strconv.Atoi(m[1])
			if err != nil {
				return nil, err
			}
			b := a
			if len(m[2]) > 0 {
				b, err = strconv.Atoi(m[2])
				if err != nil {
					return nil, err
				}
			}
			c, err := strconv.Atoi(m[3])
			if err != nil {
				return nil, err
			}
			d := c
			if len(m[4]) > 0 {
				d, err = strconv.Atoi(m[4])
				if err != nil {
					return nil, err
				}
			}
			if len(m[5]) > 0 {
				hunk.HunkHeader = m[5]
			}

			// hunk orig range.
			hunk.OrigRange = DiffRange{
				Start:  a,
				Length: b,
			}

			// hunk new range.
			hunk.NewRange = DiffRange{
				Start:  c,
				Length: d,
			}

			// (re)set line counts
			ADDEDCount = hunk.NewRange.Start
			REMOVEDCount = hunk.OrigRange.Start
		case inHunk && isSourceLine(l):
			m, err := lineMode(l)
			if err != nil {
				return nil, err
			}
			line := DiffLine{
				Mode:     *m,
				Content:  l[1:],
				Position: diffPosCount,
			}
			newLine := line
			origLine := line

			// add lines to ranges
			switch *m {
			case ADDED:
				newLine.Number = ADDEDCount
				hunk.NewRange.Lines = append(hunk.NewRange.Lines, &newLine)
				hunk.WholeRange.Lines = append(hunk.WholeRange.Lines, &newLine)
				ADDEDCount++

			case REMOVED:
				origLine.Number = REMOVEDCount
				hunk.OrigRange.Lines = append(hunk.OrigRange.Lines, &origLine)
				hunk.WholeRange.Lines = append(hunk.WholeRange.Lines, &origLine)
				REMOVEDCount++

			case UNCHANGED:
				newLine.Number = ADDEDCount
				hunk.NewRange.Lines = append(hunk.NewRange.Lines, &newLine)
				hunk.WholeRange.Lines = append(hunk.WholeRange.Lines, &newLine)
				origLine.Number = REMOVEDCount
				hunk.OrigRange.Lines = append(hunk.OrigRange.Lines, &origLine)
				ADDEDCount++
				REMOVEDCount++
			}
		}
	}

	return &diff, nil
}

func isSourceLine(line string) bool {
	if line == `\ No newline at end of file` {
		return false
	}
	if l := len(line); l == 0 || (l >= 3 && (line[:3] == "---" || line[:3] == "+++")) {
		return false
	}
	return true
}

// Length returns the hunks line length
func (hunk *DiffHunk) Length() int {
	return len(hunk.WholeRange.Lines) + 1
}
<|endoftext|>


<commit_before>package mydumpster

import (
	"fmt"
	"strings"
)

type Trigger struct {
	TableDst      Table
	TableSrcName  string
	TableSrcField string
	TableDstField string
	DumpAll       bool
}

func (t *Trigger) SelectQueryFromRowsStr(rows [][]string, columns []string) string {
	// get position
	pos := SearchStr(columns, t.TableSrcField)

	// Get all the identifiers
	ids := make([]string, len(rows))
	for i, v := range rows {
		ids[i] = v[pos]
	}

	return t.SelectQueryStr(ids)
}

func (t *Trigger) SelectQueryStr(ids []string) string {
	return fmt.Sprintf(IN_FMT, t.TableDstField, strings.Join(ids, ", "))
}
<commit_msg>Fix garbage in list when triggered destination key is null<commit_after>package mydumpster

import (
	"fmt"
	"strings"
)

type Trigger struct {
	TableDst      Table
	TableSrcName  string
	TableSrcField string
	TableDstField string
	DumpAll       bool
}

func (t *Trigger) SelectQueryFromRowsStr(rows [][]string, columns []string) string {
	// get position
	pos := SearchStr(columns, t.TableSrcField)

	// Get all the identifiers
	ids := make([]string, 0)
	for _, v := range rows {
		if v[pos] != NULL && len(v[pos]) > 0 {
			ids = append(ids, v[pos])
		}
	}

	return t.SelectQueryStr(ids)
}

func (t *Trigger) SelectQueryStr(ids []string) string {
	return fmt.Sprintf(IN_FMT, t.TableDstField, strings.Join(ids, ", "))
}
<|endoftext|>