<commit_before>package exec

import "github.com/pkg/errors"

var (
	// ErrRuntimeUnsupported encountered when a task requires a runtime
	// unsupported by the executor.
	ErrRuntimeUnsupported = errors.New("exec: unsupported runtime")

	// ErrTaskPrepared is called if the task is already prepared.
	ErrTaskPrepared = errors.New("exec: task already prepared")

	// ErrTaskStarted can be returned from any operation that cannot be
	// performed because the task has already been started. This does not imply
	// that the task is running but rather that it is no longer valid to call
	// Start.
	ErrTaskStarted = errors.New("exec: task already started")

	// ErrTaskUpdateRejected is returned if a task update is rejected by a controller.
	ErrTaskUpdateRejected = errors.New("exec: task update rejected")

	// ErrControllerClosed returned when a task controller has been closed.
	ErrControllerClosed = errors.New("exec: controller closed")

	// ErrTaskRetry is returned by Do when an operation failed by should be
	// retried. The status should still be reported in this case.
	ErrTaskRetry = errors.New("exec: task retry")

	// ErrTaskNoop returns when the a subsequent call to Do will not result in
	// advancing the task. Callers should avoid calling Do until the task has been updated.
	ErrTaskNoop = errors.New("exec: task noop")

	// ErrDependencyNotReady is returned if a given dependency can be accessed
	// through the Getter, but is not yet ready to be used. This is most
	// relevant for Volumes, which must be staged and published on the node.
	ErrDependencyNotReady error = errors.New("dependency not ready")
)

// ExitCoder is implemented by errors that have an exit code.
type ExitCoder interface {
	// ExitCode returns the exit code.
	ExitCode() int
}

// Temporary indicates whether or not the error condition is temporary.
//
// If this is encountered in the controller, the failing operation will be
// retried when this returns true. Otherwise, the operation is considered
// fatal.
type Temporary interface {
	Temporary() bool
}

// MakeTemporary makes the error temporary.
func MakeTemporary(err error) error {
	if IsTemporary(err) {
		return err
	}

	return temporary{err}
}

type temporary struct {
	error
}

func (t temporary) Cause() error    { return t.error }
func (t temporary) Temporary() bool { return true }

// IsTemporary returns true if the error or a recursive cause returns true for
// temporary.
func IsTemporary(err error) bool {
	for err != nil {
		if tmp, ok := err.(Temporary); ok && tmp.Temporary() {
			return true
		}

		cause := errors.Cause(err)
		if cause == err {
			break
		}

		err = cause
	}

	return false
}
<commit_msg>Fix panic in agent<commit_after>package exec

import "github.com/pkg/errors"

var (
	// ErrRuntimeUnsupported encountered when a task requires a runtime
	// unsupported by the executor.
	ErrRuntimeUnsupported = errors.New("exec: unsupported runtime")

	// ErrTaskPrepared is called if the task is already prepared.
	ErrTaskPrepared = errors.New("exec: task already prepared")

	// ErrTaskStarted can be returned from any operation that cannot be
	// performed because the task has already been started. This does not imply
	// that the task is running but rather that it is no longer valid to call
	// Start.
	ErrTaskStarted = errors.New("exec: task already started")

	// ErrTaskUpdateRejected is returned if a task update is rejected by a controller.
	ErrTaskUpdateRejected = errors.New("exec: task update rejected")

	// ErrControllerClosed returned when a task controller has been closed.
	ErrControllerClosed = errors.New("exec: controller closed")

	// ErrTaskRetry is returned by Do when an operation failed by should be
	// retried. The status should still be reported in this case.
	ErrTaskRetry = errors.New("exec: task retry")

	// ErrTaskNoop returns when the a subsequent call to Do will not result in
	// advancing the task. Callers should avoid calling Do until the task has been updated.
	ErrTaskNoop = errors.New("exec: task noop")

	// ErrDependencyNotReady is returned if a given dependency can be accessed
	// through the Getter, but is not yet ready to be used. This is most
	// relevant for Volumes, which must be staged and published on the node.
	ErrDependencyNotReady error = errors.New("dependency not ready")
)

// ExitCoder is implemented by errors that have an exit code.
type ExitCoder interface {
	// ExitCode returns the exit code.
	ExitCode() int
}

// Temporary indicates whether or not the error condition is temporary.
//
// If this is encountered in the controller, the failing operation will be
// retried when this returns true. Otherwise, the operation is considered
// fatal.
type Temporary interface {
	Temporary() bool
}

// MakeTemporary makes the error temporary.
func MakeTemporary(err error) error {
	if IsTemporary(err) {
		return err
	}

	return temporary{err}
}

type temporary struct {
	error
}

func (t temporary) Cause() error    { return t.error }
func (t temporary) Temporary() bool { return true }

// IsTemporary returns true if the error or a recursive cause returns true for
// temporary.
func IsTemporary(err error) bool {
	if tmp, ok := err.(Temporary); ok && tmp.Temporary() {
		return true
	}

	cause := errors.Cause(err)

	if tmp, ok := cause.(Temporary); ok && tmp.Temporary() {
		return true
	}

	return false
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	restful "github.com/emicklei/go-restful"
	"github.com/emicklei/go-restful/swagger"
	"github.com/garyburd/redigo/redis"
	"github.com/the42/ogdat/database"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

const AppID = "5bcbfc24-8e7e-4105-99c4-dd47e7e5094a"
const watcherappid = "a6545f8f-e0c9-4917-83c7-3e47bd1e0247"

var logger *log.Logger

type analyser struct {
	dbcon analyserdb
	pool  *redis.Pool
}

func NewAnalyser(dbcon *sql.DB, pool *redis.Pool) *analyser {
	analyser := &analyser{dbcon: analyserdb{DBConn: database.DBConn{Appid: AppID, DBer: dbcon}}, pool: pool}
	return analyser
}

func onlyweb() bool {
	boolval, err := strconv.ParseBool(os.Getenv("ONLYWEB"))
	if err != nil {
		return false
	}
	return boolval
}

func getredisconnect() string {
	const redisurl = "REDISCLOUD_URL"
	const redisdb = "ANALYSER_REDISDB"

	return os.Getenv(redisurl) + "/" + os.Getenv(redisdb)
}

func getheartbeatinterval() int {

	if i, err := strconv.Atoi(os.Getenv("HEARTBEAT_INTERVAL")); err == nil {
		return i
	}
	return 60 // Minutes
}

func heartbeat(interval int) chan bool {
	retchan := make(chan bool)
	go func() {
		for {
			dbconn, err := database.GetDatabaseConnection()
			if err != nil {
				logger.Panicln(err)
			}
			db := &database.DBConn{DBer: dbconn, Appid: AppID}
			if err := db.HeartBeat(); err != nil {
				logger.Panicln(err)
			}
			dbconn.Close()
			logger.Printf("Watchdog beating every %d minute\n", interval)
			retchan <- true
			time.Sleep(time.Duration(interval) * time.Minute)
		}
	}()
	return retchan
}

func main() {
	dbcon, err := database.GetDatabaseConnection()
	if err != nil {
		logger.Panicln(err)
	}
	defer dbcon.Close()
	analyser := NewAnalyser(dbcon, redis.NewPool(func() (redis.Conn, error) { return database.GetRedisConnection(getredisconnect()) }, 10))

	var datachange, urlchange chan []byte
	var heartbeatchannel chan bool

	if !onlyweb() {
		heartbeatchannel = heartbeat(getheartbeatinterval())

		<-heartbeatchannel // Wait for the first heartbeat, so the logging in the database is properly set up
		if err := analyser.populatedatasetinfo(); err != nil {
			logger.Panicln(err)
		}
		datachange = analyser.listenredischannel(watcherappid + ":DataChange")
		urlchange = analyser.listenredischannel(watcherappid + ":UrlChange")
	}

	restful.DefaultResponseMimeType = restful.MIME_JSON
	restful.Add(NewAnalyseOGDATRESTService(analyser))

	config := swagger.Config{
		WebServicesUrl:  "http://localhost:8080",
		ApiPath:         "/apidoc",
		SwaggerPath:     "/doc/v1/",
		SwaggerFilePath: "swagger-ui/dist/",
		WebServices:     restful.RegisteredWebServices()} // you control what services are visible
	swagger.InstallSwaggerService(config)

	go logger.Fatal(http.ListenAndServe(":8080", nil))

	if !onlyweb() {
		for {
			select {
			case <-urlchange:
			case <-datachange:
				if err := analyser.populatedatasetinfo(); err != nil {
					logger.Panicln(err)
				}
			case <-heartbeatchannel:
				logger.Println("Idle")
			}
		}
	}
}

func init() {
	logger = log.New(os.Stderr, filepath.Base(os.Args[0])+": ", log.LstdFlags)
}
<commit_msg>DRY - use an anoynmous function to prevent repetition in a select case<commit_after>package main

import (
	"database/sql"
	restful "github.com/emicklei/go-restful"
	"github.com/emicklei/go-restful/swagger"
	"github.com/garyburd/redigo/redis"
	"github.com/the42/ogdat/database"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

const AppID = "5bcbfc24-8e7e-4105-99c4-dd47e7e5094a"
const watcherappid = "a6545f8f-e0c9-4917-83c7-3e47bd1e0247"

var logger *log.Logger

type analyser struct {
	dbcon analyserdb
	pool  *redis.Pool
}

func NewAnalyser(dbcon *sql.DB, pool *redis.Pool) *analyser {
	analyser := &analyser{dbcon: analyserdb{DBConn: database.DBConn{Appid: AppID, DBer: dbcon}}, pool: pool}
	return analyser
}

func onlyweb() bool {
	boolval, err := strconv.ParseBool(os.Getenv("ONLYWEB"))
	if err != nil {
		return false
	}
	return boolval
}

func getredisconnect() string {
	const redisurl = "REDISCLOUD_URL"
	const redisdb = "ANALYSER_REDISDB"

	return os.Getenv(redisurl) + "/" + os.Getenv(redisdb)
}

func getheartbeatinterval() int {

	if i, err := strconv.Atoi(os.Getenv("HEARTBEAT_INTERVAL")); err == nil {
		return i
	}
	return 60 // Minutes
}

func heartbeat(interval int) chan bool {
	retchan := make(chan bool)
	go func() {
		for {
			dbconn, err := database.GetDatabaseConnection()
			if err != nil {
				logger.Panicln(err)
			}
			db := &database.DBConn{DBer: dbconn, Appid: AppID}
			if err := db.HeartBeat(); err != nil {
				logger.Panicln(err)
			}
			dbconn.Close()
			logger.Printf("Watchdog beating every %d minute\n", interval)
			retchan <- true
			time.Sleep(time.Duration(interval) * time.Minute)
		}
	}()
	return retchan
}

func main() {
	dbcon, err := database.GetDatabaseConnection()
	if err != nil {
		logger.Panicln(err)
	}
	defer dbcon.Close()
	analyser := NewAnalyser(dbcon, redis.NewPool(func() (redis.Conn, error) { return database.GetRedisConnection(getredisconnect()) }, 10))

	var datachange, urlchange chan []byte
	var heartbeatchannel chan bool

	if !onlyweb() {
		heartbeatchannel = heartbeat(getheartbeatinterval())

		<-heartbeatchannel // Wait for the first heartbeat, so the logging in the database is properly set up
		if err := analyser.populatedatasetinfo(); err != nil {
			logger.Panicln(err)
		}
		datachange = analyser.listenredischannel(watcherappid + ":DataChange")
		urlchange = analyser.listenredischannel(watcherappid + ":UrlChange")
	}

	restful.DefaultResponseMimeType = restful.MIME_JSON
	restful.Add(NewAnalyseOGDATRESTService(analyser))

	config := swagger.Config{
		WebServicesUrl:  "http://localhost:8080",
		ApiPath:         "/apidoc",
		SwaggerPath:     "/doc/v1/",
		SwaggerFilePath: "swagger-ui/dist/",
		WebServices:     restful.RegisteredWebServices()} // you control what services are visible
	swagger.InstallSwaggerService(config)

	go logger.Fatal(http.ListenAndServe(":8080", nil))

	if !onlyweb() {
		populatedatasetinfo := func() {
			if err := analyser.populatedatasetinfo(); err != nil {
				logger.Panicln(err)
			}
		}
		for {
			select {
			case <-urlchange:
				populatedatasetinfo()
			case <-datachange:
				populatedatasetinfo()
			case <-heartbeatchannel:
				logger.Println("Idle")
			}
		}
	}
}

func init() {
	logger = log.New(os.Stderr, filepath.Base(os.Args[0])+": ", log.LstdFlags)
}
<|endoftext|>


<commit_before>package pinata_test

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/robbiev/pinata"
)

const str = "test string"

func ExamplePinata() {
	const message = `
	{
		"Name": "Kevin",
		"Phone": ["+44 20 7123 4567", "+44 20 4567 7123"],
		"Address": {
			"Street": "1 Gopher Road",
			"City": "Gophertown"
		}
	}`

	var m map[string]interface{}

	err := json.Unmarshal([]byte(message), &m)
	if err != nil {
		fmt.Println(err)
		return
	}

	stick, pinata := pinata.New(m)

	type gopher struct {
		Name  string
		Phone string
		City  string
	}

	// no error handling here
	kevin := gopher{
		Name:  stick.PathString(pinata, "Name"),
		Phone: stick.IndexString(stick.PathPinata(pinata, "Phone"), 0),
		City:  stick.PathString(pinata, "Address", "City"),
	}

	if err := stick.Error(); err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println("Name:", kevin.Name)
	fmt.Println("Phone:", kevin.Phone)
	fmt.Println("City:", kevin.City)

	// Output:
	// Name: Kevin
	// Phone: +44 20 7123 4567
	// City: Gophertown
}

func TestPinata(t *testing.T) {
	const message = `
	{
		"Name": "Kevin",
		"Phone": ["+44 20 7123 4567", "+44 20 4567 7123"],
		"Address": {
			"Street": "1 Gopher Road",
			"City": "Gophertown"
		}
	}`

	var m map[string]interface{}

	err := json.Unmarshal([]byte(message), &m)
	if err != nil {
		fmt.Println(err)
		return
	}

	stick, pinata := pinata.New(m)

	stick.PathString(pinata, "Phone")
	if err := stick.Error(); err != nil {
		fmt.Println(err)
		stick.ClearError()
	}
	stick.IndexString(stick.PathPinata(pinata, "Phone"), 3)
	if err := stick.Error(); err != nil {
		fmt.Println(err)
		stick.ClearError()
	}
	stick.PathString(pinata, "Address", "City", "Town")
	if err := stick.Error(); err != nil {
		fmt.Println(err)
		stick.ClearError()
	}
}
<commit_msg>move the example to where it belongs<commit_after>package pinata_test

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/robbiev/pinata"
)

const str = "test string"

func ExampleStick() {
	const message = `
	{
		"Name": "Kevin",
		"Phone": ["+44 20 7123 4567", "+44 20 4567 7123"],
		"Address": {
			"Street": "1 Gopher Road",
			"City": "Gophertown"
		}
	}`

	var m map[string]interface{}

	err := json.Unmarshal([]byte(message), &m)
	if err != nil {
		fmt.Println(err)
		return
	}

	stick, pinata := pinata.New(m)

	type gopher struct {
		Name  string
		Phone string
		City  string
	}

	// no error handling here
	kevin := gopher{
		Name:  stick.PathString(pinata, "Name"),
		Phone: stick.IndexString(stick.PathPinata(pinata, "Phone"), 0),
		City:  stick.PathString(pinata, "Address", "City"),
	}

	if err := stick.Error(); err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println("Name:", kevin.Name)
	fmt.Println("Phone:", kevin.Phone)
	fmt.Println("City:", kevin.City)

	// Output:
	// Name: Kevin
	// Phone: +44 20 7123 4567
	// City: Gophertown
}

func TestPinata(t *testing.T) {
	const message = `
	{
		"Name": "Kevin",
		"Phone": ["+44 20 7123 4567", "+44 20 4567 7123"],
		"Address": {
			"Street": "1 Gopher Road",
			"City": "Gophertown"
		}
	}`

	var m map[string]interface{}

	err := json.Unmarshal([]byte(message), &m)
	if err != nil {
		fmt.Println(err)
		return
	}

	stick, pinata := pinata.New(m)

	stick.PathString(pinata, "Phone")
	if err := stick.Error(); err != nil {
		fmt.Println(err)
		stick.ClearError()
	}
	stick.IndexString(stick.PathPinata(pinata, "Phone"), 3)
	if err := stick.Error(); err != nil {
		fmt.Println(err)
		stick.ClearError()
	}
	stick.PathString(pinata, "Address", "City", "Town")
	if err := stick.Error(); err != nil {
		fmt.Println(err)
		stick.ClearError()
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"

	"github.com/sirupsen/logrus"

	"github.com/containerum/chkit/pkg/cli/deployment"
	"github.com/containerum/chkit/pkg/cli/namespace"
	"github.com/containerum/chkit/pkg/cli/prerun"
	"github.com/containerum/chkit/pkg/cli/service"
	"github.com/containerum/chkit/pkg/configuration"
	"github.com/containerum/chkit/pkg/context"
	"github.com/spf13/cobra"
)

var Get = &cobra.Command{
	Use: "get",
	PersistentPreRun: func(command *cobra.Command, args []string) {
		prerun.PreRun()
	},
	Run: func(command *cobra.Command, args []string) {
		command.Help()
	},
	PersistentPostRun: func(command *cobra.Command, args []string) {
		if context.GlobalContext.Changed {
			if err := configuration.SaveConfig(); err != nil {
				logrus.WithError(err).Errorf("unable to save config")
				fmt.Printf("Unable to save config: %v\n", err)
				return
			}
		}
		if err := configuration.SaveTokens(context.GlobalContext.Client.Tokens); err != nil {
			logrus.WithError(err).Errorf("unable to save tokens")
			fmt.Printf("Unable to save tokens: %v\n", err)
			return
		}
	},
}

func init() {
	Get.AddCommand(
		clideployment.Get,
		clinamespace.Get,
		cliserv.Get,
	)
}
<commit_msg>add get pod command<commit_after>package cli

import (
	"fmt"

	"github.com/sirupsen/logrus"

	"github.com/containerum/chkit/pkg/cli/deployment"
	"github.com/containerum/chkit/pkg/cli/namespace"
	"github.com/containerum/chkit/pkg/cli/pod"
	"github.com/containerum/chkit/pkg/cli/prerun"
	"github.com/containerum/chkit/pkg/cli/service"
	"github.com/containerum/chkit/pkg/configuration"
	"github.com/containerum/chkit/pkg/context"
	"github.com/spf13/cobra"
)

var Get = &cobra.Command{
	Use:   "get",
	Short: "Delete resource data",
	PersistentPreRun: func(command *cobra.Command, args []string) {
		prerun.PreRun()
	},
	Run: func(command *cobra.Command, args []string) {
		command.Help()
	},
	PersistentPostRun: func(command *cobra.Command, args []string) {
		if context.GlobalContext.Changed {
			if err := configuration.SaveConfig(); err != nil {
				logrus.WithError(err).Errorf("unable to save config")
				fmt.Printf("Unable to save config: %v\n", err)
				return
			}
		}
		if err := configuration.SaveTokens(context.GlobalContext.Client.Tokens); err != nil {
			logrus.WithError(err).Errorf("unable to save tokens")
			fmt.Printf("Unable to save tokens: %v\n", err)
			return
		}
	},
}

func init() {
	Get.AddCommand(
		clideployment.Get,
		clinamespace.Get,
		cliserv.Get,
		clipod.Get(&context.GlobalContext),
	)
}
<|endoftext|>


<commit_before>package log

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
)

// Logger specific flags
var (
	debug = flag.Bool("debug", false, "allow for verbose logging (DEBUG env can be used as well)")
)

// Infof logs info messages in verbose mode
func Infof(format string, args ...interface{}) {
	if !*debug {
		return // only in debug mode
	}

	log.Printf(message("INFO", format), args...)
}

// Warningf logs an error that can be recovered from
func Warningf(format string, args ...interface{}) {
	log.Printf(message("WARNING", format), args...)
}

// Errorf logs an error that can't be recovered from and thus is fatal
func Errorf(format string, args ...interface{}) {
	log.Fatalf(message("ERROR", format), args...)
}

// Init configures the log
func Init() {
	*debug = *debug || (os.Getenv("DEBUG") != "")
}

// message creates a informational log message
func message(level, message string) string {
	// collect fileName and lineNumber of callee
	_, fn, ln, _ := runtime.Caller(2)
	// return formatted format string
	return fmt.Sprintf("[%s] {%s:%d} %s", level, fn, ln, message)
}
<commit_msg>demote warning to verbose-mode<commit_after>package log

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
)

// Logger specific flags
var (
	debug = flag.Bool("debug", false, "allow for verbose logging (DEBUG env can be used as well)")
)

// Infof logs info messages in verbose mode
func Infof(format string, args ...interface{}) {
	if !*debug {
		return // only in debug mode
	}

	log.Printf(message("INFO", format), args...)
}

// Warningf logs an error that can be recovered from
func Warningf(format string, args ...interface{}) {
	if !*debug {
		return // only in debug mode
	}

	log.Printf(message("WARNING", format), args...)
}

// Errorf logs an error that can't be recovered from and thus is fatal
func Errorf(format string, args ...interface{}) {
	log.Fatalf(message("ERROR", format), args...)
}

// Init configures the log
func Init() {
	*debug = *debug || (os.Getenv("DEBUG") != "")
}

// message creates a informational log message
func message(level, message string) string {
	// collect fileName and lineNumber of callee
	_, fn, ln, _ := runtime.Caller(2)
	// return formatted format string
	return fmt.Sprintf("[%s] {%s:%d} %s", level, fn, ln, message)
}
<|endoftext|>


<commit_before>package sarama

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io/ioutil"

	"github.com/eapache/go-xerial-snappy"
	"github.com/pierrec/lz4"
)

const recordBatchOverhead = 49

type recordsArray []*Record

func (e recordsArray) encode(pe packetEncoder) error {
	for _, r := range e {
		if err := r.encode(pe); err != nil {
			return err
		}
	}
	return nil
}

func (e recordsArray) decode(pd packetDecoder) error {
	for i := range e {
		rec := &Record{}
		if err := rec.decode(pd); err != nil {
			return err
		}
		e[i] = rec
	}
	return nil
}

type RecordBatch struct {
	FirstOffset           int64
	PartitionLeaderEpoch  int32
	Version               int8
	Codec                 CompressionCodec
	Control               bool
	LastOffsetDelta       int32
	FirstTimestamp        int64
	MaxTimestamp          int64
	ProducerID            int64
	ProducerEpoch         int16
	FirstSequence         int32
	Records               []*Record
	PartialTrailingRecord bool

	compressedRecords []byte
	recordsLen        int // uncompressed records size
}

func (b *RecordBatch) encode(pe packetEncoder) error {
	if b.Version != 2 {
		return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)}
	}
	pe.putInt64(b.FirstOffset)
	pe.push(&lengthField{})
	pe.putInt32(b.PartitionLeaderEpoch)
	pe.putInt8(b.Version)
	pe.push(newCRC32Field(crcCastagnoli))
	pe.putInt16(b.computeAttributes())
	pe.putInt32(b.LastOffsetDelta)
	pe.putInt64(b.FirstTimestamp)
	pe.putInt64(b.MaxTimestamp)
	pe.putInt64(b.ProducerID)
	pe.putInt16(b.ProducerEpoch)
	pe.putInt32(b.FirstSequence)

	if err := pe.putArrayLength(len(b.Records)); err != nil {
		return err
	}

	if b.compressedRecords == nil {
		b.encodeRecords(pe)
	}
	if err := pe.putRawBytes(b.compressedRecords); err != nil {
		return err
	}

	if err := pe.pop(); err != nil {
		return err
	}
	return pe.pop()
}

func (b *RecordBatch) decode(pd packetDecoder) (err error) {
	if b.FirstOffset, err = pd.getInt64(); err != nil {
		return err
	}

	batchLen, err := pd.getInt32()
	if err != nil {
		return err
	}

	if b.PartitionLeaderEpoch, err = pd.getInt32(); err != nil {
		return err
	}

	if b.Version, err = pd.getInt8(); err != nil {
		return err
	}

	if err = pd.push(&crc32Field{polynomial: crcCastagnoli}); err != nil {
		return err
	}

	attributes, err := pd.getInt16()
	if err != nil {
		return err
	}
	b.Codec = CompressionCodec(int8(attributes) & compressionCodecMask)
	b.Control = attributes&controlMask == controlMask

	if b.LastOffsetDelta, err = pd.getInt32(); err != nil {
		return err
	}

	if b.FirstTimestamp, err = pd.getInt64(); err != nil {
		return err
	}

	if b.MaxTimestamp, err = pd.getInt64(); err != nil {
		return err
	}

	if b.ProducerID, err = pd.getInt64(); err != nil {
		return err
	}

	if b.ProducerEpoch, err = pd.getInt16(); err != nil {
		return err
	}

	if b.FirstSequence, err = pd.getInt32(); err != nil {
		return err
	}

	numRecs, err := pd.getArrayLength()
	if err != nil {
		return err
	}
	if numRecs >= 0 {
		b.Records = make([]*Record, numRecs)
	}

	bufSize := int(batchLen) - recordBatchOverhead
	recBuffer, err := pd.getRawBytes(bufSize)
	if err != nil {
		return err
	}

	if err = pd.pop(); err != nil {
		return err
	}

	switch b.Codec {
	case CompressionNone:
	case CompressionGZIP:
		reader, err := gzip.NewReader(bytes.NewReader(recBuffer))
		if err != nil {
			return err
		}
		if recBuffer, err = ioutil.ReadAll(reader); err != nil {
			return err
		}
	case CompressionSnappy:
		if recBuffer, err = snappy.Decode(recBuffer); err != nil {
			return err
		}
	case CompressionLZ4:
		reader := lz4.NewReader(bytes.NewReader(recBuffer))
		if recBuffer, err = ioutil.ReadAll(reader); err != nil {
			return err
		}
	default:
		return PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", b.Codec)}
	}

	err = decode(recBuffer, recordsArray(b.Records))
	if err == ErrInsufficientData {
		b.PartialTrailingRecord = true
		b.Records = nil
		return nil
	}
	return err
}

func (b *RecordBatch) encodeRecords(pe packetEncoder) error {
	var raw []byte
	if b.Codec != CompressionNone {
		var err error
		if raw, err = encode(recordsArray(b.Records), nil); err != nil {
			return err
		}
		b.recordsLen = len(raw)
	}

	switch b.Codec {
	case CompressionNone:
		offset := pe.offset()
		if err := recordsArray(b.Records).encode(pe); err != nil {
			return err
		}
		b.recordsLen = pe.offset() - offset
	case CompressionGZIP:
		var buf bytes.Buffer
		writer := gzip.NewWriter(&buf)
		if _, err := writer.Write(raw); err != nil {
			return err
		}
		if err := writer.Close(); err != nil {
			return err
		}
		b.compressedRecords = buf.Bytes()
	case CompressionSnappy:
		b.compressedRecords = snappy.Encode(raw)
	case CompressionLZ4:
		var buf bytes.Buffer
		writer := lz4.NewWriter(&buf)
		if _, err := writer.Write(raw); err != nil {
			return err
		}
		if err := writer.Close(); err != nil {
			return err
		}
		b.compressedRecords = buf.Bytes()
	default:
		return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)}
	}

	return nil
}

func (b *RecordBatch) computeAttributes() int16 {
	attr := int16(b.Codec) & int16(compressionCodecMask)
	if b.Control {
		attr |= controlMask
	}
	return attr
}

func (b *RecordBatch) addRecord(r *Record) {
	b.Records = append(b.Records, r)
}
<commit_msg>Check records encoding error<commit_after>package sarama

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io/ioutil"

	"github.com/eapache/go-xerial-snappy"
	"github.com/pierrec/lz4"
)

const recordBatchOverhead = 49

type recordsArray []*Record

func (e recordsArray) encode(pe packetEncoder) error {
	for _, r := range e {
		if err := r.encode(pe); err != nil {
			return err
		}
	}
	return nil
}

func (e recordsArray) decode(pd packetDecoder) error {
	for i := range e {
		rec := &Record{}
		if err := rec.decode(pd); err != nil {
			return err
		}
		e[i] = rec
	}
	return nil
}

type RecordBatch struct {
	FirstOffset           int64
	PartitionLeaderEpoch  int32
	Version               int8
	Codec                 CompressionCodec
	Control               bool
	LastOffsetDelta       int32
	FirstTimestamp        int64
	MaxTimestamp          int64
	ProducerID            int64
	ProducerEpoch         int16
	FirstSequence         int32
	Records               []*Record
	PartialTrailingRecord bool

	compressedRecords []byte
	recordsLen        int // uncompressed records size
}

func (b *RecordBatch) encode(pe packetEncoder) error {
	if b.Version != 2 {
		return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)}
	}
	pe.putInt64(b.FirstOffset)
	pe.push(&lengthField{})
	pe.putInt32(b.PartitionLeaderEpoch)
	pe.putInt8(b.Version)
	pe.push(newCRC32Field(crcCastagnoli))
	pe.putInt16(b.computeAttributes())
	pe.putInt32(b.LastOffsetDelta)
	pe.putInt64(b.FirstTimestamp)
	pe.putInt64(b.MaxTimestamp)
	pe.putInt64(b.ProducerID)
	pe.putInt16(b.ProducerEpoch)
	pe.putInt32(b.FirstSequence)

	if err := pe.putArrayLength(len(b.Records)); err != nil {
		return err
	}

	if b.compressedRecords == nil {
		if err := b.encodeRecords(pe); err != nil {
			return err
		}
	}
	if err := pe.putRawBytes(b.compressedRecords); err != nil {
		return err
	}

	if err := pe.pop(); err != nil {
		return err
	}
	return pe.pop()
}

func (b *RecordBatch) decode(pd packetDecoder) (err error) {
	if b.FirstOffset, err = pd.getInt64(); err != nil {
		return err
	}

	batchLen, err := pd.getInt32()
	if err != nil {
		return err
	}

	if b.PartitionLeaderEpoch, err = pd.getInt32(); err != nil {
		return err
	}

	if b.Version, err = pd.getInt8(); err != nil {
		return err
	}

	if err = pd.push(&crc32Field{polynomial: crcCastagnoli}); err != nil {
		return err
	}

	attributes, err := pd.getInt16()
	if err != nil {
		return err
	}
	b.Codec = CompressionCodec(int8(attributes) & compressionCodecMask)
	b.Control = attributes&controlMask == controlMask

	if b.LastOffsetDelta, err = pd.getInt32(); err != nil {
		return err
	}

	if b.FirstTimestamp, err = pd.getInt64(); err != nil {
		return err
	}

	if b.MaxTimestamp, err = pd.getInt64(); err != nil {
		return err
	}

	if b.ProducerID, err = pd.getInt64(); err != nil {
		return err
	}

	if b.ProducerEpoch, err = pd.getInt16(); err != nil {
		return err
	}

	if b.FirstSequence, err = pd.getInt32(); err != nil {
		return err
	}

	numRecs, err := pd.getArrayLength()
	if err != nil {
		return err
	}
	if numRecs >= 0 {
		b.Records = make([]*Record, numRecs)
	}

	bufSize := int(batchLen) - recordBatchOverhead
	recBuffer, err := pd.getRawBytes(bufSize)
	if err != nil {
		return err
	}

	if err = pd.pop(); err != nil {
		return err
	}

	switch b.Codec {
	case CompressionNone:
	case CompressionGZIP:
		reader, err := gzip.NewReader(bytes.NewReader(recBuffer))
		if err != nil {
			return err
		}
		if recBuffer, err = ioutil.ReadAll(reader); err != nil {
			return err
		}
	case CompressionSnappy:
		if recBuffer, err = snappy.Decode(recBuffer); err != nil {
			return err
		}
	case CompressionLZ4:
		reader := lz4.NewReader(bytes.NewReader(recBuffer))
		if recBuffer, err = ioutil.ReadAll(reader); err != nil {
			return err
		}
	default:
		return PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", b.Codec)}
	}

	err = decode(recBuffer, recordsArray(b.Records))
	if err == ErrInsufficientData {
		b.PartialTrailingRecord = true
		b.Records = nil
		return nil
	}
	return err
}

func (b *RecordBatch) encodeRecords(pe packetEncoder) error {
	var raw []byte
	if b.Codec != CompressionNone {
		var err error
		if raw, err = encode(recordsArray(b.Records), nil); err != nil {
			return err
		}
		b.recordsLen = len(raw)
	}

	switch b.Codec {
	case CompressionNone:
		offset := pe.offset()
		if err := recordsArray(b.Records).encode(pe); err != nil {
			return err
		}
		b.recordsLen = pe.offset() - offset
	case CompressionGZIP:
		var buf bytes.Buffer
		writer := gzip.NewWriter(&buf)
		if _, err := writer.Write(raw); err != nil {
			return err
		}
		if err := writer.Close(); err != nil {
			return err
		}
		b.compressedRecords = buf.Bytes()
	case CompressionSnappy:
		b.compressedRecords = snappy.Encode(raw)
	case CompressionLZ4:
		var buf bytes.Buffer
		writer := lz4.NewWriter(&buf)
		if _, err := writer.Write(raw); err != nil {
			return err
		}
		if err := writer.Close(); err != nil {
			return err
		}
		b.compressedRecords = buf.Bytes()
	default:
		return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)}
	}

	return nil
}

func (b *RecordBatch) computeAttributes() int16 {
	attr := int16(b.Codec) & int16(compressionCodecMask)
	if b.Control {
		attr |= controlMask
	}
	return attr
}

func (b *RecordBatch) addRecord(r *Record) {
	b.Records = append(b.Records, r)
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2012 krepa098 (krepa098 at gmail dot com)
This software is provided 'as-is', without any express or implied warranty.
In no event will the authors be held liable for any damages arising from the use of this software.
Permission is granted to anyone to use this software for any purpose, including commercial applications,
and to alter it and redistribute it freely, subject to the following restrictions:
	1.	The origin of this software must not be misrepresented; you must not claim that you wrote the original software.
		If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
	2. 	Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
	3. 	This notice may not be removed or altered from any source distribution.
*/

package gosfml2

// #include <SFML/Graphics/RenderStates.h>
import "C"
import "unsafe"

/////////////////////////////////////
///		CONSTS
/////////////////////////////////////

const (
	Blend_Alpha    = iota ///< Pixel = Src * a + Dest * (1 - a)
	Blend_Add             ///< Pixel = Src + Dest
	Blend_Multiply        ///< Pixel = Src * Dest
	Blend_None            ///< No blending
)

/////////////////////////////////////
///		STRUCTS
/////////////////////////////////////

type BlendMode int

type RenderStates struct {
	cRenderStates C.sfRenderStates
}

/////////////////////////////////////
///		VARS
/////////////////////////////////////

var renderStatesDefault = MakeRenderStates(Blend_Alpha, TransformIdentity(), nil, nil)

/////////////////////////////////////
///		CONTS
/////////////////////////////////////

func RenderStatesDefault() RenderStates {
	return renderStatesDefault
}

/////////////////////////////////////
///		FUNCS
/////////////////////////////////////

func MakeRenderStates(blendMode BlendMode, transform Transform, texture *Texture, shader *Shader) (rt RenderStates) {
	rt.cRenderStates.blendMode = C.sfBlendMode(blendMode)
	rt.cRenderStates.transform = transform.toC()
	rt.cRenderStates.shader = shader.toCPtr()
	rt.cRenderStates.texture = texture.toCPtr()
	return
}

// shader can be nil (no shader)
func (this *RenderStates) SetShader(shader *Shader) {
	this.cRenderStates.shader = shader.toCPtr()
}

// texture can be nil (no texture)
func (this *RenderStates) SetTexture(texture *Texture) {
	if texture == nil {
		this.cRenderStates.texture = nil
	} else {
		this.cRenderStates.texture = texture.cptr
	}
}

func (this *RenderStates) SetTramsform(transform Transform) {
	this.cRenderStates.transform = transform.toC()
}

func (this *RenderStates) SetBlendMode(blendMode BlendMode) {
	this.cRenderStates.blendMode = C.sfBlendMode(blendMode)
}

/////////////////////////////////////
///		GO <-> C
/////////////////////////////////////

func (this *RenderStates) toCPtr() *C.sfRenderStates {
	return (*C.sfRenderStates)(unsafe.Pointer(&this.cRenderStates))
}
<commit_msg>cleanup<commit_after>/*
Copyright (c) 2012 krepa098 (krepa098 at gmail dot com)
This software is provided 'as-is', without any express or implied warranty.
In no event will the authors be held liable for any damages arising from the use of this software.
Permission is granted to anyone to use this software for any purpose, including commercial applications,
and to alter it and redistribute it freely, subject to the following restrictions:
	1.	The origin of this software must not be misrepresented; you must not claim that you wrote the original software.
		If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required.
	2. 	Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
	3. 	This notice may not be removed or altered from any source distribution.
*/

package gosfml2

// #include <SFML/Graphics/RenderStates.h>
import "C"
import "unsafe"

/////////////////////////////////////
///		CONSTS
/////////////////////////////////////

const (
	Blend_Alpha    = iota ///< Pixel = Src * a + Dest * (1 - a)
	Blend_Add             ///< Pixel = Src + Dest
	Blend_Multiply        ///< Pixel = Src * Dest
	Blend_None            ///< No blending
)

/////////////////////////////////////
///		STRUCTS
/////////////////////////////////////

type BlendMode int

type RenderStates struct {
	cRenderStates C.sfRenderStates
}

/////////////////////////////////////
///		CONTS
/////////////////////////////////////

func RenderStatesDefault() RenderStates {
	return MakeRenderStates(Blend_Alpha, TransformIdentity(), nil, nil)
}

/////////////////////////////////////
///		FUNCS
/////////////////////////////////////

func MakeRenderStates(blendMode BlendMode, transform Transform, texture *Texture, shader *Shader) (rt RenderStates) {
	rt.cRenderStates.blendMode = C.sfBlendMode(blendMode)
	rt.cRenderStates.transform = transform.toC()
	rt.cRenderStates.shader = shader.toCPtr()
	rt.cRenderStates.texture = texture.toCPtr()
	return
}

// shader can be nil (no shader)
func (this *RenderStates) SetShader(shader *Shader) {
	this.cRenderStates.shader = shader.toCPtr()
}

// texture can be nil (no texture)
func (this *RenderStates) SetTexture(texture *Texture) {
		this.cRenderStates.texture = texture.toCPtr()
}

func (this *RenderStates) SetTramsform(transform Transform) {
	this.cRenderStates.transform = transform.toC()
}

func (this *RenderStates) SetBlendMode(blendMode BlendMode) {
	this.cRenderStates.blendMode = C.sfBlendMode(blendMode)
}

/////////////////////////////////////
///		GO <-> C
/////////////////////////////////////

func (this *RenderStates) toCPtr() *C.sfRenderStates {
	return (*C.sfRenderStates)(unsafe.Pointer(&this.cRenderStates))
}
<|endoftext|>


<commit_before>package structs

import (
	"fmt"
	"time"
)

func ExampleNew() {
	type Server struct {
		Name    string
		ID      int32
		Enabled bool
	}

	server := &Server{
		Name:    "Arslan",
		ID:      123456,
		Enabled: true,
	}

	s := New(server)

	fmt.Printf("Name        : %v\n", s.Name())
	fmt.Printf("Values      : %v\n", s.Values())
	fmt.Printf("Value of ID : %v\n", s.Field("ID").Value())
	// Output:
	// Name        : Server
	// Values      : [Arslan 123456 true]
	// Value of ID : 123456

}

func ExampleMap() {
	type Server struct {
		Name    string
		ID      int32
		Enabled bool
	}

	s := &Server{
		Name:    "Arslan",
		ID:      123456,
		Enabled: true,
	}

	m := Map(s)

	fmt.Printf("%#v\n", m["Name"])
	fmt.Printf("%#v\n", m["ID"])
	fmt.Printf("%#v\n", m["Enabled"])
	// Output:
	// "Arslan"
	// 123456
	// true

}

func ExampleMap_tags() {
	// Custom tags can change the map keys instead of using the fields name
	type Server struct {
		Name    string `structs:"server_name"`
		ID      int32  `structs:"server_id"`
		Enabled bool   `structs:"enabled"`
	}

	s := &Server{
		Name: "Zeynep",
		ID:   789012,
	}

	m := Map(s)

	// access them by the custom tags defined above
	fmt.Printf("%#v\n", m["server_name"])
	fmt.Printf("%#v\n", m["server_id"])
	fmt.Printf("%#v\n", m["enabled"])
	// Output:
	// "Zeynep"
	// 789012
	// false

}

func ExampleMap_nested() {
	// By default field with struct types are processed too. We can stop
	// processing them via "omitnested" tag option.
	type Server struct {
		Name string    `structs:"server_name"`
		ID   int32     `structs:"server_id"`
		Time time.Time `structs:"time,omitnested"` // do not convert to map[string]interface{}
	}

	const shortForm = "2006-Jan-02"
	t, _ := time.Parse("2006-Jan-02", "2013-Feb-03")

	s := &Server{
		Name: "Zeynep",
		ID:   789012,
		Time: t,
	}

	m := Map(s)

	// access them by the custom tags defined above
	fmt.Printf("%v\n", m["server_name"])
	fmt.Printf("%v\n", m["server_id"])
	fmt.Printf("%v\n", m["time"].(time.Time))
	// Output:
	// Zeynep
	// 789012
	// 2013-02-03 00:00:00 +0000 UTC
}

func ExampleValues() {
	type Server struct {
		Name    string
		ID      int32
		Enabled bool
	}

	s := &Server{
		Name:    "Fatih",
		ID:      135790,
		Enabled: false,
	}

	m := Values(s)

	fmt.Printf("Values: %+v\n", m)
	// Output:
	// Values: [Fatih 135790 false]
}

func ExampleValues_tags() {
	type Location struct {
		City    string
		Country string
	}

	type Server struct {
		Name     string
		ID       int32
		Enabled  bool
		Location Location `structs:"-"` // values from location are not included anymore
	}

	s := &Server{
		Name:     "Fatih",
		ID:       135790,
		Enabled:  false,
		Location: Location{City: "Ankara", Country: "Turkey"},
	}

	// Let get all values from the struct s. Note that we don't include values
	// from the Location field
	m := Values(s)

	fmt.Printf("Values: %+v\n", m)
	// Output:
	// Values: [Fatih 135790 false]
}

func ExampleFields() {
	type Access struct {
		Name         string
		LastAccessed time.Time
		Number       int
	}

	s := &Access{
		Name:         "Fatih",
		LastAccessed: time.Now(),
		Number:       1234567,
	}

	fields := Fields(s)

	for i, field := range fields {
		fmt.Printf("[%d] %+v\n", i, field.Name())
	}

	// Output:
	// [0] Name
	// [1] LastAccessed
	// [2] Number
}

func ExampleFields_nested() {
	type Person struct {
		Name   string
		Number int
	}

	type Access struct {
		Person        Person
		HasPermission bool
		LastAccessed  time.Time
	}

	s := &Access{
		Person:        Person{Name: "fatih", Number: 1234567},
		LastAccessed:  time.Now(),
		HasPermission: true,
	}

	// Let's get all fields from the struct s.
	fields := Fields(s)

	for _, field := range fields {
		if field.Name() == "Person" {
			fmt.Printf("Access.Person.Name: %+v\n", field.Field("Name").Value())
		}
	}

	// Output:
	// Access.Person.Name: fatih
}

func ExampleField() {
	type Person struct {
		Name   string
		Number int
	}

	type Access struct {
		Person        Person
		HasPermission bool
		LastAccessed  time.Time
	}

	access := &Access{
		Person:        Person{Name: "fatih", Number: 1234567},
		LastAccessed:  time.Now(),
		HasPermission: true,
	}

	// Create a new Struct type
	s := New(access)

	// Get the Field type for "Person" field
	p := s.Field("Person")

	// Get the underlying "Name field" and print the value of it
	name := p.Field("Name")

	fmt.Printf("Value of Person.Access.Name: %+v\n", name.Value())

	// Output:
	// Value of Person.Access.Name: fatih

}

func ExampleIsZero() {
	type Server struct {
		Name    string
		ID      int32
		Enabled bool
	}

	// Nothing is initalized
	a := &Server{}
	isZeroA := IsZero(a)

	// Name and Enabled is initialized, but not ID
	b := &Server{
		Name:    "Golang",
		Enabled: true,
	}
	isZeroB := IsZero(b)

	fmt.Printf("%#v\n", isZeroA)
	fmt.Printf("%#v\n", isZeroB)
	// Output:
	// true
	// false
}

func ExampleHasZero() {
	// Let's define an Access struct. Note that the "Enabled" field is not
	// going to be checked because we added the "structs" tag to the field.
	type Access struct {
		Name         string
		LastAccessed time.Time
		Number       int
		Enabled      bool `structs:"-"`
	}

	// Name and Number is not initialized.
	a := &Access{
		LastAccessed: time.Now(),
	}
	hasZeroA := HasZero(a)

	// Name and Number is initialized.
	b := &Access{
		Name:         "Fatih",
		LastAccessed: time.Now(),
		Number:       12345,
	}
	hasZeroB := HasZero(b)

	fmt.Printf("%#v\n", hasZeroA)
	fmt.Printf("%#v\n", hasZeroB)
	// Output:
	// true
	// false
}
<commit_msg>structs_example: add examples for omitempty usage<commit_after>package structs

import (
	"fmt"
	"time"
)

func ExampleNew() {
	type Server struct {
		Name    string
		ID      int32
		Enabled bool
	}

	server := &Server{
		Name:    "Arslan",
		ID:      123456,
		Enabled: true,
	}

	s := New(server)

	fmt.Printf("Name        : %v\n", s.Name())
	fmt.Printf("Values      : %v\n", s.Values())
	fmt.Printf("Value of ID : %v\n", s.Field("ID").Value())
	// Output:
	// Name        : Server
	// Values      : [Arslan 123456 true]
	// Value of ID : 123456

}

func ExampleMap() {
	type Server struct {
		Name    string
		ID      int32
		Enabled bool
	}

	s := &Server{
		Name:    "Arslan",
		ID:      123456,
		Enabled: true,
	}

	m := Map(s)

	fmt.Printf("%#v\n", m["Name"])
	fmt.Printf("%#v\n", m["ID"])
	fmt.Printf("%#v\n", m["Enabled"])
	// Output:
	// "Arslan"
	// 123456
	// true

}

func ExampleMap_tags() {
	// Custom tags can change the map keys instead of using the fields name
	type Server struct {
		Name    string `structs:"server_name"`
		ID      int32  `structs:"server_id"`
		Enabled bool   `structs:"enabled"`
	}

	s := &Server{
		Name: "Zeynep",
		ID:   789012,
	}

	m := Map(s)

	// access them by the custom tags defined above
	fmt.Printf("%#v\n", m["server_name"])
	fmt.Printf("%#v\n", m["server_id"])
	fmt.Printf("%#v\n", m["enabled"])
	// Output:
	// "Zeynep"
	// 789012
	// false

}

func ExampleMap_nested() {
	// By default field with struct types are processed too. We can stop
	// processing them via "omitnested" tag option.
	type Server struct {
		Name string    `structs:"server_name"`
		ID   int32     `structs:"server_id"`
		Time time.Time `structs:"time,omitnested"` // do not convert to map[string]interface{}
	}

	const shortForm = "2006-Jan-02"
	t, _ := time.Parse("2006-Jan-02", "2013-Feb-03")

	s := &Server{
		Name: "Zeynep",
		ID:   789012,
		Time: t,
	}

	m := Map(s)

	// access them by the custom tags defined above
	fmt.Printf("%v\n", m["server_name"])
	fmt.Printf("%v\n", m["server_id"])
	fmt.Printf("%v\n", m["time"].(time.Time))
	// Output:
	// Zeynep
	// 789012
	// 2013-02-03 00:00:00 +0000 UTC
}

func ExampleMap_omitEmpty() {
	// By default field with struct types of zero values are processed too. We
	// can stop processing them via "omitempty" tag option.
	type Server struct {
		Name     string `structs:",omitempty"`
		ID       int32  `structs:"server_id,omitempty"`
		Location string
	}

	// Only add location
	s := &Server{
		Location: "Tokyo",
	}

	m := Map(s)

	// map contains only the Location field
	fmt.Printf("%v\n", m)
	// Output:
	// map[Location:Tokyo]
}

func ExampleValues() {
	type Server struct {
		Name    string
		ID      int32
		Enabled bool
	}

	s := &Server{
		Name:    "Fatih",
		ID:      135790,
		Enabled: false,
	}

	m := Values(s)

	fmt.Printf("Values: %+v\n", m)
	// Output:
	// Values: [Fatih 135790 false]
}

func ExampleValues_omitEmpty() {
	// By default field with struct types of zero values are processed too. We
	// can stop processing them via "omitempty" tag option.
	type Server struct {
		Name     string `structs:",omitempty"`
		ID       int32  `structs:"server_id,omitempty"`
		Location string
	}

	// Only add location
	s := &Server{
		Location: "Ankara",
	}

	m := Values(s)

	// values contains only the Location field
	fmt.Printf("Values: %+v\n", m)
	// Output:
	// Values: [Ankara]
}

func ExampleValues_tags() {
	type Location struct {
		City    string
		Country string
	}

	type Server struct {
		Name     string
		ID       int32
		Enabled  bool
		Location Location `structs:"-"` // values from location are not included anymore
	}

	s := &Server{
		Name:     "Fatih",
		ID:       135790,
		Enabled:  false,
		Location: Location{City: "Ankara", Country: "Turkey"},
	}

	// Let get all values from the struct s. Note that we don't include values
	// from the Location field
	m := Values(s)

	fmt.Printf("Values: %+v\n", m)
	// Output:
	// Values: [Fatih 135790 false]
}

func ExampleFields() {
	type Access struct {
		Name         string
		LastAccessed time.Time
		Number       int
	}

	s := &Access{
		Name:         "Fatih",
		LastAccessed: time.Now(),
		Number:       1234567,
	}

	fields := Fields(s)

	for i, field := range fields {
		fmt.Printf("[%d] %+v\n", i, field.Name())
	}

	// Output:
	// [0] Name
	// [1] LastAccessed
	// [2] Number
}

func ExampleFields_nested() {
	type Person struct {
		Name   string
		Number int
	}

	type Access struct {
		Person        Person
		HasPermission bool
		LastAccessed  time.Time
	}

	s := &Access{
		Person:        Person{Name: "fatih", Number: 1234567},
		LastAccessed:  time.Now(),
		HasPermission: true,
	}

	// Let's get all fields from the struct s.
	fields := Fields(s)

	for _, field := range fields {
		if field.Name() == "Person" {
			fmt.Printf("Access.Person.Name: %+v\n", field.Field("Name").Value())
		}
	}

	// Output:
	// Access.Person.Name: fatih
}

func ExampleField() {
	type Person struct {
		Name   string
		Number int
	}

	type Access struct {
		Person        Person
		HasPermission bool
		LastAccessed  time.Time
	}

	access := &Access{
		Person:        Person{Name: "fatih", Number: 1234567},
		LastAccessed:  time.Now(),
		HasPermission: true,
	}

	// Create a new Struct type
	s := New(access)

	// Get the Field type for "Person" field
	p := s.Field("Person")

	// Get the underlying "Name field" and print the value of it
	name := p.Field("Name")

	fmt.Printf("Value of Person.Access.Name: %+v\n", name.Value())

	// Output:
	// Value of Person.Access.Name: fatih

}

func ExampleIsZero() {
	type Server struct {
		Name    string
		ID      int32
		Enabled bool
	}

	// Nothing is initalized
	a := &Server{}
	isZeroA := IsZero(a)

	// Name and Enabled is initialized, but not ID
	b := &Server{
		Name:    "Golang",
		Enabled: true,
	}
	isZeroB := IsZero(b)

	fmt.Printf("%#v\n", isZeroA)
	fmt.Printf("%#v\n", isZeroB)
	// Output:
	// true
	// false
}

func ExampleHasZero() {
	// Let's define an Access struct. Note that the "Enabled" field is not
	// going to be checked because we added the "structs" tag to the field.
	type Access struct {
		Name         string
		LastAccessed time.Time
		Number       int
		Enabled      bool `structs:"-"`
	}

	// Name and Number is not initialized.
	a := &Access{
		LastAccessed: time.Now(),
	}
	hasZeroA := HasZero(a)

	// Name and Number is initialized.
	b := &Access{
		Name:         "Fatih",
		LastAccessed: time.Now(),
		Number:       12345,
	}
	hasZeroB := HasZero(b)

	fmt.Printf("%#v\n", hasZeroA)
	fmt.Printf("%#v\n", hasZeroB)
	// Output:
	// true
	// false
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type restService struct {
	client   *http.Client
	endpoint string
}

func newRestService(config Config) (*restService, error) {
	client, err := httpClient(config)
	if err != nil {
		return nil, fmt.Errorf("error creating HTTP client: %v", err)
	}
	return &restService{client, extractEndpoint(config)}, nil
}

func extractEndpoint(config Config) string {
	endpoint := config.Connection.APIEndpoint
	if strings.HasSuffix(endpoint, "/") {
		return endpoint
	}
	return endpoint + "/"
}

func httpClient(config Config) (*http.Client, error) {
	// load client certificate
	cert, err := tls.LoadX509KeyPair(config.Connection.Cert, config.Connection.Key)
	if err != nil {
		return nil, fmt.Errorf("error loading X509 key pair: %v", err)
	}
	// load CA file to verify server
	caPool := x509.NewCertPool()
	severCert, err := ioutil.ReadFile(config.Connection.CACert)
	if err != nil {
		return nil, fmt.Errorf("could not load CA file: %v", err)
	}
	caPool.AppendCertsFromPEM(severCert)
	// create a client with specific transport configurations
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{
			RootCAs:      caPool,
			Certificates: []tls.Certificate{cert},
		},
	}
	client := &http.Client{Transport: transport}

	return client, nil
}

func (r *restService) Post(path string, json []byte) ([]byte, int, error) {
	output := strings.NewReader(string(json))
	log.Printf("debug: post request path: %s , body: %s", r.endpoint+path, string(json))
	response, err := r.client.Post(r.endpoint+path, "application/json", output)
	if err != nil {
		return nil, -1, fmt.Errorf("error on http request (POST %v): %v", r.endpoint+path, err)
	}
	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, -1, fmt.Errorf("error reading http request body: %v", err)
	}

	return body, response.StatusCode, err
}

func (r *restService) Delete(path string) ([]byte, int, error) {
	request, err := http.NewRequest("DELETE", r.endpoint+path, nil)
	if err != nil {
		return nil, -1, fmt.Errorf("error creating http request (DELETE %v): %v", r.endpoint+path, err)
	}
	response, err := r.client.Do(request)
	if err != nil {
		return nil, -1, fmt.Errorf("error executing http request (DELETE %v): %v", r.endpoint+path, err)
	}
	defer response.Body.Close()

	body, _ := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, -1, fmt.Errorf("error reading http request body: %v", err)
	}

	return body, response.StatusCode, nil
}

func (r *restService) Get(path string) ([]byte, int, error) {
	response, err := r.client.Get(r.endpoint + path)
	if err != nil {
		return nil, -1, fmt.Errorf("error on http request (GET %v): %v", r.endpoint+path, err)
	}
	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, -1, fmt.Errorf("error reading http request body: %v", err)
	}

	return body, response.StatusCode, nil
}
<commit_msg>Inclusion of pretty print for debugging purposes<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type restService struct {
	client   *http.Client
	endpoint string
}

func newRestService(config Config) (*restService, error) {
	client, err := httpClient(config)
	if err != nil {
		return nil, fmt.Errorf("error creating HTTP client: %v", err)
	}
	return &restService{client, extractEndpoint(config)}, nil
}

func extractEndpoint(config Config) string {
	endpoint := config.Connection.APIEndpoint
	if strings.HasSuffix(endpoint, "/") {
		return endpoint
	}
	return endpoint + "/"
}

func httpClient(config Config) (*http.Client, error) {
	// load client certificate
	cert, err := tls.LoadX509KeyPair(config.Connection.Cert, config.Connection.Key)
	if err != nil {
		return nil, fmt.Errorf("error loading X509 key pair: %v", err)
	}
	// load CA file to verify server
	caPool := x509.NewCertPool()
	severCert, err := ioutil.ReadFile(config.Connection.CACert)
	if err != nil {
		return nil, fmt.Errorf("could not load CA file: %v", err)
	}
	caPool.AppendCertsFromPEM(severCert)
	// create a client with specific transport configurations
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{
			RootCAs:      caPool,
			Certificates: []tls.Certificate{cert},
		},
	}
	client := &http.Client{Transport: transport}

	return client, nil
}

func (r *restService) prettyprint(b []byte) []byte {
	var out bytes.Buffer
	json.Indent(&out, b, "", "  ")
	return out.Bytes()
}

func (r *restService) Post(path string, json []byte) ([]byte, int, error) {
	output := strings.NewReader(string(json))
	log.Printf("debug: post request path: %s , body: \n%s\n", r.endpoint+path, string(prettyprint(json)))
	response, err := r.client.Post(r.endpoint+path, "application/json", output)
	if err != nil {
		return nil, -1, fmt.Errorf("error on http request (POST %v): %v", r.endpoint+path, err)
	}
	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, -1, fmt.Errorf("error reading http request body: %v", err)
	}

	return body, response.StatusCode, err
}

func (r *restService) Delete(path string) ([]byte, int, error) {
	request, err := http.NewRequest("DELETE", r.endpoint+path, nil)
	if err != nil {
		return nil, -1, fmt.Errorf("error creating http request (DELETE %v): %v", r.endpoint+path, err)
	}
	response, err := r.client.Do(request)
	if err != nil {
		return nil, -1, fmt.Errorf("error executing http request (DELETE %v): %v", r.endpoint+path, err)
	}
	defer response.Body.Close()

	body, _ := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, -1, fmt.Errorf("error reading http request body: %v", err)
	}

	return body, response.StatusCode, nil
}

func (r *restService) Get(path string) ([]byte, int, error) {
	response, err := r.client.Get(r.endpoint + path)
	if err != nil {
		return nil, -1, fmt.Errorf("error on http request (GET %v): %v", r.endpoint+path, err)
	}
	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, -1, fmt.Errorf("error reading http request body: %v", err)
	}

	return body, response.StatusCode, nil
}
<|endoftext|>


<commit_before>package protocol

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
)

// This is sent into the function
// All HTTP request headers should be set in env
type JSONIO struct {
	Headers    http.Header `json:"headers,omitempty"`
	Body       string      `json:"body"`
	StatusCode int         `json:"status_code,omitempty"`
}

// JSONProtocol converts stdin/stdout streams from HTTP into JSON format.
type JSONProtocol struct {
	in  io.Writer
	out io.Reader
}

func (p *JSONProtocol) IsStreamable() bool {
	return true
}

type RequestEncoder struct {
	*json.Encoder
}

func (re *RequestEncoder) EncodeRequest(rq *http.Request) error {
	bb := new(bytes.Buffer)
	_, err := bb.ReadFrom(rq.Body)
	if err != nil {
		return err
	}
	defer bb.Reset()
	return re.Encode(JSONIO{
		Headers: rq.Header,
		Body:    bb.String(),
	})
}

func (h *JSONProtocol) DumpJSON(w io.Writer, req *http.Request) error {
	_, err := io.WriteString(h.in, `{`)
	if err != nil {
		// this shouldn't happen
		return err
	}
	if req.Body != nil {
		_, err := io.WriteString(h.in, `"body":"`)
		if err != nil {
			// this shouldn't happen
			return err
		}
		_, err = io.Copy(h.in, req.Body)
		if err != nil {
			// this shouldn't happen
			return err
		}
		_, err = io.WriteString(h.in, `",`)
		if err != nil {
			// this shouldn't happen
			return err
		}
		defer req.Body.Close()
	}
	_, err = io.WriteString(h.in, `"headers:"`)
	if err != nil {
		// this shouldn't happen
		return err
	}
	err = json.NewEncoder(h.in).Encode(req.Header)
	if err != nil {
		// this shouldn't happen
		return err
	}
	_, err = io.WriteString(h.in, `"}`)
	if err != nil {
		// this shouldn't happen
		return err
	}
	return nil
}

func (h *JSONProtocol) Dispatch(w io.Writer, req *http.Request) error {
	ce := RequestEncoder{json.NewEncoder(h.in)}
	err := ce.EncodeRequest(req)
	//err := h.DumpJSON(w, req)
	if err != nil {
		return respondWithError(
			w, fmt.Errorf("unable to write JSON into STDIN: %s", err.Error()))
	}
	jout := new(JSONIO)
	dec := json.NewDecoder(h.out)
	if err := dec.Decode(jout); err != nil {
		return respondWithError(
			w, fmt.Errorf("unable to decode JSON response object: %s", err.Error()))
	}
	if rw, ok := w.(http.ResponseWriter); ok {
		// this has to be done for pulling out:
		// - status code
		// - body
		// - headers
		for k, vs := range jout.Headers {
			for _, v := range vs {
				rw.Header().Add(k, v) // on top of any specified on the route
			}
		}
		rw.WriteHeader(jout.StatusCode)
		_, err = rw.Write([]byte(jout.Body)) // TODO timeout
		if err != nil {
			return respondWithError(
				w, fmt.Errorf("unable to write JSON response object: %s", err.Error()))
		}
	} else {
		// logs can just copy the full thing in there, headers and all.
		err = json.NewEncoder(w).Encode(jout)
		if err != nil {
			return respondWithError(
				w, fmt.Errorf("error writing function response: %s", err.Error()))
		}
	}
	return nil
}

func respondWithError(w io.Writer, err error) error {
	errMsg := []byte(err.Error())
	statusCode := http.StatusInternalServerError
	if rw, ok := w.(http.ResponseWriter); ok {
		rw.WriteHeader(statusCode)
		rw.Write(errMsg)
	} else {
		// logs can just copy the full thing in there, headers and all.
		w.Write(errMsg)
	}

	return err
}
<commit_msg>Do not forget to close request body<commit_after>package protocol

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
)

// This is sent into the function
// All HTTP request headers should be set in env
type JSONIO struct {
	Headers    http.Header `json:"headers,omitempty"`
	Body       string      `json:"body"`
	StatusCode int         `json:"status_code,omitempty"`
}

// JSONProtocol converts stdin/stdout streams from HTTP into JSON format.
type JSONProtocol struct {
	in  io.Writer
	out io.Reader
}

func (p *JSONProtocol) IsStreamable() bool {
	return true
}

type RequestEncoder struct {
	*json.Encoder
}

func (re *RequestEncoder) EncodeRequest(rq *http.Request) error {
	bb := new(bytes.Buffer)
	_, err := bb.ReadFrom(rq.Body)
	if err != nil {
		return err
	}
	defer rq.Body.Close()
	defer bb.Reset()
	return re.Encode(JSONIO{
		Headers: rq.Header,
		Body:    bb.String(),
	})
}

func (h *JSONProtocol) DumpJSON(w io.Writer, req *http.Request) error {
	_, err := io.WriteString(h.in, `{`)
	if err != nil {
		// this shouldn't happen
		return err
	}
	if req.Body != nil {
		_, err := io.WriteString(h.in, `"body":"`)
		if err != nil {
			// this shouldn't happen
			return err
		}
		_, err = io.Copy(h.in, req.Body)
		if err != nil {
			// this shouldn't happen
			return err
		}
		_, err = io.WriteString(h.in, `",`)
		if err != nil {
			// this shouldn't happen
			return err
		}
		defer req.Body.Close()
	}
	_, err = io.WriteString(h.in, `"headers:"`)
	if err != nil {
		// this shouldn't happen
		return err
	}
	err = json.NewEncoder(h.in).Encode(req.Header)
	if err != nil {
		// this shouldn't happen
		return err
	}
	_, err = io.WriteString(h.in, `"}`)
	if err != nil {
		// this shouldn't happen
		return err
	}
	return nil
}

func (h *JSONProtocol) Dispatch(w io.Writer, req *http.Request) error {
	ce := RequestEncoder{json.NewEncoder(h.in)}
	err := ce.EncodeRequest(req)
	//err := h.DumpJSON(w, req)
	if err != nil {
		return respondWithError(
			w, fmt.Errorf("unable to write JSON into STDIN: %s", err.Error()))
	}
	jout := new(JSONIO)
	dec := json.NewDecoder(h.out)
	if err := dec.Decode(jout); err != nil {
		return respondWithError(
			w, fmt.Errorf("unable to decode JSON response object: %s", err.Error()))
	}
	if rw, ok := w.(http.ResponseWriter); ok {
		// this has to be done for pulling out:
		// - status code
		// - body
		// - headers
		for k, vs := range jout.Headers {
			for _, v := range vs {
				rw.Header().Add(k, v) // on top of any specified on the route
			}
		}
		rw.WriteHeader(jout.StatusCode)
		_, err = rw.Write([]byte(jout.Body)) // TODO timeout
		if err != nil {
			return respondWithError(
				w, fmt.Errorf("unable to write JSON response object: %s", err.Error()))
		}
	} else {
		// logs can just copy the full thing in there, headers and all.
		err = json.NewEncoder(w).Encode(jout)
		if err != nil {
			return respondWithError(
				w, fmt.Errorf("error writing function response: %s", err.Error()))
		}
	}
	return nil
}

func respondWithError(w io.Writer, err error) error {
	errMsg := []byte(err.Error())
	statusCode := http.StatusInternalServerError
	if rw, ok := w.(http.ResponseWriter); ok {
		rw.WriteHeader(statusCode)
		rw.Write(errMsg)
	} else {
		// logs can just copy the full thing in there, headers and all.
		w.Write(errMsg)
	}

	return err
}
<|endoftext|>


<commit_before>package reviewdog

import (
	"fmt"
	"log"
	"os/exec"
	"sync"

	"golang.org/x/sync/errgroup"

	"github.com/google/go-github/github"
)

var _ = github.ScopeAdminOrg

var _ CommentService = &GitHubPullRequest{}
var _ DiffService = &GitHubPullRequest{}

// `path` to `position`(Lnum for new file) to comment `body`s
type postedcomments map[string]map[int][]string

// IsPosted returns true if a given comment has been posted in GitHub already,
// otherwise returns false. It sees comments with same path, same position,
// and same body as same comments.
func (p postedcomments) IsPosted(c *Comment) bool {
	if _, ok := p[c.Path]; !ok {
		return false
	}
	bodys, ok := p[c.Path][c.LnumDiff]
	if !ok {
		return false
	}
	for _, body := range bodys {
		if body == commentBody(c) {
			return true
		}
	}
	return false
}

// GitHubPullRequest is a comment and diff service for GitHub PullRequest.
//
// API:
//	https://developer.github.com/v3/pulls/comments/#create-a-comment
// 	POST /repos/:owner/:repo/pulls/:number/comments
type GitHubPullRequest struct {
	postComments []*Comment

	cli   *github.Client
	owner string
	repo  string
	pr    int
	sha   string

	postedcs postedcomments

	muFlash sync.Mutex
}

// NewGitHubPullReqest returns a new GitHubPullRequest service.
func NewGitHubPullReqest(cli *github.Client, owner, repo string, pr int, sha string) *GitHubPullRequest {
	return &GitHubPullRequest{
		cli:   cli,
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
	}
}

// Post accepts a comment and holds it. Flash method actually posts comments to
// GitHub in parallel.
func (g *GitHubPullRequest) Post(c *Comment) error {
	g.muFlash.Lock()
	defer g.muFlash.Unlock()
	g.postComments = append(g.postComments, c)
	return nil
}

const bodyPrefix = `<sub>reported by [reviewdog](https://github.com/haya14busa/reviewdog) :dog:</sub>`

func commentBody(c *Comment) string {
	tool := ""
	if c.ToolName != "" {
		tool = fmt.Sprintf("**[%s]** ", c.ToolName)
	}
	return tool + bodyPrefix + "\n" + c.Body
}

var githubAPIHost = "api.github.com"

// Flash posts comments which has not been posted yet.
func (g *GitHubPullRequest) Flash() error {
	g.muFlash.Lock()
	defer g.muFlash.Unlock()

	if err := g.setPostedComment(); err != nil {
		return err
	}
	// TODO(haya14busa,#58): remove host check when GitHub Enterprise supports
	// Pull Request API.
	if g.cli.BaseURL.Host == githubAPIHost {
		return g.postAsReviewComment()
	}
	return g.postCommentsForEach()
}

func (g *GitHubPullRequest) postAsReviewComment() error {
	comments := make([]*ReviewComment, 0, len(g.postComments))
	for _, c := range g.postComments {
		if g.postedcs.IsPosted(c) {
			continue
		}
		cbody := commentBody(c)
		comments = append(comments, &ReviewComment{
			Path:     &c.Path,
			Position: &c.LnumDiff,
			Body:     &cbody,
		})
	}

	if len(comments) == 0 {
		return nil
	}

	// TODO(haya14busa): it might be useful to report overview results by "body"
	// field.
	event := "COMMENT"
	review := &Review{Event: &event, Comments: comments}

	_, _, err := g.CreateReview(g.owner, g.repo, g.pr, review)
	return err
}

func (g *GitHubPullRequest) postCommentsForEach() error {
	var eg errgroup.Group
	for _, c := range g.postComments {
		comment := c
		if g.postedcs.IsPosted(comment) {
			continue
		}
		eg.Go(func() error {
			body := commentBody(comment)
			prcomment := &github.PullRequestComment{
				CommitID: &g.sha,
				Body:     &body,
				Path:     &comment.Path,
				Position: &comment.LnumDiff,
			}
			_, _, err := g.cli.PullRequests.CreateComment(g.owner, g.repo, g.pr, prcomment)
			return err
		})
	}
	return eg.Wait()
}

func (g *GitHubPullRequest) setPostedComment() error {
	g.postedcs = make(postedcomments)
	cs, err := g.comment()
	if err != nil {
		return err
	}
	for _, c := range cs {
		if c.Position == nil || c.Path == nil || c.Body == nil {
			// skip resolved comments. Or comments which do not have "path" nor
			// "body".
			continue
		}
		path := *c.Path
		pos := *c.Position
		body := *c.Body
		if _, ok := g.postedcs[path]; !ok {
			g.postedcs[path] = make(map[int][]string)
		}
		if _, ok := g.postedcs[path][pos]; !ok {
			g.postedcs[path][pos] = make([]string, 0)
		}
		g.postedcs[path][pos] = append(g.postedcs[path][pos], body)
	}
	return nil
}

// Diff returns a diff of PullRequest. It runs `git diff` locally instead of
// diff_url of GitHub Pull Request because diff of diff_url is not suited for
// comment API in a sense that diff of diff_url is equivalent to
// `git diff --no-renames`, we want diff which is equivalent to
// `git diff --find-renames`.
func (g *GitHubPullRequest) Diff() ([]byte, error) {
	pr, _, err := g.cli.PullRequests.Get(g.owner, g.repo, g.pr)
	if err != nil {
		return nil, err
	}
	return exec.Command("git", "diff", "--find-renames", *pr.Base.SHA, g.sha).Output()
}

// Strip returns 1 as a strip of git diff.
func (g *GitHubPullRequest) Strip() int {
	return 1
}

func (g *GitHubPullRequest) comment() ([]*github.PullRequestComment, error) {
	comments, _, err := g.cli.PullRequests.ListComments(g.owner, g.repo, g.pr, nil)
	if err != nil {
		return nil, err
	}
	return comments, nil
}

// ---
// GitHub PullRequest Review API Implementation
// ref: https://github.com/google/go-github/issues/495

const (
	mediaTypePullRequestReview = "application/vnd.github.black-cat-preview+json"
)

// Review represents a pull request review.
type Review struct {
	Body     *string          `json:"body,omitempty"`
	Event    *string          `json:"event,omitempty"`
	Comments []*ReviewComment `json:"comments,omitempty"`
}

// ReviewComment represents draft review comments.
type ReviewComment struct {
	Path     *string `json:"path,omitempty"`
	Position *int    `json:"position,omitempty"`
	Body     *string `json:"body,omitempty"`
}

// CreateReview creates a new review comment on the specified pull request.
//
// GitHub API docs: https://developer.github.com/v3/pulls/reviews/#create-a-pull-request-review
func (g *GitHubPullRequest) CreateReview(owner, repo string, number int, review *Review) (*github.PullRequestReview, *github.Response, error) {
	u := fmt.Sprintf("repos/%v/%v/pulls/%d/reviews", owner, repo, number)

	req, err := g.cli.NewRequest("POST", u, review)
	if err != nil {
		return nil, nil, err
	}

	req.Header.Set("Accept", mediaTypePullRequestReview)

	r := new(github.PullRequestReview)
	resp, err := g.cli.Do(req, r)
	if err != nil {
		log.Printf("GitHub Review API error: %v", err)
		return nil, resp, err
	}
	return r, resp, err
}
<commit_msg>github: use git merge-base to calculate diff<commit_after>package reviewdog

import (
	"fmt"
	"log"
	"os/exec"
	"strings"
	"sync"

	"golang.org/x/sync/errgroup"

	"github.com/google/go-github/github"
)

var _ = github.ScopeAdminOrg

var _ CommentService = &GitHubPullRequest{}
var _ DiffService = &GitHubPullRequest{}

// `path` to `position`(Lnum for new file) to comment `body`s
type postedcomments map[string]map[int][]string

// IsPosted returns true if a given comment has been posted in GitHub already,
// otherwise returns false. It sees comments with same path, same position,
// and same body as same comments.
func (p postedcomments) IsPosted(c *Comment) bool {
	if _, ok := p[c.Path]; !ok {
		return false
	}
	bodys, ok := p[c.Path][c.LnumDiff]
	if !ok {
		return false
	}
	for _, body := range bodys {
		if body == commentBody(c) {
			return true
		}
	}
	return false
}

// GitHubPullRequest is a comment and diff service for GitHub PullRequest.
//
// API:
//	https://developer.github.com/v3/pulls/comments/#create-a-comment
// 	POST /repos/:owner/:repo/pulls/:number/comments
type GitHubPullRequest struct {
	postComments []*Comment

	cli   *github.Client
	owner string
	repo  string
	pr    int
	sha   string

	postedcs postedcomments

	muFlash sync.Mutex
}

// NewGitHubPullReqest returns a new GitHubPullRequest service.
func NewGitHubPullReqest(cli *github.Client, owner, repo string, pr int, sha string) *GitHubPullRequest {
	return &GitHubPullRequest{
		cli:   cli,
		owner: owner,
		repo:  repo,
		pr:    pr,
		sha:   sha,
	}
}

// Post accepts a comment and holds it. Flash method actually posts comments to
// GitHub in parallel.
func (g *GitHubPullRequest) Post(c *Comment) error {
	g.muFlash.Lock()
	defer g.muFlash.Unlock()
	g.postComments = append(g.postComments, c)
	return nil
}

const bodyPrefix = `<sub>reported by [reviewdog](https://github.com/haya14busa/reviewdog) :dog:</sub>`

func commentBody(c *Comment) string {
	tool := ""
	if c.ToolName != "" {
		tool = fmt.Sprintf("**[%s]** ", c.ToolName)
	}
	return tool + bodyPrefix + "\n" + c.Body
}

var githubAPIHost = "api.github.com"

// Flash posts comments which has not been posted yet.
func (g *GitHubPullRequest) Flash() error {
	g.muFlash.Lock()
	defer g.muFlash.Unlock()

	if err := g.setPostedComment(); err != nil {
		return err
	}
	// TODO(haya14busa,#58): remove host check when GitHub Enterprise supports
	// Pull Request API.
	if g.cli.BaseURL.Host == githubAPIHost {
		return g.postAsReviewComment()
	}
	return g.postCommentsForEach()
}

func (g *GitHubPullRequest) postAsReviewComment() error {
	comments := make([]*ReviewComment, 0, len(g.postComments))
	for _, c := range g.postComments {
		if g.postedcs.IsPosted(c) {
			continue
		}
		cbody := commentBody(c)
		comments = append(comments, &ReviewComment{
			Path:     &c.Path,
			Position: &c.LnumDiff,
			Body:     &cbody,
		})
	}

	if len(comments) == 0 {
		return nil
	}

	// TODO(haya14busa): it might be useful to report overview results by "body"
	// field.
	event := "COMMENT"
	review := &Review{Event: &event, Comments: comments}

	_, _, err := g.CreateReview(g.owner, g.repo, g.pr, review)
	return err
}

func (g *GitHubPullRequest) postCommentsForEach() error {
	var eg errgroup.Group
	for _, c := range g.postComments {
		comment := c
		if g.postedcs.IsPosted(comment) {
			continue
		}
		eg.Go(func() error {
			body := commentBody(comment)
			prcomment := &github.PullRequestComment{
				CommitID: &g.sha,
				Body:     &body,
				Path:     &comment.Path,
				Position: &comment.LnumDiff,
			}
			_, _, err := g.cli.PullRequests.CreateComment(g.owner, g.repo, g.pr, prcomment)
			return err
		})
	}
	return eg.Wait()
}

func (g *GitHubPullRequest) setPostedComment() error {
	g.postedcs = make(postedcomments)
	cs, err := g.comment()
	if err != nil {
		return err
	}
	for _, c := range cs {
		if c.Position == nil || c.Path == nil || c.Body == nil {
			// skip resolved comments. Or comments which do not have "path" nor
			// "body".
			continue
		}
		path := *c.Path
		pos := *c.Position
		body := *c.Body
		if _, ok := g.postedcs[path]; !ok {
			g.postedcs[path] = make(map[int][]string)
		}
		if _, ok := g.postedcs[path][pos]; !ok {
			g.postedcs[path][pos] = make([]string, 0)
		}
		g.postedcs[path][pos] = append(g.postedcs[path][pos], body)
	}
	return nil
}

// Diff returns a diff of PullRequest. It runs `git diff` locally instead of
// diff_url of GitHub Pull Request because diff of diff_url is not suited for
// comment API in a sense that diff of diff_url is equivalent to
// `git diff --no-renames`, we want diff which is equivalent to
// `git diff --find-renames`.
func (g *GitHubPullRequest) Diff() ([]byte, error) {
	pr, _, err := g.cli.PullRequests.Get(g.owner, g.repo, g.pr)
	if err != nil {
		return nil, err
	}
	b, err := exec.Command("git", "merge-base", g.sha, *pr.Base.SHA).Output()
	if err != nil {
		return nil, fmt.Errorf("failed to get merge-base commit: %v", err)
	}
	mergeBase := strings.Trim(string(b), "\n")
	return exec.Command("git", "diff", "--find-renames", mergeBase, g.sha).Output()
}

// Strip returns 1 as a strip of git diff.
func (g *GitHubPullRequest) Strip() int {
	return 1
}

func (g *GitHubPullRequest) comment() ([]*github.PullRequestComment, error) {
	comments, _, err := g.cli.PullRequests.ListComments(g.owner, g.repo, g.pr, nil)
	if err != nil {
		return nil, err
	}
	return comments, nil
}

// ---
// GitHub PullRequest Review API Implementation
// ref: https://github.com/google/go-github/issues/495

const (
	mediaTypePullRequestReview = "application/vnd.github.black-cat-preview+json"
)

// Review represents a pull request review.
type Review struct {
	Body     *string          `json:"body,omitempty"`
	Event    *string          `json:"event,omitempty"`
	Comments []*ReviewComment `json:"comments,omitempty"`
}

// ReviewComment represents draft review comments.
type ReviewComment struct {
	Path     *string `json:"path,omitempty"`
	Position *int    `json:"position,omitempty"`
	Body     *string `json:"body,omitempty"`
}

// CreateReview creates a new review comment on the specified pull request.
//
// GitHub API docs: https://developer.github.com/v3/pulls/reviews/#create-a-pull-request-review
func (g *GitHubPullRequest) CreateReview(owner, repo string, number int, review *Review) (*github.PullRequestReview, *github.Response, error) {
	u := fmt.Sprintf("repos/%v/%v/pulls/%d/reviews", owner, repo, number)

	req, err := g.cli.NewRequest("POST", u, review)
	if err != nil {
		return nil, nil, err
	}

	req.Header.Set("Accept", mediaTypePullRequestReview)

	r := new(github.PullRequestReview)
	resp, err := g.cli.Do(req, r)
	if err != nil {
		log.Printf("GitHub Review API error: %v", err)
		return nil, resp, err
	}
	return r, resp, err
}
<|endoftext|>


<commit_before>package main

import (
  "regexp"
  "strconv"
  "log"
)

type GitHub struct {
  Token     string
}

func NewGitHub(token string) *GitHub {
  t := new(GitHub)
  t.Token = token

  return t
}

type GitCommit struct {
  Commit    struct {
    Message string  `json:"message"`
  }                 `json:"commit"`
}

type WebHook struct {
  Name      string    `json:"name"`
  Active    bool      `json:"active"`
  Events    []string  `json:"events"`
  Config    struct {
    Type    string    `json:"content_type"`
    URL     string    `json:"url"`
  }                   `json:"config"`
}

func (this *GitHub) AuthQuery() string {
  return "access_token=" + this.Token
}

func (this *GitHub) BaseURL() string {
  return "https://api.github.com/"
}

type IssueSpec struct {
   rid string
   iid int
}

func extractIssueIds(message string, repoid string) []IssueSpec {
  res := make([]IssueSpec, 0)

  /* Assuming no single commit can close more than 256 issues okay */
  re := regexp.MustCompile("(?i)(?:close|closes|closed|fix|fixes|fixed|resolve|resolves|resolved)[[:space:]]*([a-z0-9][a-z0-9-]{0,38}[a-z0-9]/[a-z0-9][a-z0-9-]{0,38}[a-z0-9])?#([0-9]*)")
  if catch := re.FindAllStringSubmatch(message, 256); catch != nil {
    for _, v := range catch {
      /* Check if there was a repo specification before # */
      var repo string
      if len(v[1]) > 0 {
        repo = v[1]
      } else {
        repo = repoid
      }

      /* Add the new cath */
      iid, _ := strconv.Atoi(v[2])
      res = append(res, IssueSpec{ repo, iid } )
    }
  }

  return res
}

/* List ids of issues affected by a PR */
func (this *GitHub) AffectedIssues(pr IssueSpec) []IssueSpec {
  res := make([]IssueSpec, 0)

  /* Fetching commit data for the PR */
  var commits []GitCommit
  GenGET(this, "repos/" + pr.rid + "/pulls/" + strconv.Itoa(pr.iid) + "/commits", &commits)

  /* Parsing messages and finding relevant issues */
  for _, v := range commits {
    if issues := extractIssueIds(v.Commit.Message, pr.rid); len(issues) > 0 {
      res = append(res, issues...)
    }
  }

  return res
}

type labelSpec struct {
  Name  string  `json:"name"`
}

/* Check if the issue has a specific label attached to it */
// TODO: cache
func (this *GitHub) HasLabel(issue IssueSpec, label string) bool {
  var oldldbs []labelSpec
  GenGET(this, "repos/" + issue.rid + "/issues/" + strconv.Itoa(issue.iid) + "/labels", &oldldbs)

  for _, v := range oldldbs {
    if v.Name == label {
      return true
    }
  }

  return false
}

/* Adds a label to the issue */
func (this *GitHub) AddLabel(issue IssueSpec, label string) {
  /* Checking if the label isn't there yet to prevent Trello-GitHub recursion */
  if !this.HasLabel(issue, label) {
    log.Printf("Adding label %s to %s#%d", label, issue.rid, issue.iid)
    lbls := [...]string { label }
    GenPOSTJSON(this, "repos/" + issue.rid + "/issues/" + strconv.Itoa(issue.iid) + "/labels", nil, &lbls)
  }
}

/* Removes a label from the issue */
func (this *GitHub) DelLabel(issue IssueSpec, label string) {
  /* Checking if the label is present actually */
  if this.HasLabel(issue, label) {
    log.Printf("Removing label %s from %s#%d", label, issue.rid, issue.iid)
    GenDEL(this, "repos/" + issue.rid + "/issues/" + strconv.Itoa(issue.iid) + "/labels/" + label)
  }
}

/* Check and install webhooks on a repository */
// TODO secret
// TODO don't fail if we don't have access
func (this *GitHub) EnsureHook(repoid string, callbackURLbase string) {
  /* Retrieving previously installed hooks */
  var hooks []WebHook
  GenGET(this, "repos/" + repoid + "/hooks", &hooks)

  hookevts := map[string] struct { event string; found bool } {
    "/issues": { "issues", false },
    "/pull": { "pull_request", false},
  }

  /* Checking if there is a hook with exact same parameters */
  for _, v := range hooks {
    for k, f := range hookevts {
      if len(v.Events) > 0 && v.Events[0] == f.event && v.Config.URL == callbackURLbase + k {
        log.Printf("Found an existing GitHub hook at %s for %s, reusing.", v.Config.URL, repoid)
        hookevts[k] = struct{event string; found bool}{ f.event, true }
      }
    }
  }

  /* Creating hooks that failed */
  for k, f := range hookevts {
    if !f.found {
      /* TODO Compound initialisation? */
      var wh WebHook
      wh.Name = "web"
      wh.Active = true
      wh.Events = []string{ f.event }
      wh.Config.Type = "json"
      wh.Config.URL = callbackURLbase + k
      log.Printf("Creating a hook for %s at %s", wh.Config.URL)
      GenPOSTJSON(this, "repos/" + repoid + "/hooks", nil, &wh)
    }
  }
}
<commit_msg>Corrected printf<commit_after>package main

import (
  "regexp"
  "strconv"
  "log"
)

type GitHub struct {
  Token     string
}

func NewGitHub(token string) *GitHub {
  t := new(GitHub)
  t.Token = token

  return t
}

type GitCommit struct {
  Commit    struct {
    Message string  `json:"message"`
  }                 `json:"commit"`
}

type WebHook struct {
  Name      string    `json:"name"`
  Active    bool      `json:"active"`
  Events    []string  `json:"events"`
  Config    struct {
    Type    string    `json:"content_type"`
    URL     string    `json:"url"`
  }                   `json:"config"`
}

func (this *GitHub) AuthQuery() string {
  return "access_token=" + this.Token
}

func (this *GitHub) BaseURL() string {
  return "https://api.github.com/"
}

type IssueSpec struct {
   rid string
   iid int
}

func extractIssueIds(message string, repoid string) []IssueSpec {
  res := make([]IssueSpec, 0)

  /* Assuming no single commit can close more than 256 issues okay */
  re := regexp.MustCompile("(?i)(?:close|closes|closed|fix|fixes|fixed|resolve|resolves|resolved)[[:space:]]*([a-z0-9][a-z0-9-]{0,38}[a-z0-9]/[a-z0-9][a-z0-9-]{0,38}[a-z0-9])?#([0-9]*)")
  if catch := re.FindAllStringSubmatch(message, 256); catch != nil {
    for _, v := range catch {
      /* Check if there was a repo specification before # */
      var repo string
      if len(v[1]) > 0 {
        repo = v[1]
      } else {
        repo = repoid
      }

      /* Add the new cath */
      iid, _ := strconv.Atoi(v[2])
      res = append(res, IssueSpec{ repo, iid } )
    }
  }

  return res
}

/* List ids of issues affected by a PR */
func (this *GitHub) AffectedIssues(pr IssueSpec) []IssueSpec {
  res := make([]IssueSpec, 0)

  /* Fetching commit data for the PR */
  var commits []GitCommit
  GenGET(this, "repos/" + pr.rid + "/pulls/" + strconv.Itoa(pr.iid) + "/commits", &commits)

  /* Parsing messages and finding relevant issues */
  for _, v := range commits {
    if issues := extractIssueIds(v.Commit.Message, pr.rid); len(issues) > 0 {
      res = append(res, issues...)
    }
  }

  return res
}

type labelSpec struct {
  Name  string  `json:"name"`
}

/* Check if the issue has a specific label attached to it */
// TODO: cache
func (this *GitHub) HasLabel(issue IssueSpec, label string) bool {
  var oldldbs []labelSpec
  GenGET(this, "repos/" + issue.rid + "/issues/" + strconv.Itoa(issue.iid) + "/labels", &oldldbs)

  for _, v := range oldldbs {
    if v.Name == label {
      return true
    }
  }

  return false
}

/* Adds a label to the issue */
func (this *GitHub) AddLabel(issue IssueSpec, label string) {
  /* Checking if the label isn't there yet to prevent Trello-GitHub recursion */
  if !this.HasLabel(issue, label) {
    log.Printf("Adding label %s to %s#%d", label, issue.rid, issue.iid)
    lbls := [...]string { label }
    GenPOSTJSON(this, "repos/" + issue.rid + "/issues/" + strconv.Itoa(issue.iid) + "/labels", nil, &lbls)
  }
}

/* Removes a label from the issue */
func (this *GitHub) DelLabel(issue IssueSpec, label string) {
  /* Checking if the label is present actually */
  if this.HasLabel(issue, label) {
    log.Printf("Removing label %s from %s#%d", label, issue.rid, issue.iid)
    GenDEL(this, "repos/" + issue.rid + "/issues/" + strconv.Itoa(issue.iid) + "/labels/" + label)
  }
}

/* Check and install webhooks on a repository */
// TODO secret
// TODO don't fail if we don't have access
func (this *GitHub) EnsureHook(repoid string, callbackURLbase string) {
  /* Retrieving previously installed hooks */
  var hooks []WebHook
  GenGET(this, "repos/" + repoid + "/hooks", &hooks)

  hookevts := map[string] struct { event string; found bool } {
    "/issues": { "issues", false },
    "/pull": { "pull_request", false},
  }

  /* Checking if there is a hook with exact same parameters */
  for _, v := range hooks {
    for k, f := range hookevts {
      if len(v.Events) > 0 && v.Events[0] == f.event && v.Config.URL == callbackURLbase + k {
        log.Printf("Found an existing GitHub hook at %s for %s, reusing.", v.Config.URL, repoid)
        hookevts[k] = struct{event string; found bool}{ f.event, true }
      }
    }
  }

  /* Creating hooks that failed */
  for k, f := range hookevts {
    if !f.found {
      /* TODO Compound initialisation? */
      var wh WebHook
      wh.Name = "web"
      wh.Active = true
      wh.Events = []string{ f.event }
      wh.Config.Type = "json"
      wh.Config.URL = callbackURLbase + k
      log.Printf("Creating a hook for %s at %s", wh.Config.URL, repoid)
      GenPOSTJSON(this, "repos/" + repoid + "/hooks", nil, &wh)
    }
  }
}
<|endoftext|>


<commit_before>package client

import (
	"crypto/tls"
	"errors"
	"io"
	"log"
	"net/http"
	"time"

	"s3cli/config"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

// S3Blobstore encapsulates interactions with an S3 compatible blobstore
type S3Blobstore struct {
	s3Client    *s3.S3
	s3cliConfig config.S3Cli
}

var errorInvalidCredentialsSourceValue = errors.New("the client operates in read only mode. Change 'credentials_source' parameter value ")

// New returns a BlobstoreClient if the configuration file backing configFile is valid
func New(configFile io.Reader) (S3Blobstore, error) {
	c, err := config.NewFromReader(configFile)
	if err != nil {
		return S3Blobstore{}, err
	}

	transport := *http.DefaultTransport.(*http.Transport)
	transport.TLSClientConfig = &tls.Config{
		InsecureSkipVerify: !c.SSLVerifyPeer,
	}

	httpClient := &http.Client{Transport: &transport}

	s3Config := aws.NewConfig().
		WithLogLevel(aws.LogOff).
		WithS3ForcePathStyle(true).
		WithDisableSSL(!c.UseSSL).
		WithHTTPClient(httpClient)

	if c.UseRegion() {
		s3Config = s3Config.WithRegion(c.Region).WithEndpoint(c.S3Endpoint())
	} else {
		s3Config = s3Config.WithRegion(config.EmptyRegion).WithEndpoint(c.S3Endpoint())
	}

	if c.CredentialsSource == config.StaticCredentialsSource {
		s3Config = s3Config.WithCredentials(credentials.NewStaticCredentials(c.AccessKeyID, c.SecretAccessKey, ""))
	}

	if c.CredentialsSource == config.NoneCredentialsSource {
		s3Config = s3Config.WithCredentials(credentials.AnonymousCredentials)
	}

	s3Session := session.New(s3Config)
	s3Client := s3.New(s3Session)

	if c.UseV2SigningMethod {
		setv2Handlers(s3Client)
	}

	return S3Blobstore{s3Client: s3Client, s3cliConfig: c}, nil
}

// Get fetches a blob from an S3 compatible blobstore
// Destination will be overwritten if exists
func (client *S3Blobstore) Get(src string, dest io.WriterAt) error {
	downloader := s3manager.NewDownloaderWithClient(client.s3Client)

	_, err := downloader.Download(dest, &s3.GetObjectInput{
		Bucket: aws.String(client.s3cliConfig.BucketName),
		Key:    aws.String(src),
	})

	if err != nil {
		return err
	}

	return nil
}

// Put uploads a blob to an S3 compatible blobstore
func (client *S3Blobstore) Put(src io.ReadSeeker, dest string) error {
	if client.s3cliConfig.CredentialsSource == config.NoneCredentialsSource {
		return errorInvalidCredentialsSourceValue
	}

	uploader := s3manager.NewUploaderWithClient(client.s3Client)

	retry := 0
	maxRetries := 3
	for {
		putResult, err := uploader.Upload(&s3manager.UploadInput{
			Body:   src,
			Bucket: aws.String(client.s3cliConfig.BucketName),
			Key:    aws.String(dest),
		})

		if err != nil {
			if _, ok := err.(s3manager.MultiUploadFailure); ok {
				if retry > maxRetries {
					return err
				}
				retry++
				time.Sleep(time.Second * time.Duration(retry))
				continue
			}
			return err
		}

		log.Println("Successfully uploaded file to", putResult.Location)
	}
	return nil
}

// Delete removes a blob from an S3 compatible blobstore. If the object does
// not exist, Delete does not return an error.
func (client *S3Blobstore) Delete(dest string) error {
	if client.s3cliConfig.CredentialsSource == config.NoneCredentialsSource {
		return errorInvalidCredentialsSourceValue
	}

	deleteParams := &s3.DeleteObjectInput{
		Bucket: aws.String(client.s3cliConfig.BucketName),
		Key:    aws.String(dest),
	}

	_, err := client.s3Client.DeleteObject(deleteParams)

	if err == nil {
		return nil
	}

	if reqErr, ok := err.(awserr.RequestFailure); ok {
		if reqErr.StatusCode() == 404 {
			return nil
		}
	}
	return err
}

// Exists checks if blob exists in an S3 compatible blobstore
func (client *S3Blobstore) Exists(dest string) (bool, error) {

	existsParams := &s3.HeadObjectInput{
		Bucket: aws.String(client.s3cliConfig.BucketName),
		Key:    aws.String(dest),
	}

	_, err := client.s3Client.HeadObject(existsParams)

	if err == nil {
		log.Printf("File '%s' exists in bucket '%s'\n", dest, client.s3cliConfig.BucketName)
		return true, nil
	}

	if reqErr, ok := err.(awserr.RequestFailure); ok {
		if reqErr.StatusCode() == 404 {
			log.Printf("File '%s' does not exist in bucket '%s'\n", dest, client.s3cliConfig.BucketName)
			return false, nil
		}
	}
	return false, err
}
<commit_msg>resolve conflicts<commit_after>package client

import (
	"crypto/tls"
	"errors"
	"io"
	"log"
	"net/http"
	"time"

	"s3cli/config"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

// S3Blobstore encapsulates interactions with an S3 compatible blobstore
type S3Blobstore struct {
	s3Client    *s3.S3
	s3cliConfig config.S3Cli
}

var errorInvalidCredentialsSourceValue = errors.New("the client operates in read only mode. Change 'credentials_source' parameter value ")

// New returns a BlobstoreClient if the configuration file backing configFile is valid
func New(configFile io.Reader) (S3Blobstore, error) {
	c, err := config.NewFromReader(configFile)
	if err != nil {
		return S3Blobstore{}, err
	}

	s3Client := MakeClient(c)

	return S3Blobstore{s3Client: s3Client, s3cliConfig: c}, nil
}

func MakeClient(c config.S3Cli) *s3.S3 {
	transport := *http.DefaultTransport.(*http.Transport)
	transport.TLSClientConfig = &tls.Config{
		InsecureSkipVerify: !c.SSLVerifyPeer,
	}

	httpClient := &http.Client{Transport: &transport}

	s3Config := aws.NewConfig().
		WithLogLevel(aws.LogOff).
		WithS3ForcePathStyle(true).
		WithDisableSSL(!c.UseSSL).
		WithHTTPClient(httpClient)

	if c.UseRegion() {
		s3Config = s3Config.WithRegion(c.Region).WithEndpoint(c.S3Endpoint())
	} else {
		s3Config = s3Config.WithRegion(config.EmptyRegion).WithEndpoint(c.S3Endpoint())
	}

	if c.CredentialsSource == config.StaticCredentialsSource {
		s3Config = s3Config.WithCredentials(credentials.NewStaticCredentials(c.AccessKeyID, c.SecretAccessKey, ""))
	}

	if c.CredentialsSource == config.NoneCredentialsSource {
		s3Config = s3Config.WithCredentials(credentials.AnonymousCredentials)
	}

	s3Session := session.New(s3Config)
	s3Client := s3.New(s3Session)

	if c.UseV2SigningMethod {
		setv2Handlers(s3Client)
	}

	return s3Client
}

// Get fetches a blob from an S3 compatible blobstore
// Destination will be overwritten if exists
func (client *S3Blobstore) Get(src string, dest io.WriterAt) error {
	downloader := s3manager.NewDownloaderWithClient(client.s3Client)

	_, err := downloader.Download(dest, &s3.GetObjectInput{
		Bucket: aws.String(client.s3cliConfig.BucketName),
		Key:    aws.String(src),
	})

	if err != nil {
		return err
	}

	return nil
}

// Put uploads a blob to an S3 compatible blobstore
func (client *S3Blobstore) Put(src io.ReadSeeker, dest string) error {
	cfg := client.s3cliConfig
	if cfg.CredentialsSource == config.NoneCredentialsSource {
		return errorInvalidCredentialsSourceValue
	}

	uploader := s3manager.NewUploaderWithClient(client.s3Client)

	retry := 0
	maxRetries := 3
	for {
		uploadInput := &s3manager.UploadInput{
			Body:   src,
			Bucket: aws.String(cfg.BucketName),
			Key:    aws.String(dest),
		}
		if cfg.ServerSideEncryption != "" {
			uploadInput.ServerSideEncryption = aws.String(cfg.ServerSideEncryption)
		}
		if cfg.SSEKMSKeyID != "" {
			uploadInput.SSEKMSKeyId = aws.String(cfg.SSEKMSKeyID)
		}

		putResult, err := uploader.Upload(uploadInput)
		if err != nil {
			if _, ok := err.(s3manager.MultiUploadFailure); ok {
				if retry > maxRetries {
					return err
				}
				retry++
				time.Sleep(time.Second * time.Duration(retry))
				continue
			}
			return err
		}

		log.Println("Successfully uploaded file to", putResult.Location)
	}
	return nil
}

// Delete removes a blob from an S3 compatible blobstore. If the object does
// not exist, Delete does not return an error.
func (client *S3Blobstore) Delete(dest string) error {
	if client.s3cliConfig.CredentialsSource == config.NoneCredentialsSource {
		return errorInvalidCredentialsSourceValue
	}

	deleteParams := &s3.DeleteObjectInput{
		Bucket: aws.String(client.s3cliConfig.BucketName),
		Key:    aws.String(dest),
	}

	_, err := client.s3Client.DeleteObject(deleteParams)

	if err == nil {
		return nil
	}

	if reqErr, ok := err.(awserr.RequestFailure); ok {
		if reqErr.StatusCode() == 404 {
			return nil
		}
	}
	return err
}

// Exists checks if blob exists in an S3 compatible blobstore
func (client *S3Blobstore) Exists(dest string) (bool, error) {

	existsParams := &s3.HeadObjectInput{
		Bucket: aws.String(client.s3cliConfig.BucketName),
		Key:    aws.String(dest),
	}

	_, err := client.s3Client.HeadObject(existsParams)

	if err == nil {
		log.Printf("File '%s' exists in bucket '%s'\n", dest, client.s3cliConfig.BucketName)
		return true, nil
	}

	if reqErr, ok := err.(awserr.RequestFailure); ok {
		if reqErr.StatusCode() == 404 {
			log.Printf("File '%s' does not exist in bucket '%s'\n", dest, client.s3cliConfig.BucketName)
			return false, nil
		}
	}
	return false, err
}
<|endoftext|>