<commit_before>// Package pathio is a package that allows writing to and reading from different types of paths transparently.
// It supports two types of paths:
//  1. Local file paths
//  2. S3 File Paths (s3://bucket/key)
//
// Note that using s3 paths requires setting two environment variables
//  1. AWS_SECRET_ACCESS_KEY
//  2. AWS_ACCESS_KEY_ID
package pathio

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

const (
	defaultLocation = "us-east-1"
	aesAlgo         = "AES256"
)

// generate a mock for Pathio
//go:generate $GOPATH/bin/mockgen -source=$GOFILE -destination=gen_mock_s3handler.go -package=pathio

// Pathio is a defined interface for accessing both S3 and local files.
type Pathio interface {
	Reader(path string) (rc io.ReadCloser, err error)
	Write(path string, input []byte) error
	WriteReader(path string, input io.ReadSeeker) error
	ListFiles(path string) ([]string, error)
	Exists(path string) (bool, error)
}

// Client is the pathio client used to access the local file system and S3.
// To configure options on the client, create a new Client and call its methods
// directly.
// 	&Client{
// 		disableS3Encryption: true, // disables encryption
// 		Region: "us-east-1", // hardcodes the s3 region, instead of looking it up
// 	}.Write(...)
type Client struct {
	disableS3Encryption bool
	Region              string
	providedConfig      *aws.Config
}

// DefaultClient is the default pathio client called by the Reader, Writer, and
// WriteReader methods. It has S3 encryption enabled.
var DefaultClient Pathio = &Client{}

// NewClient creates a new client that utilizes the provided AWS config. This can
// be leveraged to enforce more limited permissions.
func NewClient(cfg *aws.Config) *Client {
	return &Client{
		providedConfig: cfg,
		Region:         "us-west-1",
	}
}

// Reader calls DefaultClient's Reader method.
func Reader(path string) (rc io.ReadCloser, err error) {
	return DefaultClient.Reader(path)
}

// Write calls DefaultClient's Write method.
func Write(path string, input []byte) error {
	return DefaultClient.Write(path, input)
}

// WriteReader calls DefaultClient's WriteReader method.
func WriteReader(path string, input io.ReadSeeker) error {
	return DefaultClient.WriteReader(path, input)
}

// ListFiles calls DefaultClient's ListFiles method.
func ListFiles(path string) ([]string, error) {
	return DefaultClient.ListFiles(path)
}

// s3Handler defines the interface that pathio needs for AWS access.
type s3Handler interface {
	GetBucketLocation(input *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error)
	GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error)
	PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error)
	ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
	HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
}

type s3Connection struct {
	handler s3Handler
	bucket  string
	key     string
}

// Reader returns an io.Reader for the specified path. The path can either be a local file path
// or an S3 path. It is the caller's responsibility to close rc.
func (c *Client) Reader(path string) (rc io.ReadCloser, err error) {
	if strings.HasPrefix(path, "s3://") {
		s3Conn, err := c.s3ConnectionInformation(path, c.Region)
		if err != nil {
			return nil, err
		}
		return s3FileReader(s3Conn)
	}
	// Local file path
	return os.Open(path)
}

// Write writes a byte array to the specified path. The path can be either a local file path or an
// S3 path.
func (c *Client) Write(path string, input []byte) error {
	return c.WriteReader(path, bytes.NewReader(input))
}

// WriteReader writes all the data read from the specified io.Reader to the
// output path. The path can either a local file path or an S3 path.
func (c *Client) WriteReader(path string, input io.ReadSeeker) error {
	// return the file pointer to the start before reading from it when writing
	if offset, err := input.Seek(0, os.SEEK_SET); err != nil || offset != 0 {
		return fmt.Errorf("failed to reset the file pointer to 0. offset: %d; error %s", offset, err)
	}

	if strings.HasPrefix(path, "s3://") {
		s3Conn, err := c.s3ConnectionInformation(path, c.Region)
		if err != nil {
			return err
		}
		return writeToS3(s3Conn, input, c.disableS3Encryption)
	}
	return writeToLocalFile(path, input)
}

// ListFiles lists all the files/directories in the directory. It does not recurse
func (c *Client) ListFiles(path string) ([]string, error) {
	if strings.HasPrefix(path, "s3://") {
		s3Conn, err := c.s3ConnectionInformation(path, c.Region)
		if err != nil {
			return nil, err
		}
		return lsS3(s3Conn)
	}
	return lsLocal(path)
}

// Exists determines if a path does or does not exist.
// NOTE: S3 is eventually consistent so keep in mind that there is a delay.
func (c *Client) Exists(path string) (bool, error) {
	if strings.HasPrefix(path, "s3://") {
		s3Conn, err := c.s3ConnectionInformation(path, c.Region)
		if err != nil {
			return false, err
		}
		return existsS3(s3Conn)
	}
	return existsLocal(path)
}

func existsS3(s3Conn s3Connection) (bool, error) {
	_, err := s3Conn.handler.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(s3Conn.bucket),
		Key:    aws.String(s3Conn.key),
	})
	if err != nil {
		if aerr, ok := err.(s3.RequestFailure); ok && aerr.StatusCode() == 404 {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func existsLocal(path string) (bool, error) {
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		return false, nil
	}
	return err == nil, err
}

func lsS3(s3Conn s3Connection) ([]string, error) {
	params := s3.ListObjectsInput{
		Bucket:    aws.String(s3Conn.bucket),
		Prefix:    aws.String(s3Conn.key),
		Delimiter: aws.String("/"),
	}
	finalResults := []string{}

	// s3 ListObjects limits the respose to 1000 objects and marks as truncated if there were more
	// In this case we set a Marker that the next query will start from.
	// We also ensure that prefixes are not duplicated
	for {
		resp, err := s3Conn.handler.ListObjects(&params)
		if err != nil {
			return nil, err
		}
		if len(resp.CommonPrefixes) > 0 && elementInSlice(finalResults, *resp.CommonPrefixes[0].Prefix) {
			resp.CommonPrefixes = resp.CommonPrefixes[1:]
		}
		results := make([]string, len(resp.Contents)+len(resp.CommonPrefixes))
		for i, val := range resp.CommonPrefixes {
			results[i] = *val.Prefix
		}
		for i, val := range resp.Contents {
			results[i+len(resp.CommonPrefixes)] = *val.Key
		}
		finalResults = append(finalResults, results...)

		if resp.IsTruncated != nil && *resp.IsTruncated {
			params.Marker = aws.String(results[len(results)-1])
		} else {
			break
		}
	}
	return finalResults, nil
}

func elementInSlice(slice []string, elem string) bool {
	for _, v := range slice {
		if elem == v {
			return true
		}
	}
	return false
}

func lsLocal(path string) ([]string, error) {
	resp, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}
	results := make([]string, len(resp))
	for i, val := range resp {
		results[i] = val.Name()
	}
	return results, nil
}

// s3FileReader converts an S3Path into an io.ReadCloser
func s3FileReader(s3Conn s3Connection) (io.ReadCloser, error) {
	params := s3.GetObjectInput{
		Bucket: aws.String(s3Conn.bucket),
		Key:    aws.String(s3Conn.key),
	}
	resp, err := s3Conn.handler.GetObject(&params)
	if err != nil {
		return nil, err
	}
	return resp.Body, nil
}

// writeToS3 uploads the given file to S3
func writeToS3(s3Conn s3Connection, input io.ReadSeeker, disableEncryption bool) error {
	params := s3.PutObjectInput{
		Bucket: aws.String(s3Conn.bucket),
		Key:    aws.String(s3Conn.key),
		Body:   input,
	}
	if !disableEncryption {
		algo := aesAlgo
		params.ServerSideEncryption = &algo
	}
	_, err := s3Conn.handler.PutObject(&params)
	return err
}

// writeToLocalFile writes the given file locally
func writeToLocalFile(path string, input io.ReadSeeker) error {
	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
		return err
	}
	file, err := os.Create(path)
	defer file.Close()
	if err != nil {
		return err
	}
	_, err = io.Copy(file, input)
	return err
}

// parseS3path parses an S3 path (s3://bucket/key) and returns a bucket, key, error tuple
func parseS3Path(path string) (string, string, error) {
	// S3 path names are of the form s3://bucket/key
	stringsArray := strings.SplitN(path, "/", 4)
	if len(stringsArray) < 4 {
		return "", "", fmt.Errorf("Invalid s3 path %s", path)
	}
	bucketName := stringsArray[2]
	// Everything after the third slash is the key
	key := stringsArray[3]
	return bucketName, key, nil
}

// s3ConnectionInformation parses the s3 path and returns the s3 connection from the
// correct region, as well as the bucket, and key
func (c *Client) s3ConnectionInformation(path, region string) (s3Connection, error) {
	bucket, key, err := parseS3Path(path)
	if err != nil {
		return s3Connection{}, err
	}

	// If no region passed in, look up region in S3
	if region == "" {
		region, err = getRegionForBucket(c.newS3Handler(defaultLocation), bucket)
		if err != nil {
			return s3Connection{}, err
		}
	}

	return s3Connection{c.newS3Handler(region), bucket, key}, nil
}

// getRegionForBucket looks up the region name for the given bucket
func getRegionForBucket(svc s3Handler, name string) (string, error) {
	// Any region will work for the region lookup, but the request MUST use
	// PathStyle
	params := s3.GetBucketLocationInput{
		Bucket: aws.String(name),
	}
	resp, err := svc.GetBucketLocation(&params)
	if err != nil {
		return "", fmt.Errorf("Failed to get location for bucket '%s', %s", name, err)
	}
	if resp.LocationConstraint == nil {
		// "US Standard", returns an empty region, which means us-east-1
		// See http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETlocation.html
		return defaultLocation, nil
	}
	return *resp.LocationConstraint, nil
}

type liveS3Handler struct {
	liveS3 *s3.S3
}

func (m *liveS3Handler) GetBucketLocation(input *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) {
	return m.liveS3.GetBucketLocation(input)
}

func (m *liveS3Handler) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
	return m.liveS3.GetObject(input)
}

func (m *liveS3Handler) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
	return m.liveS3.PutObject(input)
}

func (m *liveS3Handler) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
	return m.liveS3.ListObjects(input)
}

func (m *liveS3Handler) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
	return m.liveS3.HeadObject(input)
}

func (c *Client) newS3Handler(region string) *liveS3Handler {
	if c.providedConfig != nil {
		return &liveS3Handler{
			liveS3: s3.New(session.New(), c.providedConfig.WithRegion(region).WithS3ForcePathStyle(true)),
		}
	}

	config := aws.NewConfig().WithRegion(region).WithS3ForcePathStyle(true)
	session := session.New()
	return &liveS3Handler{s3.New(session, config)}
}
<commit_msg>remove hardcoded region<commit_after>// Package pathio is a package that allows writing to and reading from different types of paths transparently.
// It supports two types of paths:
//  1. Local file paths
//  2. S3 File Paths (s3://bucket/key)
//
// Note that using s3 paths requires setting two environment variables
//  1. AWS_SECRET_ACCESS_KEY
//  2. AWS_ACCESS_KEY_ID
package pathio

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

const (
	defaultLocation = "us-east-1"
	aesAlgo         = "AES256"
)

// generate a mock for Pathio
//go:generate $GOPATH/bin/mockgen -source=$GOFILE -destination=gen_mock_s3handler.go -package=pathio

// Pathio is a defined interface for accessing both S3 and local files.
type Pathio interface {
	Reader(path string) (rc io.ReadCloser, err error)
	Write(path string, input []byte) error
	WriteReader(path string, input io.ReadSeeker) error
	ListFiles(path string) ([]string, error)
	Exists(path string) (bool, error)
}

// Client is the pathio client used to access the local file system and S3.
// To configure options on the client, create a new Client and call its methods
// directly.
// 	&Client{
// 		disableS3Encryption: true, // disables encryption
// 		Region: "us-east-1", // hardcodes the s3 region, instead of looking it up
// 	}.Write(...)
type Client struct {
	disableS3Encryption bool
	Region              string
	providedConfig      *aws.Config
}

// DefaultClient is the default pathio client called by the Reader, Writer, and
// WriteReader methods. It has S3 encryption enabled.
var DefaultClient Pathio = &Client{}

// NewClient creates a new client that utilizes the provided AWS config. This can
// be leveraged to enforce more limited permissions.
func NewClient(cfg *aws.Config) *Client {
	return &Client{
		providedConfig: cfg,
	}
}

// Reader calls DefaultClient's Reader method.
func Reader(path string) (rc io.ReadCloser, err error) {
	return DefaultClient.Reader(path)
}

// Write calls DefaultClient's Write method.
func Write(path string, input []byte) error {
	return DefaultClient.Write(path, input)
}

// WriteReader calls DefaultClient's WriteReader method.
func WriteReader(path string, input io.ReadSeeker) error {
	return DefaultClient.WriteReader(path, input)
}

// ListFiles calls DefaultClient's ListFiles method.
func ListFiles(path string) ([]string, error) {
	return DefaultClient.ListFiles(path)
}

// s3Handler defines the interface that pathio needs for AWS access.
type s3Handler interface {
	GetBucketLocation(input *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error)
	GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error)
	PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error)
	ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
	HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
}

type s3Connection struct {
	handler s3Handler
	bucket  string
	key     string
}

// Reader returns an io.Reader for the specified path. The path can either be a local file path
// or an S3 path. It is the caller's responsibility to close rc.
func (c *Client) Reader(path string) (rc io.ReadCloser, err error) {
	if strings.HasPrefix(path, "s3://") {
		s3Conn, err := c.s3ConnectionInformation(path, c.Region)
		if err != nil {
			return nil, err
		}
		return s3FileReader(s3Conn)
	}
	// Local file path
	return os.Open(path)
}

// Write writes a byte array to the specified path. The path can be either a local file path or an
// S3 path.
func (c *Client) Write(path string, input []byte) error {
	return c.WriteReader(path, bytes.NewReader(input))
}

// WriteReader writes all the data read from the specified io.Reader to the
// output path. The path can either a local file path or an S3 path.
func (c *Client) WriteReader(path string, input io.ReadSeeker) error {
	// return the file pointer to the start before reading from it when writing
	if offset, err := input.Seek(0, os.SEEK_SET); err != nil || offset != 0 {
		return fmt.Errorf("failed to reset the file pointer to 0. offset: %d; error %s", offset, err)
	}

	if strings.HasPrefix(path, "s3://") {
		s3Conn, err := c.s3ConnectionInformation(path, c.Region)
		if err != nil {
			return err
		}
		return writeToS3(s3Conn, input, c.disableS3Encryption)
	}
	return writeToLocalFile(path, input)
}

// ListFiles lists all the files/directories in the directory. It does not recurse
func (c *Client) ListFiles(path string) ([]string, error) {
	if strings.HasPrefix(path, "s3://") {
		s3Conn, err := c.s3ConnectionInformation(path, c.Region)
		if err != nil {
			return nil, err
		}
		return lsS3(s3Conn)
	}
	return lsLocal(path)
}

// Exists determines if a path does or does not exist.
// NOTE: S3 is eventually consistent so keep in mind that there is a delay.
func (c *Client) Exists(path string) (bool, error) {
	if strings.HasPrefix(path, "s3://") {
		s3Conn, err := c.s3ConnectionInformation(path, c.Region)
		if err != nil {
			return false, err
		}
		return existsS3(s3Conn)
	}
	return existsLocal(path)
}

func existsS3(s3Conn s3Connection) (bool, error) {
	_, err := s3Conn.handler.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(s3Conn.bucket),
		Key:    aws.String(s3Conn.key),
	})
	if err != nil {
		if aerr, ok := err.(s3.RequestFailure); ok && aerr.StatusCode() == 404 {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func existsLocal(path string) (bool, error) {
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		return false, nil
	}
	return err == nil, err
}

func lsS3(s3Conn s3Connection) ([]string, error) {
	params := s3.ListObjectsInput{
		Bucket:    aws.String(s3Conn.bucket),
		Prefix:    aws.String(s3Conn.key),
		Delimiter: aws.String("/"),
	}
	finalResults := []string{}

	// s3 ListObjects limits the respose to 1000 objects and marks as truncated if there were more
	// In this case we set a Marker that the next query will start from.
	// We also ensure that prefixes are not duplicated
	for {
		resp, err := s3Conn.handler.ListObjects(&params)
		if err != nil {
			return nil, err
		}
		if len(resp.CommonPrefixes) > 0 && elementInSlice(finalResults, *resp.CommonPrefixes[0].Prefix) {
			resp.CommonPrefixes = resp.CommonPrefixes[1:]
		}
		results := make([]string, len(resp.Contents)+len(resp.CommonPrefixes))
		for i, val := range resp.CommonPrefixes {
			results[i] = *val.Prefix
		}
		for i, val := range resp.Contents {
			results[i+len(resp.CommonPrefixes)] = *val.Key
		}
		finalResults = append(finalResults, results...)

		if resp.IsTruncated != nil && *resp.IsTruncated {
			params.Marker = aws.String(results[len(results)-1])
		} else {
			break
		}
	}
	return finalResults, nil
}

func elementInSlice(slice []string, elem string) bool {
	for _, v := range slice {
		if elem == v {
			return true
		}
	}
	return false
}

func lsLocal(path string) ([]string, error) {
	resp, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}
	results := make([]string, len(resp))
	for i, val := range resp {
		results[i] = val.Name()
	}
	return results, nil
}

// s3FileReader converts an S3Path into an io.ReadCloser
func s3FileReader(s3Conn s3Connection) (io.ReadCloser, error) {
	params := s3.GetObjectInput{
		Bucket: aws.String(s3Conn.bucket),
		Key:    aws.String(s3Conn.key),
	}
	resp, err := s3Conn.handler.GetObject(&params)
	if err != nil {
		return nil, err
	}
	return resp.Body, nil
}

// writeToS3 uploads the given file to S3
func writeToS3(s3Conn s3Connection, input io.ReadSeeker, disableEncryption bool) error {
	params := s3.PutObjectInput{
		Bucket: aws.String(s3Conn.bucket),
		Key:    aws.String(s3Conn.key),
		Body:   input,
	}
	if !disableEncryption {
		algo := aesAlgo
		params.ServerSideEncryption = &algo
	}
	_, err := s3Conn.handler.PutObject(&params)
	return err
}

// writeToLocalFile writes the given file locally
func writeToLocalFile(path string, input io.ReadSeeker) error {
	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
		return err
	}
	file, err := os.Create(path)
	defer file.Close()
	if err != nil {
		return err
	}
	_, err = io.Copy(file, input)
	return err
}

// parseS3path parses an S3 path (s3://bucket/key) and returns a bucket, key, error tuple
func parseS3Path(path string) (string, string, error) {
	// S3 path names are of the form s3://bucket/key
	stringsArray := strings.SplitN(path, "/", 4)
	if len(stringsArray) < 4 {
		return "", "", fmt.Errorf("Invalid s3 path %s", path)
	}
	bucketName := stringsArray[2]
	// Everything after the third slash is the key
	key := stringsArray[3]
	return bucketName, key, nil
}

// s3ConnectionInformation parses the s3 path and returns the s3 connection from the
// correct region, as well as the bucket, and key
func (c *Client) s3ConnectionInformation(path, region string) (s3Connection, error) {
	bucket, key, err := parseS3Path(path)
	if err != nil {
		return s3Connection{}, err
	}

	// If no region passed in, look up region in S3
	if region == "" {
		region, err = getRegionForBucket(c.newS3Handler(defaultLocation), bucket)
		if err != nil {
			return s3Connection{}, err
		}
	}

	return s3Connection{c.newS3Handler(region), bucket, key}, nil
}

// getRegionForBucket looks up the region name for the given bucket
func getRegionForBucket(svc s3Handler, name string) (string, error) {
	// Any region will work for the region lookup, but the request MUST use
	// PathStyle
	params := s3.GetBucketLocationInput{
		Bucket: aws.String(name),
	}
	resp, err := svc.GetBucketLocation(&params)
	if err != nil {
		return "", fmt.Errorf("Failed to get location for bucket '%s', %s", name, err)
	}
	if resp.LocationConstraint == nil {
		// "US Standard", returns an empty region, which means us-east-1
		// See http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETlocation.html
		return defaultLocation, nil
	}
	return *resp.LocationConstraint, nil
}

type liveS3Handler struct {
	liveS3 *s3.S3
}

func (m *liveS3Handler) GetBucketLocation(input *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) {
	return m.liveS3.GetBucketLocation(input)
}

func (m *liveS3Handler) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
	return m.liveS3.GetObject(input)
}

func (m *liveS3Handler) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
	return m.liveS3.PutObject(input)
}

func (m *liveS3Handler) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
	return m.liveS3.ListObjects(input)
}

func (m *liveS3Handler) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
	return m.liveS3.HeadObject(input)
}

func (c *Client) newS3Handler(region string) *liveS3Handler {
	if c.providedConfig != nil {
		return &liveS3Handler{
			liveS3: s3.New(session.New(), c.providedConfig.WithRegion(region).WithS3ForcePathStyle(true)),
		}
	}

	config := aws.NewConfig().WithRegion(region).WithS3ForcePathStyle(true)
	session := session.New()
	return &liveS3Handler{s3.New(session, config)}
}
<|endoftext|>


<commit_before>package todotxt

import (
        "time"
        "os"
        "bufio"
        "strings"
        "regexp"
        "sort"
        "unicode"
        "fmt"
)

type Task struct {
        id int
        todo string
        priority byte
        create_date time.Time
        contexts []string
        projects []string
        raw_todo string
        finished bool
        id_padding int
}

type TaskList []Task

func CreateTask(id int, text string) (Task) {
        var task = Task{}
        task.id = id
        task.raw_todo = text

        splits := strings.Split(text, " ")

        if text[0] == 'x' &&
           text[1] == ' ' &&
           !unicode.IsSpace(rune(text[2])) {
                task.finished = true
                splits = splits[1:]
        }

        head := splits[0]

        if (len(head) == 3) &&
           (head[0] == '(') &&
           (head[2] == ')') &&
           (head[1] >= 65 && head[1] <= 90) { // checking if it's in range [A-Z]
                task.priority = head[1]
                splits = splits[1:]
        }

        date_regexp := "([\\d]{4})-([\\d]{2})-([\\d]{2})"
        if match, _ := regexp.MatchString(date_regexp, splits[0]); match {
                if date, e := time.Parse("2006-01-02", splits[0]); e != nil {
                        panic(e)
                } else {
                        task.create_date = date
                }

                task.todo = strings.Join(splits[1:], " ")
        } else {
                task.todo = strings.Join(splits[0:], " ")
        }

        context_regexp, _ := regexp.Compile("@[[:word:]]+")
        contexts := context_regexp.FindAllStringSubmatch(text, -1)
        if len(contexts) != 0 {
                task.contexts = contexts[0]
        }

        project_regexp, _ := regexp.Compile("\\+[[:word:]]+")
        projects := project_regexp.FindAllStringSubmatch(text, -1)
        if len(projects) != 0 {
                task.projects = projects[0]
        }

        return task
}

func LoadTaskList (filename string) (TaskList) {

        var f, err = os.Open(filename)

        if err != nil {
                panic(err)
        }

        defer f.Close()

        var tasklist = TaskList{}

        scanner := bufio.NewScanner(f)

        for scanner.Scan() {
                text := scanner.Text()
                tasklist.Add(text)
        }

        if err := scanner.Err(); err != nil {
                panic(scanner.Err())
        }

        return tasklist
}

type By func(t1, t2 Task) bool

func (by By) Sort(tasks TaskList) {
        ts := &taskSorter{
                tasks: tasks,
                by:    by,
        }
        sort.Sort(ts)
}

type taskSorter struct {
        tasks TaskList
        by func(t1, t2 Task) bool
}

func (s *taskSorter) Len() int {
        return len(s.tasks)
}

func (s *taskSorter) Swap(i, j int) {
        s.tasks[i], s.tasks[j] = s.tasks[j], s.tasks[i]
}

func (s *taskSorter) Less(i, j int) bool {
        return s.by(s.tasks[i], s.tasks[j])
}

func (tasks TaskList) Len() int {
        return len(tasks)
}

func prioCmp(t1, t2 Task) bool {
        return t1.Priority() < t2.Priority()
}

func prioRevCmp(t1, t2 Task) bool {
        return t1.Priority() > t2.Priority()
}

func dateCmp(t1, t2 Task) bool {
        tm1 := t1.CreateDate().Unix()
        tm2 := t2.CreateDate().Unix()

        // if the dates equal, let's use priority
        if tm1 == tm2 {
                return prioCmp(t1, t2)
        } else {
                return tm1 > tm2
        }
}

func dateRevCmp(t1, t2 Task) bool {
        tm1 := t1.CreateDate().Unix()
        tm2 := t2.CreateDate().Unix()

        // if the dates equal, let's use priority
        if tm1 == tm2 {
                return prioCmp(t1, t2)
        } else {
                return tm1 < tm2
        }
}

func lenCmp(t1, t2 Task) bool {
        tl1 := len(t1.raw_todo)
        tl2 := len(t2.raw_todo)
        if tl1 == tl2 {
                return prioCmp(t1, t2)
        } else {
                return tl1 < tl2
        }
}

func lenRevCmp(t1, t2 Task) bool {
        tl1 := len(t1.raw_todo)
        tl2 := len(t2.raw_todo)
        if tl1 == tl2 {
                return prioCmp(t1, t2)
        } else {
                return tl1 > tl2
        }
}

func idCmp(t1, t2 Task) bool {
        return t1.Id() < t2.Id()
}

func (tasks TaskList) Sort(by string) {
        switch by {
        default:
        case "prio":
                By(prioCmp).Sort(tasks)
        case "prio-rev":
                By(prioRevCmp).Sort(tasks)
        case "date":
                By(dateCmp).Sort(tasks)
        case "date-rev":
                By(dateRevCmp).Sort(tasks)
        case "len":
                By(lenCmp).Sort(tasks)
        case "len-rev":
                By(lenRevCmp).Sort(tasks)
        case "id":
                By(idCmp).Sort(tasks)
        }
}

func (tasks TaskList) Save(filename string) {
        tasks.Sort("id")

        f, err := os.Create(filename)
        if err != nil {
                panic(err)
        }

        defer f.Close()

        for _, task := range tasks {
                f.WriteString(task.RawText() + "\n")
        }
        f.Sync()
}

func (tasks *TaskList) Add(todo string) {
        task := CreateTask(tasks.Len(), todo)
        *tasks = append(*tasks, task)
}

func (tasks TaskList) Done(id int, finish_date bool) error {
        if id > tasks.Len() || id < 0 {
                return fmt.Errorf("Error: id is %v", id)
        }

        tasks[id].finished = true
        if finish_date {
                t := time.LocalTime()
                tasks[id].raw_todo = "x " + t.Format("2006-01-02") + " " +
                                        tasks[id].raw_todo
        } else {
                tasks[id].raw_todo = "x " + tasks[id].raw_todo
        }

        return nil
}

func (task Task) Id() int {
        return task.id
}

func (task Task) Text() string {
        return task.todo
}

func (task Task) RawText() string {
        return task.raw_todo
}

func (task Task) Priority() byte {
        // if priority is not from [A-Z], let it be 94 (^)
        if task.priority < 65 || task.priority > 90 {
                return 94 // you know, ^
        } else {
                return task.priority
        }
}

func (task Task) Contexts() []string {
        return task.contexts
}

func (task Task) Projects() []string {
        return task.projects
}

func (task Task) CreateDate() time.Time {
        return task.create_date
}

func (task Task) Finished() bool {
        return task.finished
}

func (task *Task) SetIdPaddingBy(tasklist TaskList) {
        l := tasklist.Len()

        if l >= 10000 {
                task.id_padding = 5
        } else if l >= 1000 {
                task.id_padding = 4
        } else if l >= 100 {
                task.id_padding = 3
        } else if l >= 10 {
                task.id_padding = 2
        } else {
                task.id_padding = 1
        }
}

func (task Task) IdPadding() int {
        return task.id_padding
}

func (task Task) PrettyPrint(pretty string) string {
        rp := regexp.MustCompile("(%[a-zA-Z])")
        out := rp.ReplaceAllStringFunc(pretty, func(s string) string {

                switch s{
                case "%i":
                        str := fmt.Sprintf("%%0%dd", task.IdPadding())
                        return fmt.Sprintf(str, task.Id())
                case "%t":
                        return task.Text()
                case "%T":
                        return task.RawText()
                case "%p":
                        return string(task.Priority())
                default:
                        return s
                }
        })
        return out
}
<commit_msg>Now instead of LocalTime<commit_after>package todotxt

import (
        "time"
        "os"
        "bufio"
        "strings"
        "regexp"
        "sort"
        "unicode"
        "fmt"
)

type Task struct {
        id int
        todo string
        priority byte
        create_date time.Time
        contexts []string
        projects []string
        raw_todo string
        finished bool
        id_padding int
}

type TaskList []Task

func CreateTask(id int, text string) (Task) {
        var task = Task{}
        task.id = id
        task.raw_todo = text

        splits := strings.Split(text, " ")

        if text[0] == 'x' &&
           text[1] == ' ' &&
           !unicode.IsSpace(rune(text[2])) {
                task.finished = true
                splits = splits[1:]
        }

        head := splits[0]

        if (len(head) == 3) &&
           (head[0] == '(') &&
           (head[2] == ')') &&
           (head[1] >= 65 && head[1] <= 90) { // checking if it's in range [A-Z]
                task.priority = head[1]
                splits = splits[1:]
        }

        date_regexp := "([\\d]{4})-([\\d]{2})-([\\d]{2})"
        if match, _ := regexp.MatchString(date_regexp, splits[0]); match {
                if date, e := time.Parse("2006-01-02", splits[0]); e != nil {
                        panic(e)
                } else {
                        task.create_date = date
                }

                task.todo = strings.Join(splits[1:], " ")
        } else {
                task.todo = strings.Join(splits[0:], " ")
        }

        context_regexp, _ := regexp.Compile("@[[:word:]]+")
        contexts := context_regexp.FindAllStringSubmatch(text, -1)
        if len(contexts) != 0 {
                task.contexts = contexts[0]
        }

        project_regexp, _ := regexp.Compile("\\+[[:word:]]+")
        projects := project_regexp.FindAllStringSubmatch(text, -1)
        if len(projects) != 0 {
                task.projects = projects[0]
        }

        return task
}

func LoadTaskList (filename string) (TaskList) {

        var f, err = os.Open(filename)

        if err != nil {
                panic(err)
        }

        defer f.Close()

        var tasklist = TaskList{}

        scanner := bufio.NewScanner(f)

        for scanner.Scan() {
                text := scanner.Text()
                tasklist.Add(text)
        }

        if err := scanner.Err(); err != nil {
                panic(scanner.Err())
        }

        return tasklist
}

type By func(t1, t2 Task) bool

func (by By) Sort(tasks TaskList) {
        ts := &taskSorter{
                tasks: tasks,
                by:    by,
        }
        sort.Sort(ts)
}

type taskSorter struct {
        tasks TaskList
        by func(t1, t2 Task) bool
}

func (s *taskSorter) Len() int {
        return len(s.tasks)
}

func (s *taskSorter) Swap(i, j int) {
        s.tasks[i], s.tasks[j] = s.tasks[j], s.tasks[i]
}

func (s *taskSorter) Less(i, j int) bool {
        return s.by(s.tasks[i], s.tasks[j])
}

func (tasks TaskList) Len() int {
        return len(tasks)
}

func prioCmp(t1, t2 Task) bool {
        return t1.Priority() < t2.Priority()
}

func prioRevCmp(t1, t2 Task) bool {
        return t1.Priority() > t2.Priority()
}

func dateCmp(t1, t2 Task) bool {
        tm1 := t1.CreateDate().Unix()
        tm2 := t2.CreateDate().Unix()

        // if the dates equal, let's use priority
        if tm1 == tm2 {
                return prioCmp(t1, t2)
        } else {
                return tm1 > tm2
        }
}

func dateRevCmp(t1, t2 Task) bool {
        tm1 := t1.CreateDate().Unix()
        tm2 := t2.CreateDate().Unix()

        // if the dates equal, let's use priority
        if tm1 == tm2 {
                return prioCmp(t1, t2)
        } else {
                return tm1 < tm2
        }
}

func lenCmp(t1, t2 Task) bool {
        tl1 := len(t1.raw_todo)
        tl2 := len(t2.raw_todo)
        if tl1 == tl2 {
                return prioCmp(t1, t2)
        } else {
                return tl1 < tl2
        }
}

func lenRevCmp(t1, t2 Task) bool {
        tl1 := len(t1.raw_todo)
        tl2 := len(t2.raw_todo)
        if tl1 == tl2 {
                return prioCmp(t1, t2)
        } else {
                return tl1 > tl2
        }
}

func idCmp(t1, t2 Task) bool {
        return t1.Id() < t2.Id()
}

func (tasks TaskList) Sort(by string) {
        switch by {
        default:
        case "prio":
                By(prioCmp).Sort(tasks)
        case "prio-rev":
                By(prioRevCmp).Sort(tasks)
        case "date":
                By(dateCmp).Sort(tasks)
        case "date-rev":
                By(dateRevCmp).Sort(tasks)
        case "len":
                By(lenCmp).Sort(tasks)
        case "len-rev":
                By(lenRevCmp).Sort(tasks)
        case "id":
                By(idCmp).Sort(tasks)
        }
}

func (tasks TaskList) Save(filename string) {
        tasks.Sort("id")

        f, err := os.Create(filename)
        if err != nil {
                panic(err)
        }

        defer f.Close()

        for _, task := range tasks {
                f.WriteString(task.RawText() + "\n")
        }
        f.Sync()
}

func (tasks *TaskList) Add(todo string) {
        task := CreateTask(tasks.Len(), todo)
        *tasks = append(*tasks, task)
}

func (tasks TaskList) Done(id int, finish_date bool) error {
        if id > tasks.Len() || id < 0 {
                return fmt.Errorf("Error: id is %v", id)
        }

        tasks[id].finished = true
        if finish_date {
                t := time.Now()
                tasks[id].raw_todo = "x " + t.Format("2006-01-02") + " " +
                                        tasks[id].raw_todo
        } else {
                tasks[id].raw_todo = "x " + tasks[id].raw_todo
        }

        return nil
}

func (task Task) Id() int {
        return task.id
}

func (task Task) Text() string {
        return task.todo
}

func (task Task) RawText() string {
        return task.raw_todo
}

func (task Task) Priority() byte {
        // if priority is not from [A-Z], let it be 94 (^)
        if task.priority < 65 || task.priority > 90 {
                return 94 // you know, ^
        } else {
                return task.priority
        }
}

func (task Task) Contexts() []string {
        return task.contexts
}

func (task Task) Projects() []string {
        return task.projects
}

func (task Task) CreateDate() time.Time {
        return task.create_date
}

func (task Task) Finished() bool {
        return task.finished
}

func (task *Task) SetIdPaddingBy(tasklist TaskList) {
        l := tasklist.Len()

        if l >= 10000 {
                task.id_padding = 5
        } else if l >= 1000 {
                task.id_padding = 4
        } else if l >= 100 {
                task.id_padding = 3
        } else if l >= 10 {
                task.id_padding = 2
        } else {
                task.id_padding = 1
        }
}

func (task Task) IdPadding() int {
        return task.id_padding
}

func (task Task) PrettyPrint(pretty string) string {
        rp := regexp.MustCompile("(%[a-zA-Z])")
        out := rp.ReplaceAllStringFunc(pretty, func(s string) string {

                switch s{
                case "%i":
                        str := fmt.Sprintf("%%0%dd", task.IdPadding())
                        return fmt.Sprintf(str, task.Id())
                case "%t":
                        return task.Text()
                case "%T":
                        return task.RawText()
                case "%p":
                        return string(task.Priority())
                default:
                        return s
                }
        })
        return out
}
<|endoftext|>


<commit_before>package client

import (
	"errors"
	"fmt"
	"time"

	"sourcegraph.com/sourcegraph/api_router"
	"sourcegraph.com/sourcegraph/srcgraph/db_common"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
)

type BuildsService interface {
	Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error)
	ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error)
}

type buildsService struct {
	client *Client
}

var _ BuildsService = &buildsService{}

type BuildSpec struct {
	Repo RepositorySpec
	BID  int64
}

// A Build represents a scheduled, completed, or failed repository analysis and
// import job.
type Build struct {
	BID       int64
	Repo      repo.RID
	CommitID  string             `db:"commit_id"`
	CreatedAt time.Time          `db:"created_at"`
	StartedAt db_common.NullTime `db:"started_at"`
	EndedAt   db_common.NullTime `db:"ended_at"`
	Success   bool
	Failure   bool
}

var ErrBuildNotFound = errors.New("build not found")

type BuildGetOptions struct{}

func (s *buildsService) Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error) {
	url, err := s.client.url(api_router.RepositoryBuild, map[string]string{"RepoURI": build.Repo.URI, "BID": fmt.Sprintf("%d", build.BID)}, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var build_ *Build
	resp, err := s.client.Do(req, &build_)
	if err != nil {
		return nil, resp, err
	}

	return build_, nil, nil
}

type BuildListByRepositoryOptions struct{ ListOptions }

func (s *buildsService) ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error) {
	url, err := s.client.url(api_router.RepositoryBuilds, map[string]string{"RepoURI": repo.URI}, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var builds []*Build
	resp, err := s.client.Do(req, &builds)
	if err != nil {
		return nil, resp, err
	}

	return builds, resp, nil
}

type MockBuildsService struct {
	Get_              func(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error)
	ListByRepository_ func(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error)
}

var _ BuildsService = MockBuildsService{}

func (s MockBuildsService) Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error) {
	if s.Get_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.Get_(build, opt)
}
func (s MockBuildsService) ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error) {
	if s.ListByRepository_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.ListByRepository_(repo, opt)
}
<commit_msg>show repo commit chooser<commit_after>package client

import (
	"errors"
	"fmt"
	"time"

	"sourcegraph.com/sourcegraph/api_router"
	"sourcegraph.com/sourcegraph/srcgraph/db_common"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
)

type BuildsService interface {
	Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error)
	ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error)
}

type buildsService struct {
	client *Client
}

var _ BuildsService = &buildsService{}

type BuildSpec struct {
	Repo RepositorySpec
	BID  int64
}

// A Build represents a scheduled, completed, or failed repository analysis and
// import job.
type Build struct {
	BID       int64
	Repo      repo.RID
	CommitID  string             `db:"commit_id"`
	CreatedAt time.Time          `db:"created_at"`
	StartedAt db_common.NullTime `db:"started_at"`
	EndedAt   db_common.NullTime `db:"ended_at"`
	Success   bool
	Failure   bool
}

var ErrBuildNotFound = errors.New("build not found")

type BuildGetOptions struct{}

func (s *buildsService) Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error) {
	url, err := s.client.url(api_router.RepositoryBuild, map[string]string{"RepoURI": build.Repo.URI, "BID": fmt.Sprintf("%d", build.BID)}, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var build_ *Build
	resp, err := s.client.Do(req, &build_)
	if err != nil {
		return nil, resp, err
	}

	return build_, nil, nil
}

type BuildListByRepositoryOptions struct {
	Ended     bool   `url:",omitempty"`
	Succeeded bool   `url:",omitempty"`
	Rev       string `url:",omitempty"`

	ListOptions
}

func (s *buildsService) ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error) {
	url, err := s.client.url(api_router.RepositoryBuilds, map[string]string{"RepoURI": repo.URI}, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var builds []*Build
	resp, err := s.client.Do(req, &builds)
	if err != nil {
		return nil, resp, err
	}

	return builds, resp, nil
}

type MockBuildsService struct {
	Get_              func(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error)
	ListByRepository_ func(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error)
}

var _ BuildsService = MockBuildsService{}

func (s MockBuildsService) Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error) {
	if s.Get_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.Get_(build, opt)
}
func (s MockBuildsService) ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error) {
	if s.ListByRepository_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.ListByRepository_(repo, opt)
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
)

var (
	ErrTimeout          = context.DeadlineExceeded
	ErrCanceled         = context.Canceled
	ErrUnavailable      = errors.New("client: no available etcd endpoints")
	ErrNoLeader         = errors.New("client: no leader")
	ErrNoEndpoints      = errors.New("no endpoints available")
	ErrTooManyRedirects = errors.New("too many redirects")

	ErrKeyNoExist = errors.New("client: key does not exist")
	ErrKeyExists  = errors.New("client: key already exists")

	DefaultRequestTimeout = 5 * time.Second
	DefaultMaxRedirects   = 10
)

type Config struct {
	Endpoints []string
	Transport CancelableTransport
}

// CancelableTransport mimics http.Transport to provide an interface which can be
// substituted for testing (since the RoundTripper interface alone does not
// require the CancelRequest method)
type CancelableTransport interface {
	http.RoundTripper
	CancelRequest(req *http.Request)
}

type Client interface {
	Sync(context.Context) error
	Endpoints() []string

	httpClient
}

func New(cfg Config) (Client, error) {
	c := &httpClusterClient{clientFactory: newHTTPClientFactory(cfg.Transport)}
	if err := c.reset(cfg.Endpoints); err != nil {
		return nil, err
	}
	return c, nil
}

type httpClient interface {
	Do(context.Context, httpAction) (*http.Response, []byte, error)
}

func newHTTPClientFactory(tr CancelableTransport) httpClientFactory {
	return func(ep url.URL) httpClient {
		return &redirectFollowingHTTPClient{
			max: DefaultMaxRedirects,
			client: &simpleHTTPClient{
				transport: tr,
				endpoint:  ep,
			},
		}
	}
}

type httpClientFactory func(url.URL) httpClient

type httpAction interface {
	HTTPRequest(url.URL) *http.Request
}

type httpClusterClient struct {
	clientFactory httpClientFactory
	endpoints     []url.URL
	sync.RWMutex
}

func (c *httpClusterClient) reset(eps []string) error {
	if len(eps) == 0 {
		return ErrNoEndpoints
	}

	neps := make([]url.URL, len(eps))
	for i, ep := range eps {
		u, err := url.Parse(ep)
		if err != nil {
			return err
		}
		neps[i] = *u
	}

	c.endpoints = neps

	return nil
}

func (c *httpClusterClient) Do(ctx context.Context, act httpAction) (resp *http.Response, body []byte, err error) {
	c.RLock()
	leps := len(c.endpoints)
	eps := make([]url.URL, leps)
	n := copy(eps, c.endpoints)
	c.RUnlock()

	if leps == 0 {
		err = ErrNoEndpoints
		return
	}

	if leps != n {
		err = errors.New("unable to pick endpoint: copy failed")
		return
	}

	for _, ep := range eps {
		hc := c.clientFactory(ep)
		resp, body, err = hc.Do(ctx, act)
		if err != nil {
			if err == ErrTimeout || err == ErrCanceled {
				return nil, nil, err
			}
			continue
		}
		if resp.StatusCode/100 == 5 {
			continue
		}
		break
	}

	return
}

func (c *httpClusterClient) Endpoints() []string {
	c.RLock()
	defer c.RUnlock()

	eps := make([]string, len(c.endpoints))
	for i, ep := range c.endpoints {
		eps[i] = ep.String()
	}

	return eps
}

func (c *httpClusterClient) Sync(ctx context.Context) error {
	c.Lock()
	defer c.Unlock()

	mAPI := NewMembersAPI(c)
	ms, err := mAPI.List(ctx)
	if err != nil {
		return err
	}

	eps := make([]string, 0)
	for _, m := range ms {
		eps = append(eps, m.ClientURLs...)
	}

	return c.reset(eps)
}

type roundTripResponse struct {
	resp *http.Response
	err  error
}

type simpleHTTPClient struct {
	transport CancelableTransport
	endpoint  url.URL
}

func (c *simpleHTTPClient) Do(ctx context.Context, act httpAction) (*http.Response, []byte, error) {
	req := act.HTTPRequest(c.endpoint)

	rtchan := make(chan roundTripResponse, 1)
	go func() {
		resp, err := c.transport.RoundTrip(req)
		rtchan <- roundTripResponse{resp: resp, err: err}
		close(rtchan)
	}()

	var resp *http.Response
	var err error

	select {
	case rtresp := <-rtchan:
		resp, err = rtresp.resp, rtresp.err
	case <-ctx.Done():
		c.transport.CancelRequest(req)
		// wait for request to actually exit before continuing
		<-rtchan
		err = ctx.Err()
	}

	// always check for resp nil-ness to deal with possible
	// race conditions between channels above
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	if err != nil {
		return nil, nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	return resp, body, err
}

type redirectFollowingHTTPClient struct {
	client httpClient
	max    int
}

func (r *redirectFollowingHTTPClient) Do(ctx context.Context, act httpAction) (*http.Response, []byte, error) {
	for i := 0; i <= r.max; i++ {
		resp, body, err := r.client.Do(ctx, act)
		if err != nil {
			return nil, nil, err
		}
		if resp.StatusCode/100 == 3 {
			hdr := resp.Header.Get("Location")
			if hdr == "" {
				return nil, nil, fmt.Errorf("Location header not set")
			}
			loc, err := url.Parse(hdr)
			if err != nil {
				return nil, nil, fmt.Errorf("Location header not valid URL: %s", hdr)
			}
			act = &redirectedHTTPAction{
				action:   act,
				location: *loc,
			}
			continue
		}
		return resp, body, nil
	}
	return nil, nil, ErrTooManyRedirects
}

type redirectedHTTPAction struct {
	action   httpAction
	location url.URL
}

func (r *redirectedHTTPAction) HTTPRequest(ep url.URL) *http.Request {
	orig := r.action.HTTPRequest(ep)
	orig.URL = &r.location
	return orig
}
<commit_msg>client: document Config<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
)

var (
	ErrTimeout          = context.DeadlineExceeded
	ErrCanceled         = context.Canceled
	ErrUnavailable      = errors.New("client: no available etcd endpoints")
	ErrNoLeader         = errors.New("client: no leader")
	ErrNoEndpoints      = errors.New("no endpoints available")
	ErrTooManyRedirects = errors.New("too many redirects")

	ErrKeyNoExist = errors.New("client: key does not exist")
	ErrKeyExists  = errors.New("client: key already exists")

	DefaultRequestTimeout = 5 * time.Second
	DefaultMaxRedirects   = 10
)

type Config struct {
	// Endpoints defines a set of URLs (schemes, hosts and ports only)
	// that can be used to communicate with a logical etcd cluster. For
	// example, a three-node cluster could be provided like so:
	//
	// 	Endpoints: []string{
	//		"http://node1.example.com:4001",
	//		"http://node2.example.com:2379",
	//		"http://node3.example.com:4001",
	//	}
	//
	// If multiple endpoints are provided, the Client will attempt to
	// use them all in the event that one or more of them are unusable.
	//
	// If Client.Sync is ever called, the Client may cache an alternate
	// set of endpoints to continue operation.
	Endpoints []string

	// Transport is used by the Client to drive HTTP requests. If not
	// provided, net/http.DefaultTransport will be used.
	Transport CancelableTransport
}

// CancelableTransport mimics http.Transport to provide an interface which can be
// substituted for testing (since the RoundTripper interface alone does not
// require the CancelRequest method)
type CancelableTransport interface {
	http.RoundTripper
	CancelRequest(req *http.Request)
}

type Client interface {
	Sync(context.Context) error
	Endpoints() []string

	httpClient
}

func New(cfg Config) (Client, error) {
	c := &httpClusterClient{clientFactory: newHTTPClientFactory(cfg.Transport)}
	if err := c.reset(cfg.Endpoints); err != nil {
		return nil, err
	}
	return c, nil
}

type httpClient interface {
	Do(context.Context, httpAction) (*http.Response, []byte, error)
}

func newHTTPClientFactory(tr CancelableTransport) httpClientFactory {
	return func(ep url.URL) httpClient {
		return &redirectFollowingHTTPClient{
			max: DefaultMaxRedirects,
			client: &simpleHTTPClient{
				transport: tr,
				endpoint:  ep,
			},
		}
	}
}

type httpClientFactory func(url.URL) httpClient

type httpAction interface {
	HTTPRequest(url.URL) *http.Request
}

type httpClusterClient struct {
	clientFactory httpClientFactory
	endpoints     []url.URL
	sync.RWMutex
}

func (c *httpClusterClient) reset(eps []string) error {
	if len(eps) == 0 {
		return ErrNoEndpoints
	}

	neps := make([]url.URL, len(eps))
	for i, ep := range eps {
		u, err := url.Parse(ep)
		if err != nil {
			return err
		}
		neps[i] = *u
	}

	c.endpoints = neps

	return nil
}

func (c *httpClusterClient) Do(ctx context.Context, act httpAction) (resp *http.Response, body []byte, err error) {
	c.RLock()
	leps := len(c.endpoints)
	eps := make([]url.URL, leps)
	n := copy(eps, c.endpoints)
	c.RUnlock()

	if leps == 0 {
		err = ErrNoEndpoints
		return
	}

	if leps != n {
		err = errors.New("unable to pick endpoint: copy failed")
		return
	}

	for _, ep := range eps {
		hc := c.clientFactory(ep)
		resp, body, err = hc.Do(ctx, act)
		if err != nil {
			if err == ErrTimeout || err == ErrCanceled {
				return nil, nil, err
			}
			continue
		}
		if resp.StatusCode/100 == 5 {
			continue
		}
		break
	}

	return
}

func (c *httpClusterClient) Endpoints() []string {
	c.RLock()
	defer c.RUnlock()

	eps := make([]string, len(c.endpoints))
	for i, ep := range c.endpoints {
		eps[i] = ep.String()
	}

	return eps
}

func (c *httpClusterClient) Sync(ctx context.Context) error {
	c.Lock()
	defer c.Unlock()

	mAPI := NewMembersAPI(c)
	ms, err := mAPI.List(ctx)
	if err != nil {
		return err
	}

	eps := make([]string, 0)
	for _, m := range ms {
		eps = append(eps, m.ClientURLs...)
	}

	return c.reset(eps)
}

type roundTripResponse struct {
	resp *http.Response
	err  error
}

type simpleHTTPClient struct {
	transport CancelableTransport
	endpoint  url.URL
}

func (c *simpleHTTPClient) Do(ctx context.Context, act httpAction) (*http.Response, []byte, error) {
	req := act.HTTPRequest(c.endpoint)

	rtchan := make(chan roundTripResponse, 1)
	go func() {
		resp, err := c.transport.RoundTrip(req)
		rtchan <- roundTripResponse{resp: resp, err: err}
		close(rtchan)
	}()

	var resp *http.Response
	var err error

	select {
	case rtresp := <-rtchan:
		resp, err = rtresp.resp, rtresp.err
	case <-ctx.Done():
		c.transport.CancelRequest(req)
		// wait for request to actually exit before continuing
		<-rtchan
		err = ctx.Err()
	}

	// always check for resp nil-ness to deal with possible
	// race conditions between channels above
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	if err != nil {
		return nil, nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	return resp, body, err
}

type redirectFollowingHTTPClient struct {
	client httpClient
	max    int
}

func (r *redirectFollowingHTTPClient) Do(ctx context.Context, act httpAction) (*http.Response, []byte, error) {
	for i := 0; i <= r.max; i++ {
		resp, body, err := r.client.Do(ctx, act)
		if err != nil {
			return nil, nil, err
		}
		if resp.StatusCode/100 == 3 {
			hdr := resp.Header.Get("Location")
			if hdr == "" {
				return nil, nil, fmt.Errorf("Location header not set")
			}
			loc, err := url.Parse(hdr)
			if err != nil {
				return nil, nil, fmt.Errorf("Location header not valid URL: %s", hdr)
			}
			act = &redirectedHTTPAction{
				action:   act,
				location: *loc,
			}
			continue
		}
		return resp, body, nil
	}
	return nil, nil, ErrTooManyRedirects
}

type redirectedHTTPAction struct {
	action   httpAction
	location url.URL
}

func (r *redirectedHTTPAction) HTTPRequest(ep url.URL) *http.Request {
	orig := r.action.HTTPRequest(ep)
	orig.URL = &r.location
	return orig
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"net/http"
	"net/url"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/tedsuo/rata"

	"github.com/concourse/baggageclaim"
	"github.com/concourse/baggageclaim/api"
	"github.com/concourse/baggageclaim/volume"
	"github.com/concourse/retryhttp"
)

type Client interface {
	baggageclaim.Client
}

type client struct {
	requestGenerator *rata.RequestGenerator

	retryBackOffFactory retryhttp.BackOffFactory
	nestedRoundTripper  http.RoundTripper
}

func New(apiURL string) Client {
	return &client{
		requestGenerator: rata.NewRequestGenerator(apiURL, baggageclaim.Routes),

		retryBackOffFactory: retryhttp.NewExponentialBackOffFactory(60 * time.Minute),

		nestedRoundTripper: &http.Transport{
			DisableKeepAlives: true,
		},
	}
}

func (c *client) httpClient(logger lager.Logger) *http.Client {
	return &http.Client{
		Transport: &retryhttp.RetryRoundTripper{
			Logger:         logger.Session("retry-round-tripper"),
			BackOffFactory: c.retryBackOffFactory,
			RoundTripper:   c.nestedRoundTripper,
		},
	}
}

func (c *client) CreateVolume(logger lager.Logger, volumeSpec baggageclaim.VolumeSpec) (baggageclaim.Volume, error) {
	strategy := volumeSpec.Strategy
	if strategy == nil {
		strategy = baggageclaim.EmptyStrategy{}
	}

	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.VolumeRequest{
		Strategy:     strategy.Encode(),
		TTLInSeconds: uint(math.Ceil(volumeSpec.TTL.Seconds())),
		Properties:   volumeSpec.Properties,
		Privileged:   volumeSpec.Privileged,
	})

	request, _ := c.requestGenerator.CreateRequest(baggageclaim.CreateVolume, nil, buffer)
	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != 201 {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeResponse baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumeResponse)
	if err != nil {
		return nil, err
	}

	v, initialHeartbeatSuccess := c.newVolume(logger, volumeResponse)
	if !initialHeartbeatSuccess {
		return nil, volume.ErrVolumeDoesNotExist
	}
	return v, nil
}

func (c *client) ListVolumes(logger lager.Logger, properties baggageclaim.VolumeProperties) (baggageclaim.Volumes, error) {
	if properties == nil {
		properties = baggageclaim.VolumeProperties{}
	}

	request, err := c.requestGenerator.CreateRequest(baggageclaim.ListVolumes, nil, nil)
	if err != nil {
		return nil, err
	}

	queryString := request.URL.Query()
	for key, val := range properties {
		queryString.Add(key, val)
	}

	request.URL.RawQuery = queryString.Encode()

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != 200 {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumesResponse []baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumesResponse)
	if err != nil {
		return nil, err
	}

	var volumes baggageclaim.Volumes
	for _, vr := range volumesResponse {
		v, initialHeartbeatSuccess := c.newVolume(logger, vr)
		if initialHeartbeatSuccess {
			volumes = append(volumes, v)
		}
	}

	return volumes, nil
}

func (c *client) LookupVolume(logger lager.Logger, handle string) (baggageclaim.Volume, bool, error) {

	volumeResponse, found, err := c.getVolumeResponse(logger, handle)
	if err != nil {
		return nil, false, err
	}
	if !found {
		return nil, found, nil
	}

	v, initialHeartbeatSuccess := c.newVolume(logger, volumeResponse)
	if !initialHeartbeatSuccess {
		return nil, false, nil
	}
	return v, true, nil
}

func (c *client) newVolume(logger lager.Logger, apiVolume baggageclaim.VolumeResponse) (baggageclaim.Volume, bool) {
	volume := &clientVolume{
		logger: logger,

		handle: apiVolume.Handle,
		path:   apiVolume.Path,

		bcClient:     c,
		heartbeating: new(sync.WaitGroup),
		release:      make(chan *time.Duration, 1),
	}

	initialHeartbeatSuccess := volume.startHeartbeating(logger, time.Duration(apiVolume.TTLInSeconds)*time.Second)

	return volume, initialHeartbeatSuccess
}

func (c *client) streamIn(logger lager.Logger, destHandle string, path string, tarContent io.Reader) error {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamIn, rata.Params{
		"handle": destHandle,
	}, tarContent)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()
	if response.StatusCode == http.StatusNoContent {
		return nil
	}
	return getError(response)
}

func (c *client) streamOut(logger lager.Logger, srcHandle string, path string) (io.ReadCloser, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamOut, rata.Params{
		"handle": srcHandle,
	}, nil)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return nil, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode != http.StatusOK {
		return nil, getError(response)
	}

	return response.Body, nil
}

func getError(response *http.Response) error {
	var errorResponse *api.ErrorResponse
	err := json.NewDecoder(response.Body).Decode(&errorResponse)
	if err != nil {
		return err
	}

	if errorResponse.Message == api.ErrStreamOutNotFound.Error() {
		return baggageclaim.ErrFileNotFound
	}

	if response.StatusCode == 404 {
		return baggageclaim.ErrVolumeNotFound
	}

	return errors.New(errorResponse.Message)
}

func (c *client) getVolumeResponse(logger lager.Logger, handle string) (baggageclaim.VolumeResponse, bool, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolume, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeResponse{}, false, nil
		}

		return baggageclaim.VolumeResponse{}, false, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeResponse{}, false, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeResponse baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumeResponse)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	return volumeResponse, true, nil
}

func (c *client) getVolumeStatsResponse(logger lager.Logger, handle string) (baggageclaim.VolumeStatsResponse, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolumeStats, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeStatsResponse{}, nil
		}
		return baggageclaim.VolumeStatsResponse{}, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeStatsResponse{}, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeStatsResponse baggageclaim.VolumeStatsResponse
	err = json.NewDecoder(response.Body).Decode(&volumeStatsResponse)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	return volumeStatsResponse, nil
}

func (c *client) setTTL(logger lager.Logger, handle string, ttl time.Duration) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.TTLRequest{
		Value: uint(math.Ceil(ttl.Seconds())),
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetTTL, rata.Params{
		"handle": handle,
	}, buffer)
	if err != nil {
		return err
	}

	request.Header.Add("Content-type", "application/json")

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) destroy(logger lager.Logger, handle string) error {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.DestroyVolume, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return err
	}

	request.Header.Add("Content-type", "application/json")

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) setProperty(logger lager.Logger, handle string, propertyName string, propertyValue string) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.PropertyRequest{
		Value: propertyValue,
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetProperty, rata.Params{
		"handle":   handle,
		"property": propertyName,
	}, buffer)
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}
<commit_msg>fix retryhttp.RoundTrippper to take in Retryer<commit_after>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"net/http"
	"net/url"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/tedsuo/rata"

	"github.com/concourse/baggageclaim"
	"github.com/concourse/baggageclaim/api"
	"github.com/concourse/baggageclaim/volume"
	"github.com/concourse/retryhttp"
)

type Client interface {
	baggageclaim.Client
}

type client struct {
	requestGenerator *rata.RequestGenerator

	retryBackOffFactory retryhttp.BackOffFactory
	nestedRoundTripper  http.RoundTripper
}

func New(apiURL string) Client {
	return &client{
		requestGenerator: rata.NewRequestGenerator(apiURL, baggageclaim.Routes),

		retryBackOffFactory: retryhttp.NewExponentialBackOffFactory(60 * time.Minute),

		nestedRoundTripper: &http.Transport{
			DisableKeepAlives: true,
		},
	}
}

func (c *client) httpClient(logger lager.Logger) *http.Client {
	return &http.Client{
		Transport: &retryhttp.RetryRoundTripper{
			Logger:         logger.Session("retry-round-tripper"),
			BackOffFactory: c.retryBackOffFactory,
			RoundTripper:   c.nestedRoundTripper,
			Retryer:        &retryhttp.DefaultRetryer{},
		},
	}
}

func (c *client) CreateVolume(logger lager.Logger, volumeSpec baggageclaim.VolumeSpec) (baggageclaim.Volume, error) {
	strategy := volumeSpec.Strategy
	if strategy == nil {
		strategy = baggageclaim.EmptyStrategy{}
	}

	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.VolumeRequest{
		Strategy:     strategy.Encode(),
		TTLInSeconds: uint(math.Ceil(volumeSpec.TTL.Seconds())),
		Properties:   volumeSpec.Properties,
		Privileged:   volumeSpec.Privileged,
	})

	request, _ := c.requestGenerator.CreateRequest(baggageclaim.CreateVolume, nil, buffer)
	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != 201 {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeResponse baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumeResponse)
	if err != nil {
		return nil, err
	}

	v, initialHeartbeatSuccess := c.newVolume(logger, volumeResponse)
	if !initialHeartbeatSuccess {
		return nil, volume.ErrVolumeDoesNotExist
	}
	return v, nil
}

func (c *client) ListVolumes(logger lager.Logger, properties baggageclaim.VolumeProperties) (baggageclaim.Volumes, error) {
	if properties == nil {
		properties = baggageclaim.VolumeProperties{}
	}

	request, err := c.requestGenerator.CreateRequest(baggageclaim.ListVolumes, nil, nil)
	if err != nil {
		return nil, err
	}

	queryString := request.URL.Query()
	for key, val := range properties {
		queryString.Add(key, val)
	}

	request.URL.RawQuery = queryString.Encode()

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != 200 {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumesResponse []baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumesResponse)
	if err != nil {
		return nil, err
	}

	var volumes baggageclaim.Volumes
	for _, vr := range volumesResponse {
		v, initialHeartbeatSuccess := c.newVolume(logger, vr)
		if initialHeartbeatSuccess {
			volumes = append(volumes, v)
		}
	}

	return volumes, nil
}

func (c *client) LookupVolume(logger lager.Logger, handle string) (baggageclaim.Volume, bool, error) {

	volumeResponse, found, err := c.getVolumeResponse(logger, handle)
	if err != nil {
		return nil, false, err
	}
	if !found {
		return nil, found, nil
	}

	v, initialHeartbeatSuccess := c.newVolume(logger, volumeResponse)
	if !initialHeartbeatSuccess {
		return nil, false, nil
	}
	return v, true, nil
}

func (c *client) newVolume(logger lager.Logger, apiVolume baggageclaim.VolumeResponse) (baggageclaim.Volume, bool) {
	volume := &clientVolume{
		logger: logger,

		handle: apiVolume.Handle,
		path:   apiVolume.Path,

		bcClient:     c,
		heartbeating: new(sync.WaitGroup),
		release:      make(chan *time.Duration, 1),
	}

	initialHeartbeatSuccess := volume.startHeartbeating(logger, time.Duration(apiVolume.TTLInSeconds)*time.Second)

	return volume, initialHeartbeatSuccess
}

func (c *client) streamIn(logger lager.Logger, destHandle string, path string, tarContent io.Reader) error {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamIn, rata.Params{
		"handle": destHandle,
	}, tarContent)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()
	if response.StatusCode == http.StatusNoContent {
		return nil
	}
	return getError(response)
}

func (c *client) streamOut(logger lager.Logger, srcHandle string, path string) (io.ReadCloser, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamOut, rata.Params{
		"handle": srcHandle,
	}, nil)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return nil, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode != http.StatusOK {
		return nil, getError(response)
	}

	return response.Body, nil
}

func getError(response *http.Response) error {
	var errorResponse *api.ErrorResponse
	err := json.NewDecoder(response.Body).Decode(&errorResponse)
	if err != nil {
		return err
	}

	if errorResponse.Message == api.ErrStreamOutNotFound.Error() {
		return baggageclaim.ErrFileNotFound
	}

	if response.StatusCode == 404 {
		return baggageclaim.ErrVolumeNotFound
	}

	return errors.New(errorResponse.Message)
}

func (c *client) getVolumeResponse(logger lager.Logger, handle string) (baggageclaim.VolumeResponse, bool, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolume, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeResponse{}, false, nil
		}

		return baggageclaim.VolumeResponse{}, false, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeResponse{}, false, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeResponse baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumeResponse)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	return volumeResponse, true, nil
}

func (c *client) getVolumeStatsResponse(logger lager.Logger, handle string) (baggageclaim.VolumeStatsResponse, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolumeStats, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeStatsResponse{}, nil
		}
		return baggageclaim.VolumeStatsResponse{}, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeStatsResponse{}, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeStatsResponse baggageclaim.VolumeStatsResponse
	err = json.NewDecoder(response.Body).Decode(&volumeStatsResponse)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	return volumeStatsResponse, nil
}

func (c *client) setTTL(logger lager.Logger, handle string, ttl time.Duration) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.TTLRequest{
		Value: uint(math.Ceil(ttl.Seconds())),
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetTTL, rata.Params{
		"handle": handle,
	}, buffer)
	if err != nil {
		return err
	}

	request.Header.Add("Content-type", "application/json")

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) destroy(logger lager.Logger, handle string) error {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.DestroyVolume, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return err
	}

	request.Header.Add("Content-type", "application/json")

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) setProperty(logger lager.Logger, handle string, propertyName string, propertyValue string) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.PropertyRequest{
		Value: propertyValue,
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetProperty, rata.Params{
		"handle":   handle,
		"property": propertyName,
	}, buffer)
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}
<|endoftext|>


<commit_before>// Package client implements a low-level OAuth2 client to perform the various
// request/response flows against a OAuth2 authentication server.
package client

import (
	"net/http"

	"github.com/256dpi/oauth2"
)

// Config is used to configure a client.
type Config struct {
	BaseURI               string
	TokenEndpoint         string
	IntrospectionEndpoint string
	RevocationEndpoint    string
}

// Default will return a default configuration.
func Default(baseURI string) Config {
	return Config{
		BaseURI:               baseURI,
		TokenEndpoint:         "/oauth2/token",
		IntrospectionEndpoint: "/oauth2/introspect",
		RevocationEndpoint:    "/oauth2/revoke",
	}
}

// Client is a low-level OAuth2 client.
type Client struct {
	config Config
	client http.Client
}

// New will create and return a new client.
func New(config Config) *Client {
	return &Client{
		config: config,
	}
}

// Authenticate will send the provided token request and return the servers
// token response or an error if failed.
func (c *Client) Authenticate(trq oauth2.TokenRequest) (*oauth2.TokenResponse, error) {
	// prepare endpoint
	endpoint := c.config.BaseURI + c.config.TokenEndpoint

	// build request
	req, err := BuildTokenRequest(endpoint, trq)
	if err != nil {
		return nil, err
	}

	// perform request
	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	// check status
	if res.StatusCode != http.StatusOK {
		return nil, ParseRequestError(res)
	}

	// parse response
	trs, err := ParseTokenResponse(res)
	if err != nil {
		return nil, err
	}

	return trs, nil
}

// Introspect will send the provided introspection request and return the servers
// response of an error if failed.
func (c *Client) Introspect(irq oauth2.IntrospectionRequest) (*oauth2.IntrospectionResponse, error) {
	// prepare endpoint
	endpoint := c.config.BaseURI + c.config.IntrospectionEndpoint

	// build request
	req, err := BuildIntrospectionRequest(endpoint, irq)
	if err != nil {
		return nil, err
	}

	// perform request
	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	// check status
	if res.StatusCode != http.StatusOK {
		return nil, ParseRequestError(res)
	}

	// parse response
	irs, err := ParseIntrospectionResponse(res)
	if err != nil {
		return nil, err
	}

	return irs, nil
}

// Revoke will send the provided revocation request and return and error if it
// failed.
func (c *Client) Revoke(rrq oauth2.RevocationRequest) error {
	// prepare endpoint
	endpoint := c.config.BaseURI + c.config.RevocationEndpoint

	// build request
	req, err := BuildRevocationRequest(endpoint, rrq)
	if err != nil {
		return err
	}

	// perform request
	res, err := c.client.Do(req)
	if err != nil {
		return err
	}

	// check status
	if res.StatusCode != http.StatusOK {
		return ParseRequestError(res)
	}

	return nil
}
<commit_msg>client: added alternative constructor<commit_after>// Package client implements a low-level OAuth2 client to perform the various
// request/response flows against a OAuth2 authentication server.
package client

import (
	"net/http"

	"github.com/256dpi/oauth2"
)

// Config is used to configure a client.
type Config struct {
	BaseURI               string
	TokenEndpoint         string
	IntrospectionEndpoint string
	RevocationEndpoint    string
}

// Default will return a default configuration.
func Default(baseURI string) Config {
	return Config{
		BaseURI:               baseURI,
		TokenEndpoint:         "/oauth2/token",
		IntrospectionEndpoint: "/oauth2/introspect",
		RevocationEndpoint:    "/oauth2/revoke",
	}
}

// Client is a low-level OAuth2 client.
type Client struct {
	config Config
	client *http.Client
}

// New will create and return a new client.
func New(config Config) *Client {
	return NewWithClient(config, new(http.Client))
}

// NewWithClient will create and return an new client using the provided client.
func NewWithClient(config Config, client *http.Client) *Client {
	return &Client{
		config: config,
		client: client,
	}
}

// Authenticate will send the provided token request and return the servers
// token response or an error if failed.
func (c *Client) Authenticate(trq oauth2.TokenRequest) (*oauth2.TokenResponse, error) {
	// prepare endpoint
	endpoint := c.config.BaseURI + c.config.TokenEndpoint

	// build request
	req, err := BuildTokenRequest(endpoint, trq)
	if err != nil {
		return nil, err
	}

	// perform request
	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	// check status
	if res.StatusCode != http.StatusOK {
		return nil, ParseRequestError(res)
	}

	// parse response
	trs, err := ParseTokenResponse(res)
	if err != nil {
		return nil, err
	}

	return trs, nil
}

// Introspect will send the provided introspection request and return the servers
// response of an error if failed.
func (c *Client) Introspect(irq oauth2.IntrospectionRequest) (*oauth2.IntrospectionResponse, error) {
	// prepare endpoint
	endpoint := c.config.BaseURI + c.config.IntrospectionEndpoint

	// build request
	req, err := BuildIntrospectionRequest(endpoint, irq)
	if err != nil {
		return nil, err
	}

	// perform request
	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	// check status
	if res.StatusCode != http.StatusOK {
		return nil, ParseRequestError(res)
	}

	// parse response
	irs, err := ParseIntrospectionResponse(res)
	if err != nil {
		return nil, err
	}

	return irs, nil
}

// Revoke will send the provided revocation request and return and error if it
// failed.
func (c *Client) Revoke(rrq oauth2.RevocationRequest) error {
	// prepare endpoint
	endpoint := c.config.BaseURI + c.config.RevocationEndpoint

	// build request
	req, err := BuildRevocationRequest(endpoint, rrq)
	if err != nil {
		return err
	}

	// perform request
	res, err := c.client.Do(req)
	if err != nil {
		return err
	}

	// check status
	if res.StatusCode != http.StatusOK {
		return ParseRequestError(res)
	}

	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"hash/crc32"
	"strconv"

	"github.com/vsco/dcdr/cli/printer"
	"github.com/vsco/dcdr/client/models"
	"github.com/vsco/dcdr/client/watcher"
	"github.com/vsco/dcdr/config"
)

type ClientIFace interface {
	IsAvailable(feature string) bool
	IsAvailableForId(feature string, id uint64) bool
	ScaleValue(feature string, min float64, max float64) float64
	UpdateFeatures(bts []byte)
	FeatureExists(feature string) bool
	Features() models.Features
	FeatureMap() *models.FeatureMap
	ScopedMap() *models.FeatureMap
	Scopes() []string
	CurrentSha() string
	WithScopes(scopes ...string) *Client
}

type Client struct {
	featureMap *models.FeatureMap
	config     *config.Config
	watcher    watcher.WatcherIFace
	features   models.Features
	scopes     []string
}

func New(cfg *config.Config) (c *Client) {
	c = &Client{
		config: cfg,
	}

	if c.config.Watcher.OutputPath != "" {
		printer.Say("started watching %s", c.config.Watcher.OutputPath)
		c.watcher = watcher.NewWatcher(c.config.Watcher.OutputPath)
	}

	return
}

func NewDefault() (c *Client) {
	c = New(config.LoadConfig())

	return
}

func (c *Client) WithScopes(scopes ...string) *Client {
	if len(scopes) == 0 {
		return c
	}

	if len(scopes) == 1 && scopes[0] == "" {
		return c
	}

	newScopes := append(c.scopes, scopes...)

	newClient := &Client{
		featureMap: c.featureMap,
		scopes:     newScopes,
	}

	newClient.MergeScopes()

	if c.watcher != nil {
		newClient.watcher = watcher.NewWatcher(c.config.Watcher.OutputPath)
		newClient.Watch()
	}

	return newClient
}

func (c *Client) MergeScopes() {
	if c.featureMap != nil {
		c.features = c.featureMap.Dcdr.MergedScopes(c.scopes...)
	}
}

func (c *Client) Scopes() []string {
	return c.scopes
}

func (c *Client) SetFeatureMap(fm *models.FeatureMap) *Client {
	c.featureMap = fm

	c.MergeScopes()

	return c
}

func (c *Client) FeatureMap() *models.FeatureMap {
	return c.featureMap
}

func (c *Client) ScopedMap() *models.FeatureMap {
	fm := &models.FeatureMap{
		Dcdr: models.Root{
			Features: c.Features(),
		},
	}

	return fm
}

func (c *Client) Features() models.Features {
	return c.features
}

func (c *Client) CurrentSha() string {
	return c.FeatureMap().Dcdr.Info.CurrentSha
}

// FeatureExists checks the existence of a key
func (c *Client) FeatureExists(feature string) bool {
	_, exists := c.Features()[feature]

	return exists
}

// IsAvailable used to check features with boolean values.
func (c *Client) IsAvailable(feature string) bool {
	val, exists := c.Features()[feature]

	switch val.(type) {
	case bool:
		return exists && val.(bool)
	default:
		return false
	}
}

// IsAvailableForId used to check features with float values between 0.0-1.0.
func (c *Client) IsAvailableForId(feature string, id uint64) bool {
	val, exists := c.Features()[feature]

	switch val.(type) {
	case float64, int:
		return exists && c.withinPercentile(id, val.(float64), feature)
	default:
		return false
	}
}

func (c *Client) UpdateFeatures(bts []byte) {
	fm, err := models.NewFeatureMap(bts)

	if err != nil {
		printer.SayErr("parse error: %v", err)
		return
	}

	c.SetFeatureMap(fm)
}

// ScaleValue returns a value scaled between min and max
// given the current value of the feature.
func (c *Client) ScaleValue(feature string, min float64, max float64) float64 {
	val, exists := c.Features()[feature]

	if !exists {
		return min
	}

	switch val.(type) {
	case float64, int:
		return min + (max-min)*val.(float64)
	default:
		return min
	}
}

func (c *Client) Watch() (*Client, error) {
	if c.watcher != nil {
		err := c.watcher.Init()

		if err != nil {
			return nil, err
		}

		c.watcher.Register(c.UpdateFeatures)
		go c.watcher.Watch()
	}

	return c, nil
}

func (c *Client) withinPercentile(id uint64, val float64, feature string) bool {
	uid := c.crc(id, feature)
	percentage := uint32(val * 100)

	return uid%100 < percentage
}

func (c *Client) crc(id uint64, feature string) uint32 {
	b := []byte(feature + strconv.FormatInt(int64(id), 10))

	return crc32.ChecksumIEEE(b)
}
<commit_msg>Commenting the Client methods<commit_after>package client

import (
	"hash/crc32"
	"strconv"

	"github.com/vsco/dcdr/cli/printer"
	"github.com/vsco/dcdr/client/models"
	"github.com/vsco/dcdr/client/watcher"
	"github.com/vsco/dcdr/config"
)

// ClientIFace interface for Decider Clients
type ClientIFace interface {
	IsAvailable(feature string) bool
	IsAvailableForId(feature string, id uint64) bool
	ScaleValue(feature string, min float64, max float64) float64
	UpdateFeatures(bts []byte)
	FeatureExists(feature string) bool
	Features() models.Features
	FeatureMap() *models.FeatureMap
	ScopedMap() *models.FeatureMap
	Scopes() []string
	CurrentSha() string
	WithScopes(scopes ...string) *Client
}

// Client handles access to the FeatureMap
type Client struct {
	featureMap *models.FeatureMap
	config     *config.Config
	watcher    watcher.WatcherIFace
	features   models.Features
	scopes     []string
}

// New creates a new Client with a custom Config
func New(cfg *config.Config) (c *Client) {
	c = &Client{
		config: cfg,
	}

	if c.config.Watcher.OutputPath != "" {
		printer.Say("started watching %s", c.config.Watcher.OutputPath)
		c.watcher = watcher.NewWatcher(c.config.Watcher.OutputPath)
	}

	return
}

// NewDefault creates a new default Client
func NewDefault() (c *Client) {
	c = New(config.LoadConfig())

	return
}

// WithScopes creates a new Client from an existing one that is "scoped"
// to the provided scopes param. `scopes` are provided in priority order.
// For example, when given WithScopes("a", "b", "c"). Keys found in "a"
// will override the same keys found in "b" and so on for "c".
//
// The provided `scopes` are appended to the existing Client's `scopes`,
// merged, and then a new `Watcher` is assigned to the new `Client` so
// that future changes to the `FeatureMap` will be observed.
func (c *Client) WithScopes(scopes ...string) *Client {
	if len(scopes) == 0 {
		return c
	}

	if len(scopes) == 1 && scopes[0] == "" {
		return c
	}

	newScopes := append(c.scopes, scopes...)

	newClient := &Client{
		featureMap: c.featureMap,
		scopes:     newScopes,
	}

	newClient.MergeScopes()

	if c.watcher != nil {
		newClient.watcher = watcher.NewWatcher(c.config.Watcher.OutputPath)
		newClient.Watch()
	}

	return newClient
}

// MergeScopes delegates merging to the underlying `FeatureMap`
func (c *Client) MergeScopes() {
	if c.featureMap != nil {
		c.features = c.featureMap.Dcdr.MergedScopes(c.scopes...)
	}
}

// Scopes `scopes` accessor
func (c *Client) Scopes() []string {
	return c.scopes
}

// SetFeatureMap assigns a `FeatureMap` and merges the current scopes
func (c *Client) SetFeatureMap(fm *models.FeatureMap) *Client {
	c.featureMap = fm

	c.MergeScopes()

	return c
}

// FeatureMap `featureMap` accessor
func (c *Client) FeatureMap() *models.FeatureMap {
	return c.featureMap
}

// ScopedMap a `FeatureMap` containing only merged features.
// Mostly used for JSON output.
func (c *Client) ScopedMap() *models.FeatureMap {
	fm := &models.FeatureMap{
		Dcdr: models.Root{
			Features: c.Features(),
		},
	}

	return fm
}

// Features `features` accessor
func (c *Client) Features() models.Features {
	return c.features
}

// CurrentSha accessor for the underlying `CurrentSha` from
// the `FeatureMap`
func (c *Client) CurrentSha() string {
	return c.FeatureMap().Dcdr.Info.CurrentSha
}

// FeatureExists checks the existence of a key
func (c *Client) FeatureExists(feature string) bool {
	_, exists := c.Features()[feature]

	return exists
}

// IsAvailable used to check features with boolean values.
func (c *Client) IsAvailable(feature string) bool {
	val, exists := c.Features()[feature]

	switch val.(type) {
	case bool:
		return exists && val.(bool)
	default:
		return false
	}
}

// IsAvailableForId used to check features with float values between 0.0-1.0.
func (c *Client) IsAvailableForId(feature string, id uint64) bool {
	val, exists := c.Features()[feature]

	switch val.(type) {
	case float64, int:
		return exists && c.withinPercentile(id, val.(float64), feature)
	default:
		return false
	}
}

// UpdateFeatures creates and assigns a new `FeatureMap` from a
// Marshalled JSON byte array
func (c *Client) UpdateFeatures(bts []byte) {
	fm, err := models.NewFeatureMap(bts)

	if err != nil {
		printer.SayErr("parse error: %v", err)
		return
	}

	c.SetFeatureMap(fm)
}

// ScaleValue returns a value scaled between min and max
// given the current value of the feature.
//
// Given the K/V dcdr/features/scalar => 0.5
// ScaleValue("scalar", 0, 10) => 5
func (c *Client) ScaleValue(feature string, min float64, max float64) float64 {
	val, exists := c.Features()[feature]

	if !exists {
		return min
	}

	switch val.(type) {
	case float64, int:
		return min + (max-min)*val.(float64)
	default:
		return min
	}
}

// Watch initializes the `Watcher`, registers the `UpdateFeatures`
// method with it and spawns the watch in a go routine returning the
// `Client` for a fluent interface.
func (c *Client) Watch() (*Client, error) {
	if c.watcher != nil {
		err := c.watcher.Init()

		if err != nil {
			return nil, err
		}

		c.watcher.Register(c.UpdateFeatures)
		go c.watcher.Watch()
	}

	return c, nil
}

func (c *Client) withinPercentile(id uint64, val float64, feature string) bool {
	uid := c.crc(id, feature)
	percentage := uint32(val * 100)

	return uid%100 < percentage
}

func (c *Client) crc(id uint64, feature string) uint32 {
	b := []byte(feature + strconv.FormatInt(int64(id), 10))

	return crc32.ChecksumIEEE(b)
}
<|endoftext|>