<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that scans all of the blobs in a GCS bucket, verifying their
// scores against their content.
//
// Output is space-separated lines of the form:
//
//     <score> <child score>*
//
// where <score> is a verified score, and the list is a list of scores of
// children for directories (and empty for files).

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/crypto"
	"github.com/jacobsa/comeback/repr"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdScanBlobs = &Command{
	Name: "scan_blobs",
}

var fOutputFile = cmdScanBlobs.Flags.String(
	"output_file", "", "Path to scan_blobs output file. Will be truncated.")

func init() {
	cmdScanBlobs.Run = runScanBlobs // Break flag-related dependency loop.
}

type scoreAndContents struct {
	score    blob.Score
	contents []byte
}

type verifiedScore struct {
	score    blob.Score
	children []blob.Score
}

// List all scores in the GCS bucket into the channel. Do not close the
// channel.
func listBlobs(
	ctx context.Context,
	bucket gcs.Bucket,
	scores chan<- blob.Score) (err error) {
	req := &gcs.ListObjectsRequest{
		Prefix: blobKeyPrefix,
	}

	// List until we run out.
	for {
		// Fetch the next batch.
		var listing *gcs.Listing
		listing, err = bucket.ListObjects(ctx, req)
		if err != nil {
			err = fmt.Errorf("ListObjects: %v", err)
			return
		}

		// Transform to scores.
		var batch []blob.Score
		for _, o := range listing.Objects {
			var score blob.Score
			score, err = blob.ParseHexScore(strings.TrimPrefix(o.Name, blobKeyPrefix))
			if err != nil {
				err = fmt.Errorf("Parsing object name \"%s\": %v", o.Name, err)
				return
			}

			batch = append(batch, score)
		}

		// Feed out each score.
		for _, score := range batch {
			select {
			case scores <- score:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		// Are we done?
		if listing.ContinuationToken == "" {
			break
		}

		req.ContinuationToken = listing.ContinuationToken
	}

	return
}

func readAndClose(rc io.ReadCloser) (d []byte, err error) {
	// Read.
	d, err = ioutil.ReadAll(rc)
	if err != nil {
		rc.Close()
		err = fmt.Errorf("ReadAll: %v", err)
		return
	}

	// Close.
	err = rc.Close()
	if err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

func readBlob(
	ctx context.Context,
	bucket gcs.Bucket,
	score blob.Score) (blob scoreAndContents, err error) {
	// Fill in the score.
	blob.score = score

	// Obtain a reader.
	req := &gcs.ReadObjectRequest{
		Name: blobKeyPrefix + score.Hex(),
	}

	rc, err := bucket.NewReader(ctx, req)
	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	// Consume it.
	blob.contents, err = readAndClose(rc)
	if err != nil {
		err = fmt.Errorf("readAndClose: %v", err)
		return
	}

	return
}

func readBlobWithRetryLoop(
	ctx context.Context,
	bucket gcs.Bucket,
	score blob.Score) (blob scoreAndContents, err error) {
	const maxRetries = 5
	for n := 0; n < maxRetries; n++ {
		blob, err = readBlob(ctx, bucket, score)
		if err == nil {
			return
		}

		time.Sleep(500 * time.Millisecond)
	}

	return
}

// Read the contents of blobs specified on the incoming channel. Do not close
// the outgoing channel.
func readBlobs(
	ctx context.Context,
	bucket gcs.Bucket,
	scores <-chan blob.Score,
	blobs chan<- scoreAndContents) (err error) {
	for score := range scores {
		var blob scoreAndContents

		// Read the contents for this score.
		blob, err = readBlobWithRetryLoop(ctx, bucket, score)
		if err != nil {
			err = fmt.Errorf("readBlob: %v", err)
			return
		}

		select {
		case blobs <- blob:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

func scoresEqual(a, b blob.Score) bool {
	for i := 0; i < blob.ScoreLength; i++ {
		if a[i] != b[i] {
			return false
		}
	}

	return true
}

// Parse the blob ciphertext as a directory if appropriate, returning a list of
// children. If not a directory, return the empty list.
func unmarshalBlob(
	crypter crypto.Crypter,
	b []byte) (children []blob.Score, err error) {
	// Attempt to decrypt.
	b, err = crypter.Decrypt(b)
	if err != nil {
		err = fmt.Errorf("Decrypt: %v", err)
		return
	}

	// If this is a file, simply make sure it is in a legal format.
	if !repr.IsDir(b) {
		_, err = repr.UnmarshalFile(b)
		if err != nil {
			err = fmt.Errorf("UnmarshalFile: %v", err)
			return
		}

		return
	}

	// Otherwise, parse it as a directory.
	entries, err := repr.UnmarshalDir(b)
	if err != nil {
		err = fmt.Errorf("UnmarshalDir: %v", err)
		return
	}

	// Pull out the child scores.
	for _, entry := range entries {
		for _, score := range entry.Scores {
			children = append(children, score)
		}
	}

	return
}

// Verify the contents of the incoming blobs. Do not close the outgoing
// channel.
func verifyScores(
	ctx context.Context,
	crypter crypto.Crypter,
	blobs <-chan scoreAndContents,
	results chan<- verifiedScore) (err error) {
	for b := range blobs {
		result := verifiedScore{
			score: b.score,
		}

		// Make sure the score matches.
		computed := blob.ComputeScore(b.contents)
		if !scoresEqual(b.score, computed) {
			err = fmt.Errorf(
				"Score mismatch: %v vs. %v",
				b.score.Hex(),
				computed.Hex())

			return
		}

		// Parse the blob.
		result.children, err = unmarshalBlob(crypter, b.contents)
		if err != nil {
			err = fmt.Errorf("unmarshalBlob: %v", err)
			return
		}

		// Write out the result.
		select {
		case results <- result:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

// Write results to stdout.
func writeResults(
	ctx context.Context,
	output io.Writer,
	results <-chan verifiedScore) (err error) {
	// Log also to stdout.
	output = io.MultiWriter(output, os.Stdout)

	// Process each result.
	for result := range results {
		// Write the score.
		_, err = fmt.Fprintf(output, "%s", result.score.Hex())
		if err != nil {
			return
		}

		// Write its children.
		for _, child := range result.children {
			_, err = fmt.Fprintf(output, " %s", child.Hex())
			if err != nil {
				return
			}
		}

		// Add a line separator.
		_, err = fmt.Fprintf(output, "\n")
		if err != nil {
			return
		}
	}

	return
}

func runScanBlobs(args []string) {
	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Open the output file.
	if *fOutputFile == "" {
		err = fmt.Errorf("You must set --output_file.")
		return
	}

	output, err := os.OpenFile(
		*fOutputFile,
		os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
		0666)

	if err != nil {
		err = fmt.Errorf("OpenFile: %v", err)
		return
	}

	defer output.Close()

	// Grab dependencies.
	bucket := getBucket()
	crypter := getCrypter()

	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	b := syncutil.NewBundle(context.Background())

	// List all of the scores in the bucket.
	scores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(scores)
		err = listBlobs(ctx, bucket, scores)
		return
	})

	// Read the contents of the corresponding blobs in parallel, bounding how
	// hard we hammer GCS by bounding the parallelism.
	const readWorkers = 8
	var readWaitGroup sync.WaitGroup

	blobs := make(chan scoreAndContents, 10)
	for i := 0; i < readWorkers; i++ {
		readWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer readWaitGroup.Done()
			err = readBlobs(ctx, bucket, scores, blobs)
			return
		})
	}

	go func() {
		readWaitGroup.Wait()
		close(blobs)
	}()

	// Verify the blob contents and summarize their children. Use one worker per
	// CPU.
	var verifyWaitGroup sync.WaitGroup

	results := make(chan verifiedScore, 100)
	for i := 0; i < runtime.NumCPU(); i++ {
		verifyWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer verifyWaitGroup.Done()
			err = verifyScores(ctx, crypter, blobs, results)
			return
		})
	}

	go func() {
		verifyWaitGroup.Wait()
		close(results)
	}()

	// Process results.
	b.Add(func(ctx context.Context) (err error) {
		err = writeResults(ctx, output, results)
		return
	})

	// Wait for everything to complete.
	err = b.Join()
	return
}
<commit_msg>Fixed some more build errors.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that scans all of the blobs in a GCS bucket, verifying their
// scores against their content.
//
// Output is space-separated lines of the form:
//
//     <score> <child score>*
//
// where <score> is a verified score, and the list is a list of scores of
// children for directories (and empty for files).

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/crypto"
	"github.com/jacobsa/comeback/repr"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdScanBlobs = &Command{
	Name: "scan_blobs",
}

var fOutputFile = cmdScanBlobs.Flags.String(
	"output_file", "", "Path to scan_blobs output file. Will be truncated.")

func init() {
	cmdScanBlobs.Run = runScanBlobs // Break flag-related dependency loop.
}

type scoreAndContents struct {
	score    blob.Score
	contents []byte
}

type verifiedScore struct {
	score    blob.Score
	children []blob.Score
}

// List all scores in the GCS bucket into the channel. Do not close the
// channel.
func listBlobs(
	ctx context.Context,
	bucket gcs.Bucket,
	scores chan<- blob.Score) (err error) {
	req := &gcs.ListObjectsRequest{
		Prefix: blobObjectNamePrefix,
	}

	// List until we run out.
	for {
		// Fetch the next batch.
		var listing *gcs.Listing
		listing, err = bucket.ListObjects(ctx, req)
		if err != nil {
			err = fmt.Errorf("ListObjects: %v", err)
			return
		}

		// Transform to scores.
		var batch []blob.Score
		for _, o := range listing.Objects {
			var score blob.Score
			score, err = blob.ParseHexScore(
				strings.TrimPrefix(o.Name, blobObjectNamePrefix))

			if err != nil {
				err = fmt.Errorf("Parsing object name \"%s\": %v", o.Name, err)
				return
			}

			batch = append(batch, score)
		}

		// Feed out each score.
		for _, score := range batch {
			select {
			case scores <- score:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		// Are we done?
		if listing.ContinuationToken == "" {
			break
		}

		req.ContinuationToken = listing.ContinuationToken
	}

	return
}

func readAndClose(rc io.ReadCloser) (d []byte, err error) {
	// Read.
	d, err = ioutil.ReadAll(rc)
	if err != nil {
		rc.Close()
		err = fmt.Errorf("ReadAll: %v", err)
		return
	}

	// Close.
	err = rc.Close()
	if err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

func readBlob(
	ctx context.Context,
	bucket gcs.Bucket,
	score blob.Score) (blob scoreAndContents, err error) {
	// Fill in the score.
	blob.score = score

	// Obtain a reader.
	req := &gcs.ReadObjectRequest{
		Name: blobObjectNamePrefix + score.Hex(),
	}

	rc, err := bucket.NewReader(ctx, req)
	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	// Consume it.
	blob.contents, err = readAndClose(rc)
	if err != nil {
		err = fmt.Errorf("readAndClose: %v", err)
		return
	}

	return
}

func readBlobWithRetryLoop(
	ctx context.Context,
	bucket gcs.Bucket,
	score blob.Score) (blob scoreAndContents, err error) {
	const maxRetries = 5
	for n := 0; n < maxRetries; n++ {
		blob, err = readBlob(ctx, bucket, score)
		if err == nil {
			return
		}

		time.Sleep(500 * time.Millisecond)
	}

	return
}

// Read the contents of blobs specified on the incoming channel. Do not close
// the outgoing channel.
func readBlobs(
	ctx context.Context,
	bucket gcs.Bucket,
	scores <-chan blob.Score,
	blobs chan<- scoreAndContents) (err error) {
	for score := range scores {
		var blob scoreAndContents

		// Read the contents for this score.
		blob, err = readBlobWithRetryLoop(ctx, bucket, score)
		if err != nil {
			err = fmt.Errorf("readBlob: %v", err)
			return
		}

		select {
		case blobs <- blob:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

func scoresEqual(a, b blob.Score) bool {
	for i := 0; i < blob.ScoreLength; i++ {
		if a[i] != b[i] {
			return false
		}
	}

	return true
}

// Parse the blob ciphertext as a directory if appropriate, returning a list of
// children. If not a directory, return the empty list.
func unmarshalBlob(
	crypter crypto.Crypter,
	b []byte) (children []blob.Score, err error) {
	// Attempt to decrypt.
	b, err = crypter.Decrypt(b)
	if err != nil {
		err = fmt.Errorf("Decrypt: %v", err)
		return
	}

	// If this is a file, simply make sure it is in a legal format.
	if !repr.IsDir(b) {
		_, err = repr.UnmarshalFile(b)
		if err != nil {
			err = fmt.Errorf("UnmarshalFile: %v", err)
			return
		}

		return
	}

	// Otherwise, parse it as a directory.
	entries, err := repr.UnmarshalDir(b)
	if err != nil {
		err = fmt.Errorf("UnmarshalDir: %v", err)
		return
	}

	// Pull out the child scores.
	for _, entry := range entries {
		for _, score := range entry.Scores {
			children = append(children, score)
		}
	}

	return
}

// Verify the contents of the incoming blobs. Do not close the outgoing
// channel.
func verifyScores(
	ctx context.Context,
	crypter crypto.Crypter,
	blobs <-chan scoreAndContents,
	results chan<- verifiedScore) (err error) {
	for b := range blobs {
		result := verifiedScore{
			score: b.score,
		}

		// Make sure the score matches.
		computed := blob.ComputeScore(b.contents)
		if !scoresEqual(b.score, computed) {
			err = fmt.Errorf(
				"Score mismatch: %v vs. %v",
				b.score.Hex(),
				computed.Hex())

			return
		}

		// Parse the blob.
		result.children, err = unmarshalBlob(crypter, b.contents)
		if err != nil {
			err = fmt.Errorf("unmarshalBlob: %v", err)
			return
		}

		// Write out the result.
		select {
		case results <- result:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

// Write results to stdout.
func writeResults(
	ctx context.Context,
	output io.Writer,
	results <-chan verifiedScore) (err error) {
	// Log also to stdout.
	output = io.MultiWriter(output, os.Stdout)

	// Process each result.
	for result := range results {
		// Write the score.
		_, err = fmt.Fprintf(output, "%s", result.score.Hex())
		if err != nil {
			return
		}

		// Write its children.
		for _, child := range result.children {
			_, err = fmt.Fprintf(output, " %s", child.Hex())
			if err != nil {
				return
			}
		}

		// Add a line separator.
		_, err = fmt.Fprintf(output, "\n")
		if err != nil {
			return
		}
	}

	return
}

func runScanBlobs(args []string) {
	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Open the output file.
	if *fOutputFile == "" {
		err = fmt.Errorf("You must set --output_file.")
		return
	}

	output, err := os.OpenFile(
		*fOutputFile,
		os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
		0666)

	if err != nil {
		err = fmt.Errorf("OpenFile: %v", err)
		return
	}

	defer output.Close()

	// Grab dependencies.
	bucket := getBucket()
	crypter := getCrypter()

	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	b := syncutil.NewBundle(context.Background())

	// List all of the scores in the bucket.
	scores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(scores)
		err = listBlobs(ctx, bucket, scores)
		return
	})

	// Read the contents of the corresponding blobs in parallel, bounding how
	// hard we hammer GCS by bounding the parallelism.
	const readWorkers = 8
	var readWaitGroup sync.WaitGroup

	blobs := make(chan scoreAndContents, 10)
	for i := 0; i < readWorkers; i++ {
		readWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer readWaitGroup.Done()
			err = readBlobs(ctx, bucket, scores, blobs)
			return
		})
	}

	go func() {
		readWaitGroup.Wait()
		close(blobs)
	}()

	// Verify the blob contents and summarize their children. Use one worker per
	// CPU.
	var verifyWaitGroup sync.WaitGroup

	results := make(chan verifiedScore, 100)
	for i := 0; i < runtime.NumCPU(); i++ {
		verifyWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer verifyWaitGroup.Done()
			err = verifyScores(ctx, crypter, blobs, results)
			return
		})
	}

	go func() {
		verifyWaitGroup.Wait()
		close(results)
	}()

	// Process results.
	b.Add(func(ctx context.Context) (err error) {
		err = writeResults(ctx, output, results)
		return
	})

	// Wait for everything to complete.
	err = b.Join()
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"fmt"
	id3 "github.com/mikkyang/id3-go"
	kingpin "gopkg.in/alecthomas/kingpin.v1"
	"net/url"
	"os"
	"path/filepath"
	"time"
)

const (
	Header   = `<?xml version="1.0" encoding="UTF-8"?>` + "\n"
	iTunesNs = "http://www.itunes.com/dtds/podcast-1.0.dtd"
)

type Rss struct {
	XMLName xml.Name `xml:"rss"`
	Channel Channel  `xml:"channel"`
	Version string   `xml:"version,attr"`
	NS      string   `xml:"xmlns:itunes,attr"`
}

type Channel struct {
	PubDate     string  `xml:"pubDate,omitempty"`
	Title       string  `xml:"title,omitempty"`
	Link        string  `xml:"link,omitempty"`
	Description string  `xml:"description,omitempty"`
	Language    string  `xml:"language,omitempty"`
	Images      []Image `xml:"image,omitempty"`
	Items       []Item  `xml:"item"`
}

type Item struct {
	Title       string    `xml:"title"`
	Description string    `xml:"description"`
	Enclosure   Enclosure `xml:"enclosure"`
	Guid        string    `xml:"guid"`
	Subtitle    string    `xml:"itunes:subtitle,omitempty"`
	PubDate     string    `xml:"pubDate,omitempty"`
}

type Image struct {
	Link  string `xml:"link"`
	Title string `xml:"title"`
	Url   string `xml:"url"`
}

type Enclosure struct {
	Url    string `xml:"url,attr"`
	Length int64  `xml:"length,attr"`
	Type   string `xml:"type,attr"`
}

func fileUrl(relativePath string, baseUrl string) string {
	Url, _ := url.Parse(baseUrl)
	Url.Path += relativePath
	return Url.String()
}

func formatYear(year string) string {
	if len(year) > 0 {
		t, err := time.Parse("20060102", year)
		if err != nil {
			t, err = time.Parse("20060102", year[0:len(year)-1])
			if err == nil {
				return t.String()
			}
			t, err = time.Parse("2006", year)
			if err == nil {
				return t.String()
			}
			t, err = time.Parse("20060201", year)
			if err != nil {
				return ""
			}
		}
		return t.String()
	}
	return year
}

func addMeta(path string, f os.FileInfo, item *Item) {
	fd, err := id3.Open(path)
	if err != nil {
		item.Title = f.Name()
	} else {
		title := fd.Title()
		author := fd.Artist()
		if len(title) > 0 {
			item.Title = title
		} else {
			item.Title = author
			if len(author) > 0 {
				item.Title += " - "
			}
			item.Title += f.Name()
		}
		item.Subtitle = author
		item.PubDate = formatYear(fd.Year())
	}
}

func visitFiles(workDir string, channel *Channel, publicUrl string, recursive bool) filepath.WalkFunc {
	return func(path string, f os.FileInfo, err error) error {

		if err != nil {
			return err
		}

		if f.IsDir() && path != workDir && !recursive {
			return filepath.SkipDir
		}

		if !!f.IsDir() {
			return nil
		}

		matched, _ := filepath.Match("*.mp3", f.Name())
		if matched {
			url := fileUrl(path[len(workDir)-1:], publicUrl)
			item := Item{Enclosure: Enclosure{Length: f.Size(), Type: "audio/mpeg",
				Url: url}, Guid: url}
			addMeta(path, f, &item)
			channel.Items = append(channel.Items, item)
		}

		return nil

	}
}

var (
	baseUrl     = kingpin.Flag("server", "hostname (and path) to the root e.g. http://myserver.com/rss").Short('s').Default("http://localhost").URL()
	recursive   = kingpin.Flag("recursive", "how to handle the directory scan").Short('r').Bool()
	language    = kingpin.Flag("language", "the language of the RSS document, a ISO 639 value").Short('l').String()
	title       = kingpin.Flag("title", "RSS channel title").Short('t').Default("RSS FEED").String()
	description = kingpin.Flag("description", "RSS channel description").Short('d').String()
	imageUrl    = kingpin.Flag("image", "Image URL for the RSS channel image").Short('i').URL()
	path        = kingpin.Arg("directory", "directory to read files relative from").Required().ExistingDir()
)

func main() {

	kingpin.Version("0.0.1")
	kingpin.Parse()

	channel := &Channel{
		PubDate:     time.Now().String(),
		Title:       *title,
		Link:        (*baseUrl).String(),
		Description: *description,
		Language:    *language}

	err := filepath.Walk(*path, visitFiles(*path, channel, (*baseUrl).String(), *recursive))
	if *imageUrl != nil {
		channel.Images = append(channel.Images, Image{Title: channel.Title, Link: channel.Link, Url: (*imageUrl).String()})
	}

	if err != nil {
		fmt.Printf("%s: %v\n", os.Args[0], err)
	} else {
		output, err := xml.MarshalIndent(
			&Rss{Channel: *channel, Version: "2.0", NS: iTunesNs}, "", "  ")
		if err != nil {
			fmt.Printf("error: %v\n", err)
		} else {
			os.Stdout.WriteString(Header)
			os.Stdout.Write(output)
		}
	}

}
<commit_msg>fix year format for pubDate <commit_after>package main

import (
	"encoding/xml"
	"fmt"
	id3 "github.com/mikkyang/id3-go"
	kingpin "gopkg.in/alecthomas/kingpin.v1"
	"net/url"
	"os"
	"path/filepath"
	"time"
)

const (
	Header   = `<?xml version="1.0" encoding="UTF-8"?>` + "\n"
	iTunesNs = "http://www.itunes.com/dtds/podcast-1.0.dtd"
)

type Rss struct {
	XMLName xml.Name `xml:"rss"`
	Channel Channel  `xml:"channel"`
	Version string   `xml:"version,attr"`
	NS      string   `xml:"xmlns:itunes,attr"`
}

type Channel struct {
	PubDate     string  `xml:"pubDate,omitempty"`
	Title       string  `xml:"title,omitempty"`
	Link        string  `xml:"link,omitempty"`
	Description string  `xml:"description,omitempty"`
	Language    string  `xml:"language,omitempty"`
	Images      []Image `xml:"image,omitempty"`
	Items       []Item  `xml:"item"`
}

type Item struct {
	Title       string    `xml:"title"`
	Description string    `xml:"description"`
	Enclosure   Enclosure `xml:"enclosure"`
	Guid        string    `xml:"guid"`
	Subtitle    string    `xml:"itunes:subtitle,omitempty"`
	PubDate     string    `xml:"pubDate,omitempty"`
}

type Image struct {
	Link  string `xml:"link"`
	Title string `xml:"title"`
	Url   string `xml:"url"`
}

type Enclosure struct {
	Url    string `xml:"url,attr"`
	Length int64  `xml:"length,attr"`
	Type   string `xml:"type,attr"`
}

func fileUrl(relativePath string, baseUrl string) string {
	Url, _ := url.Parse(baseUrl)
	Url.Path += relativePath
	return Url.String()
}

func formatYear(year string) string {
	if len(year) > 0 {
		t, err := time.Parse("20060102", year)
		if err != nil {
			t, err = time.Parse("20060102", year[0:len(year)-1])
			if err != nil {
				t, err = time.Parse("2006", year)
				if err != nil {
					t, err = time.Parse("20060201", year)
					if err != nil {
						return ""
					}
				}
			}
		}
		return t.Format(time.RFC1123Z)
	}
	return year
}

func addMeta(path string, f os.FileInfo, item *Item) {
	fd, err := id3.Open(path)
	if err != nil {
		item.Title = f.Name()
	} else {
		title := fd.Title()
		author := fd.Artist()
		if len(title) > 0 {
			item.Title = title
		} else {
			item.Title = author
			if len(author) > 0 {
				item.Title += " - "
			}
			item.Title += f.Name()
		}
		item.Subtitle = author
		item.PubDate = formatYear(fd.Year())
	}
}

func visitFiles(workDir string, channel *Channel, publicUrl string, recursive bool) filepath.WalkFunc {
	return func(path string, f os.FileInfo, err error) error {

		if err != nil {
			return err
		}

		if f.IsDir() && path != workDir && !recursive {
			return filepath.SkipDir
		}

		if !!f.IsDir() {
			return nil
		}

		matched, _ := filepath.Match("*.mp3", f.Name())
		if matched {
			url := fileUrl(path[len(workDir)-1:], publicUrl)
			item := Item{Enclosure: Enclosure{Length: f.Size(), Type: "audio/mpeg",
				Url: url}, Guid: url}
			addMeta(path, f, &item)
			channel.Items = append(channel.Items, item)
		}

		return nil

	}
}

var (
	baseUrl     = kingpin.Flag("server", "hostname (and path) to the root e.g. http://myserver.com/rss").Short('s').Default("http://localhost").URL()
	recursive   = kingpin.Flag("recursive", "how to handle the directory scan").Short('r').Bool()
	language    = kingpin.Flag("language", "the language of the RSS document, a ISO 639 value").Short('l').String()
	title       = kingpin.Flag("title", "RSS channel title").Short('t').Default("RSS FEED").String()
	description = kingpin.Flag("description", "RSS channel description").Short('d').String()
	imageUrl    = kingpin.Flag("image", "Image URL for the RSS channel image").Short('i').URL()
	path        = kingpin.Arg("directory", "directory to read files relative from").Required().ExistingDir()
)

func main() {

	kingpin.Version("0.0.1")
	kingpin.Parse()

	channel := &Channel{
		PubDate:     time.Now().Format(time.RFC1123Z),
		Title:       *title,
		Link:        (*baseUrl).String(),
		Description: *description,
		Language:    *language}

	err := filepath.Walk(*path, visitFiles(*path, channel, (*baseUrl).String(), *recursive))
	if *imageUrl != nil {
		channel.Images = append(channel.Images, Image{Title: channel.Title, Link: channel.Link, Url: (*imageUrl).String()})
	}

	if err != nil {
		fmt.Printf("%s: %v\n", os.Args[0], err)
	} else {
		output, err := xml.MarshalIndent(
			&Rss{Channel: *channel, Version: "2.0", NS: iTunesNs}, "", "  ")
		if err != nil {
			fmt.Printf("error: %v\n", err)
		} else {
			os.Stdout.WriteString(Header)
			os.Stdout.Write(output)
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/libgit2/git2go"
)

// type Repos struct {
// 	repo []Repo
// }

type Repo struct {
	Url  string
	Path string
}

var clone string = "./test"
var repo string = "https://gitlab.com/rollbrettler/go-playground.git"

func main() {

	// repos := []Repos{
	// 	{Url: "https://gitlab.com/rollbrettler/go-playground.git", Path: "./test"},
	// 	{Url: "https://github.com/rollbrettler/compress-videos.git", Path: "./test2"},
	// }
	//
	// for _, repo := range repos {
	// 	fmt.Printf("%s -> %s\n", repo.Url, repo.Path)
	// }

	os.Exit(0)

	_, err := ioutil.ReadDir(clone)
	if err != nil {
		log.Println("Directory not pressent")

		err := cloneRepository()
		if err != nil {
			log.Println(err)
			os.Exit(1)
		}
	}

	log.Println("Folder already cloned")

	repository, err := git.OpenRepository(clone)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	err = updateRepository(*repository)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	log.Println("Sucessfully updated")
	os.Exit(0)
}

func (repo Repo) cloneRepository() (err error) {

	cloneOptions := &git.CloneOptions{
		Bare: true,
	}

	repository, err := git.Clone(repo.Url, repo.Path, cloneOptions)
	if err != nil {
		log.Println(err)
		return
	}

	config, err := repository.Config()
	if err != nil {
		log.Println(err)
		return
	}

	err = repo.changeToMirrorConfig(*config)
	if err != nil {
		log.Println(err)
	}

	err = repo.updateRepository(*repository)
	if err != nil {
		log.Println(err)
	}

	return nil
}

func (repo Repo) changeToMirrorConfig(config git.Config) (err error) {

	fetch, err := config.LookupString("remote.origin.fetch")
	if err != nil {
		return err
	}

	mirror, err := config.LookupBool("remote.origin.mirror")
	if err != nil {
		return err
	}

	log.Println(fmt.Sprintf("Refspec: %v\n", fetch))
	config.SetString("remote.origin.fetch", "+refs/*:refs/*")

	log.Println(fmt.Sprintf("Mirror: %v\n", mirror))
	config.SetBool("remote.origin.mirror", true)

	return nil
}

func (repo Repo) updateRepository(repository git.Repository) (err error) {

	remote, err := repository.LookupRemote("origin")
	defer remote.Free()

	//refspecs := []string{"+refs/*:refs/*"}
	err = remote.Fetch([]string{}, nil, "")
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Update git mirror concurrent draft<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/libgit2/git2go"
)

// type Repos struct {
// 	repo []Repo
// }

type Repo struct {
	Url        string
	Path       string
	repository *git.Repository
}

var clone string = "./test"
var repo string = "https://gitlab.com/rollbrettler/go-playground.git"

func main() {

	// repos := []Repos{
	// 	{Url: "https://gitlab.com/rollbrettler/go-playground.git", Path: "./test"},
	// 	{Url: "https://github.com/rollbrettler/compress-videos.git", Path: "./test2"},
	// }
	//
	// for _, repo := range repos {
	// 	fmt.Printf("%s -> %s\n", repo.Url, repo.Path)
	// }

	os.Exit(0)

	var err error

	repo := Repo{Url: "https://gitlab.com/rollbrettler/go-playground.git", Path: "./test"}

	err = repo.openRepository()
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	err = repo.updateRepository()
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	log.Println("Sucessfully updated")
	os.Exit(0)
}

func (repo Repo) cloneRepository() (err error) {

	cloneOptions := &git.CloneOptions{
		Bare: true,
	}

	repo.repository, err = git.Clone(repo.Url, repo.Path, cloneOptions)
	if err != nil {
		log.Println(err)
		return
	}

	config, err := repo.repository.Config()
	if err != nil {
		log.Println(err)
		return
	}

	err = repo.changeToMirrorConfig(*config)
	if err != nil {
		log.Println(err)
	}

	err = repo.updateRepository()
	if err != nil {
		log.Println(err)
	}

	return nil
}

func (repo Repo) changeToMirrorConfig(config git.Config) (err error) {

	fetch, err := config.LookupString("remote.origin.fetch")
	if err != nil {
		return err
	}

	mirror, err := config.LookupBool("remote.origin.mirror")
	if err != nil {
		return err
	}

	log.Println(fmt.Sprintf("Refspec: %v\n", fetch))
	config.SetString("remote.origin.fetch", "+refs/*:refs/*")

	log.Println(fmt.Sprintf("Mirror: %v\n", mirror))
	config.SetBool("remote.origin.mirror", true)

	return nil
}

func (repo Repo) updateRepository() (err error) {

	remote, err := repo.repository.LookupRemote("origin")
	defer remote.Free()

	//refspecs := []string{"+refs/*:refs/*"}
	err = remote.Fetch([]string{}, nil, "")
	if err != nil {
		return err
	}

	return nil
}

func (repo Repo) openRepository() (err error) {
	_, err = ioutil.ReadDir(repo.Path)
	if err != nil {
		log.Println("Directory not pressent")

		err := repo.cloneRepository()
		if err != nil {
			log.Println(err)
			os.Exit(1)
		}
	}

	log.Println("Folder already cloned")

	repo.repository, err = git.OpenRepository(repo.Path)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	return err
}
<|endoftext|>


<commit_before>package users

import (
	"reflect"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestCreateUser(t *testing.T) {

	for _, testdb := range testdatabases {
		err := testdb.CreateUser("TestCreateUser_name", "TestCreateUser_email", "TestCreateUser_pass")
		require.Nil(t, err)

		err = testdb.CreateUser("TestCreateUser_name", "TestCreateUser_email2", "TestCreateUser_pass2")
		require.NotNil(t, err)

		err = testdb.CreateUser("TestCreateUser_name2", "TestCreateUser_email", "TestCreateUser_pass2")
		require.NotNil(t, err)
	}
}

func TestReadAllUsers(t *testing.T) {

	for _, testdb := range testdatabases {
		for i := 0; i < 5; i++ {
			_, err := CreateTestUser(testdb)
			require.Nil(t, err)
		}

		users, err := testdb.ReadAllUsers()
		require.Nil(t, err)
		require.NotNil(t, users)

		err = testdb.CreateUser("TestReadAllUsers", "TestReadAllUsers_email", "TestReadAllUsers_pass")
		require.Nil(t, err)

		users2, err := testdb.ReadAllUsers()
		assert.Nil(t, err, "got err from read all users %v", err)
		require.NotNil(t, users2, "Could not get all users, was nil")

		assert.Equal(t, 1, len(users2)-len(users), "not taking into account changes")
	}
}

func TestReadUserByName(t *testing.T) {

	for _, testdb := range testdatabases {
		// test failures on non existance
		usr, err := testdb.ReadUserByName("")
		assert.NotNil(t, err)
		assert.NotNil(t, usr)

		// setup for reading
		err = testdb.CreateUser("TestReadUserByName_name", "TestReadUserByName_email", "TestReadUserByName_pass")
		require.Nil(t, err)

		usr, err = testdb.ReadUserByName("TestReadUserByName_name")
		assert.NotNil(t, usr, "did not get a user by name")
		assert.Nil(t, err, "got an error when trying to get a user that should exist")
	}
}

func TestReadUserById(t *testing.T) {

	for _, testdb := range testdatabases {
		// test failures on non existance
		usr, err := testdb.ReadUserById(-1)
		assert.NotNil(t, err)

		// setup for reading
		err = testdb.CreateUser("ReadUserById_name", "ReadUserById_email", "ReadUserById_pass")
		assert.Nil(t, err)

		usr, err = testdb.ReadUserByName("ReadUserById_name")
		assert.NotNil(t, usr)
		assert.Nil(t, err)
	}
}

func TestUpdateUser(t *testing.T) {

	for _, testdb := range testdatabases {
		err := testdb.UpdateUser(nil)
		assert.Equal(t, err, InvalidPointerError, "Didn't catch nil")

		usr, err := CreateTestUser(testdb)
		require.Nil(t, err)

		usr.Name = "Hello"
		usr.Email = "hello@example.com"
		usr.Admin = true
		usr.UploadLimit_Items = 1
		usr.ProcessingLimit_S = 1
		usr.StorageLimit_Gb = 1

		err = testdb.UpdateUser(usr)
		require.Nil(t, err)

		usr2, err := testdb.ReadUserByName(usr.Name)

		if !reflect.DeepEqual(usr, usr2) {
			t.Errorf("The original and updated objects don't match orig: %v updated %v", usr, usr2)
		}
	}
}

func TestDeleteUser(t *testing.T) {

	for _, testdb := range testdatabases {
		usr, err := CreateTestUser(testdb)
		require.Nil(t, err)

		err = testdb.DeleteUser(usr.UserId)
		require.Nil(t, err)

		_, err = testdb.ReadUserById(usr.UserId)
		require.NotNil(t, err, "The user with ID %v should have errored out, but it did not", usr.UserId)

		err = testdb.DeleteUser(usr.UserId)
		require.Equal(t, err, ErrNothingToDelete, "Didn't catch try to delete deleted user")
	}
}

func TestReadUserDevice(t *testing.T) {

	for _, testdb := range testdatabases {
		user, err := CreateTestUser(testdb)
		require.Nil(t, err)

		dev, err := testdb.ReadUserOperatingDevice(user)
		require.Nil(t, err)

		assert.Equal(t, dev.UserId, user.UserId, "Incorrect device returned.")

		user.Admin = true
		err = testdb.UpdateUser(user)
		require.Nil(t, err)

		dev, err = testdb.ReadUserOperatingDevice(user)
		require.Nil(t, err)

		assert.True(t, dev.IsAdmin)
	}
}

func TestLogin(t *testing.T) {

	for _, testdb := range testdatabases {
		user, err := CreateTestUser(testdb)

		_, _, err = testdb.Login(user.Name, testPassword)
		assert.Nil(t, err)

		_, _, err = testdb.Login(user.Email, testPassword)
		assert.Nil(t, err)

		_, _, err = testdb.Login("", testPassword)
		assert.Equal(t, err, ErrInvalidUsername, "Wrong type returned %v", err)

		_, _, err = testdb.Login(user.Name, "")
		assert.NotNil(t, err, "Accepted blank password")
	}
}

func TestUpgradePassword(t *testing.T) {

	for _, testdb := range testdatabases {
		user, err := CreateTestUser(testdb)
		require.Nil(t, err)

		res := user.UpgradePassword(testPassword)
		assert.Equal(t, res, false, "Should not need to upgrade a password with the same scheme")

		user.PasswordHashScheme = ""
		res = user.UpgradePassword(testPassword)
		assert.Equal(t, res, true, "Should want to upgrade a password with an old has type")

		assert.NotEqual(t, "", user.PasswordHashScheme, "The has scheme was not updated")
	}
}

func TestRevertUneditableFields(t *testing.T) {
	// The original value we're trying to change
	orig := User{1, "Name", "Email", "Password", "passsalt", "hash", true, 1, 1, 1}

	// the one we're trying to submit
	blank := User{0, "", "", "", "", "", false, 0, 0, 0}

	// nobody's version
	nobody := blank
	// root's version of blank:
	root := User{1, "", "", "", "", "", false, 0, 0, 0}
	// User's version of blank
	user := User{1, "Name", "", "", "", "", true, 1, 1, 1}
	// all the rest shouldn't be able to do anything
	device := orig
	family := orig
	enabled := orig
	anybody := orig

	tmpu := blank
	tmpu.RevertUneditableFields(orig, NOBODY)
	assert.Equal(t, tmpu, nobody, "Conversion as nobody didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, ROOT)
	assert.Equal(t, tmpu, root, "Conversion as root didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, USER)
	assert.Equal(t, tmpu, user, "Conversion as user didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, DEVICE)
	assert.Equal(t, tmpu, device, "Conversion as device didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, FAMILY)
	assert.Equal(t, tmpu, family, "Conversion as family didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, ENABLED)
	assert.Equal(t, tmpu, enabled, "Conversion as enabled didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, ANYBODY)
	assert.Equal(t, tmpu, anybody, "Conversion as anybody didn't work got %v, expected %v", tmpu, root)
}
<commit_msg>Fixed test that was using undocumented output<commit_after>package users

import (
	"reflect"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestCreateUser(t *testing.T) {

	for _, testdb := range testdatabases {
		err := testdb.CreateUser("TestCreateUser_name", "TestCreateUser_email", "TestCreateUser_pass")
		require.Nil(t, err)

		err = testdb.CreateUser("TestCreateUser_name", "TestCreateUser_email2", "TestCreateUser_pass2")
		require.NotNil(t, err)

		err = testdb.CreateUser("TestCreateUser_name2", "TestCreateUser_email", "TestCreateUser_pass2")
		require.NotNil(t, err)
	}
}

func TestReadAllUsers(t *testing.T) {

	for _, testdb := range testdatabases {
		for i := 0; i < 5; i++ {
			_, err := CreateTestUser(testdb)
			require.Nil(t, err)
		}

		users, err := testdb.ReadAllUsers()
		require.Nil(t, err)
		require.NotNil(t, users)

		err = testdb.CreateUser("TestReadAllUsers", "TestReadAllUsers_email", "TestReadAllUsers_pass")
		require.Nil(t, err)

		users2, err := testdb.ReadAllUsers()
		assert.Nil(t, err, "got err from read all users %v", err)
		require.NotNil(t, users2, "Could not get all users, was nil")

		assert.Equal(t, 1, len(users2)-len(users), "not taking into account changes")
	}
}

func TestReadUserByName(t *testing.T) {

	for _, testdb := range testdatabases {
		// test failures on non existance
		usr, err := testdb.ReadUserByName("")
		assert.NotNil(t, err)

		// setup for reading
		err = testdb.CreateUser("TestReadUserByName_name", "TestReadUserByName_email", "TestReadUserByName_pass")
		require.Nil(t, err)

		usr, err = testdb.ReadUserByName("TestReadUserByName_name")
		assert.NotNil(t, usr, "did not get a user by name")
		assert.Nil(t, err, "got an error when trying to get a user that should exist")
	}
}

func TestReadUserById(t *testing.T) {

	for _, testdb := range testdatabases {
		// test failures on non existance
		usr, err := testdb.ReadUserById(-1)
		assert.NotNil(t, err)

		// setup for reading
		err = testdb.CreateUser("ReadUserById_name", "ReadUserById_email", "ReadUserById_pass")
		assert.Nil(t, err)

		usr, err = testdb.ReadUserByName("ReadUserById_name")
		assert.NotNil(t, usr)
		assert.Nil(t, err)
	}
}

func TestUpdateUser(t *testing.T) {

	for _, testdb := range testdatabases {
		err := testdb.UpdateUser(nil)
		assert.Equal(t, err, InvalidPointerError, "Didn't catch nil")

		usr, err := CreateTestUser(testdb)
		require.Nil(t, err)

		usr.Name = "Hello"
		usr.Email = "hello@example.com"
		usr.Admin = true
		usr.UploadLimit_Items = 1
		usr.ProcessingLimit_S = 1
		usr.StorageLimit_Gb = 1

		err = testdb.UpdateUser(usr)
		require.Nil(t, err)

		usr2, err := testdb.ReadUserByName(usr.Name)

		if !reflect.DeepEqual(usr, usr2) {
			t.Errorf("The original and updated objects don't match orig: %v updated %v", usr, usr2)
		}
	}
}

func TestDeleteUser(t *testing.T) {

	for _, testdb := range testdatabases {
		usr, err := CreateTestUser(testdb)
		require.Nil(t, err)

		err = testdb.DeleteUser(usr.UserId)
		require.Nil(t, err)

		_, err = testdb.ReadUserById(usr.UserId)
		require.NotNil(t, err, "The user with ID %v should have errored out, but it did not", usr.UserId)

		err = testdb.DeleteUser(usr.UserId)
		require.Equal(t, err, ErrNothingToDelete, "Didn't catch try to delete deleted user")
	}
}

func TestReadUserDevice(t *testing.T) {

	for _, testdb := range testdatabases {
		user, err := CreateTestUser(testdb)
		require.Nil(t, err)

		dev, err := testdb.ReadUserOperatingDevice(user)
		require.Nil(t, err)

		assert.Equal(t, dev.UserId, user.UserId, "Incorrect device returned.")

		user.Admin = true
		err = testdb.UpdateUser(user)
		require.Nil(t, err)

		dev, err = testdb.ReadUserOperatingDevice(user)
		require.Nil(t, err)

		assert.True(t, dev.IsAdmin)
	}
}

func TestLogin(t *testing.T) {

	for _, testdb := range testdatabases {
		user, err := CreateTestUser(testdb)

		_, _, err = testdb.Login(user.Name, testPassword)
		assert.Nil(t, err)

		_, _, err = testdb.Login(user.Email, testPassword)
		assert.Nil(t, err)

		_, _, err = testdb.Login("", testPassword)
		assert.Equal(t, err, ErrInvalidUsername, "Wrong type returned %v", err)

		_, _, err = testdb.Login(user.Name, "")
		assert.NotNil(t, err, "Accepted blank password")
	}
}

func TestUpgradePassword(t *testing.T) {

	for _, testdb := range testdatabases {
		user, err := CreateTestUser(testdb)
		require.Nil(t, err)

		res := user.UpgradePassword(testPassword)
		assert.Equal(t, res, false, "Should not need to upgrade a password with the same scheme")

		user.PasswordHashScheme = ""
		res = user.UpgradePassword(testPassword)
		assert.Equal(t, res, true, "Should want to upgrade a password with an old has type")

		assert.NotEqual(t, "", user.PasswordHashScheme, "The has scheme was not updated")
	}
}

func TestRevertUneditableFields(t *testing.T) {
	// The original value we're trying to change
	orig := User{1, "Name", "Email", "Password", "passsalt", "hash", true, 1, 1, 1}

	// the one we're trying to submit
	blank := User{0, "", "", "", "", "", false, 0, 0, 0}

	// nobody's version
	nobody := blank
	// root's version of blank:
	root := User{1, "", "", "", "", "", false, 0, 0, 0}
	// User's version of blank
	user := User{1, "Name", "", "", "", "", true, 1, 1, 1}
	// all the rest shouldn't be able to do anything
	device := orig
	family := orig
	enabled := orig
	anybody := orig

	tmpu := blank
	tmpu.RevertUneditableFields(orig, NOBODY)
	assert.Equal(t, tmpu, nobody, "Conversion as nobody didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, ROOT)
	assert.Equal(t, tmpu, root, "Conversion as root didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, USER)
	assert.Equal(t, tmpu, user, "Conversion as user didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, DEVICE)
	assert.Equal(t, tmpu, device, "Conversion as device didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, FAMILY)
	assert.Equal(t, tmpu, family, "Conversion as family didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, ENABLED)
	assert.Equal(t, tmpu, enabled, "Conversion as enabled didn't work got %v, expected %v", tmpu, root)

	tmpu = blank
	tmpu.RevertUneditableFields(orig, ANYBODY)
	assert.Equal(t, tmpu, anybody, "Conversion as anybody didn't work got %v, expected %v", tmpu, root)
}
<|endoftext|>


<commit_before>package packet
<commit_msg>Add mqtt/packet/publish.go<commit_after>package packet

import "github.com/yosssi/gmq/mqtt"

// publish represents a PUBLISH Packet.
type publish struct {
	base
	// dup is the DUP flag of the fixed header.
	dup bool
	// qos is the QoS of the fixed header.
	qos byte
	// retain is the Retain of the fixed header.
	retain bool
	// topicName is the Topic Name of the varible header.
	topicName []byte
	// packetID is the Packet Identifier of the variable header.
	packetID uint16
	// message is the Application Message of the payload.
	message []byte
}

// setFixedHeader sets the fixed header to the Packet.
func (p *publish) setFixedHeader() {
	// Define the first byte of the fixed header.
	b := TypePUBLISH << 4

	// Set 1 to the Bit 3 if the DUP flag is true.
	if p.dup {
		b |= 0x80
	}

	// Set the value of the Will QoS to the Bit 2 and 1.
	b |= p.qos << 1

	// Set 1 to the Bit 0 if the Retain is true.
	if p.retain {
		b |= 0x01
	}

	// Append the first byte to the fixed header.
	p.fixedHeader = append(p.fixedHeader, b)

	// Append the Remaining Length to the fixed header.
	p.appendRemainingLength()
}

// setVariableHeader sets the variable header to the Packet.
func (p *publish) setVariableHeader() {
	// Append the Topic Name to the variable header.
	p.variableHeader = appendLenStr(p.variableHeader, p.topicName)

	if p.qos != mqtt.QoS0 {
		// Append the Packet Identifier to the variable header.
		p.variableHeader = append(p.variableHeader, encodeUint16(p.packetID)...)
	}
}

// setPayload sets the payload to the Packet.
func (p *publish) setPayload() {
	p.payload = p.message
}

// NewPUBLISH creates and returns a PUBLISH Packet.
func NewPUBLISH(opts *PUBLISHOptions) (Packet, error) {
	// Initialize the options.
	if opts == nil {
		opts = &PUBLISHOptions{}
	}

	// Validate the options.
	if err := opts.validate(); err != nil {
		return nil, err
	}

	// Create a PUBLISH Packet.
	p := &publish{
		dup:       opts.DUP,
		qos:       opts.QoS,
		retain:    opts.Retain,
		topicName: opts.TopicName,
		packetID:  opts.PacketID,
		message:   opts.Message,
	}

	// Set the variable header to the Packet.
	p.setVariableHeader()

	// Set the payload to the Packet.
	p.setPayload()

	// Set the Fixed header to the Packet.
	p.setFixedHeader()

	// Return the Packet.
	return p, nil
}
<|endoftext|>


<commit_before>package pfstest

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net"
	"os"
	"strings"
	"sync/atomic"
	"testing"

	"golang.org/x/net/context"

	"google.golang.org/grpc"

	"github.com/facebookgo/freeport"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pfs/drive/btrfs"
	"github.com/pachyderm/pachyderm/src/pfs/route"
	"github.com/pachyderm/pachyderm/src/pfs/server"
	"github.com/pachyderm/pachyderm/src/pkg/executil"
	"github.com/pachyderm/pachyderm/src/pkg/protoutil"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
)

const (
	// TODO(pedge): large numbers of shards takes forever because
	// we are doing tons of btrfs operations on init, is there anything
	// we can do about that?
	testDefaultNumShards = 16
)

var (
	counter int32
)

func init() {
	executil.SetDebug(true)
}

func TestBtrfs(t *testing.T) {
	driver := btrfs.NewDriver(getBtrfsRootDir(t))
	numShards := testDefaultNumShards
	runTest(t, driver, numShards, testSimple)
}

func getBtrfsRootDir(t *testing.T) string {
	// TODO(pedge)
	rootDir := os.Getenv("PFS_BTRFS_ROOT")
	if rootDir == "" {
		t.Fatal("PFS_BTRFS_ROOT not set")
	}
	return rootDir
}

func testSimple(t *testing.T, apiClient pfs.ApiClient) {
	repositoryName := testRepositoryName()
	testSize := 10000

	err := initRepository(apiClient, repositoryName)
	require.NoError(t, err)

	getCommitInfoResponse, err := getCommitInfo(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Nil(t, getCommitInfoResponse.CommitInfo.ParentCommit)

	branchResponse, err := branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, branchResponse)
	newCommitID := branchResponse.Commit.Id

	getCommitInfoResponse, err = getCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	err = makeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = makeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)

	for i := 0; i < testSize; i++ {
		err = putFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
		require.NoError(t, err)
		err = putFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
		require.NoError(t, err)
	}

	err = commit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	getCommitInfoResponse, err = getCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	for i := 0; i < testSize; i++ {
		readStringer, err := getFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i))
		require.NoError(t, err)
		require.Equal(t, fmt.Sprintf("hello%d", i), readStringer.String())
		readStringer, err = getFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i))
		require.NoError(t, err)
		require.Equal(t, fmt.Sprintf("hello%d", i), readStringer.String())
	}

	listFilesResponse, err := listFiles(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))
	listFilesResponse, err = listFiles(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))
	listFilesResponse, err = listFiles(apiClient, repositoryName, newCommitID, "a/b", 0, 2)
	require.NoError(t, err)
	require.Equal(t, testSize/2, len(listFilesResponse.FileInfo))
	listFilesResponse, err = listFiles(apiClient, repositoryName, newCommitID, "a/c", 1, 2)
	require.NoError(t, err)
	require.Equal(t, testSize/2, len(listFilesResponse.FileInfo))
}

func testRepositoryName() string {
	return fmt.Sprintf("test-%d", atomic.AddInt32(&counter, 1))
}

func initRepository(apiClient pfs.ApiClient, repositoryName string) error {
	_, err := apiClient.InitRepository(
		context.Background(),
		&pfs.InitRepositoryRequest{
			Repository: &pfs.Repository{
				Name: repositoryName,
			},
		},
	)
	return err
}

func branch(apiClient pfs.ApiClient, repositoryName string, commitID string) (*pfs.BranchResponse, error) {
	return apiClient.Branch(
		context.Background(),
		&pfs.BranchRequest{
			Commit: &pfs.Commit{
				Repository: &pfs.Repository{
					Name: repositoryName,
				},
				Id: commitID,
			},
		},
	)
}

func makeDirectory(apiClient pfs.ApiClient, repositoryName string, commitID string, path string) error {
	_, err := apiClient.MakeDirectory(
		context.Background(),
		&pfs.MakeDirectoryRequest{
			Path: &pfs.Path{
				Commit: &pfs.Commit{
					Repository: &pfs.Repository{
						Name: repositoryName,
					},
					Id: commitID,
				},
				Path: path,
			},
		},
	)
	return err
}

func putFile(apiClient pfs.ApiClient, repositoryName string, commitID string, path string, reader io.Reader) error {
	value, err := ioutil.ReadAll(reader)
	if err != nil {
		return err
	}
	_, err = apiClient.PutFile(
		context.Background(),
		&pfs.PutFileRequest{
			Path: &pfs.Path{
				Commit: &pfs.Commit{
					Repository: &pfs.Repository{
						Name: repositoryName,
					},
					Id: commitID,
				},
				Path: path,
			},
			Value: value,
		},
	)
	return err
}

type readStringer interface {
	io.Reader
	fmt.Stringer
}

func getFile(apiClient pfs.ApiClient, repositoryName string, commitID string, path string) (readStringer, error) {
	apiGetFileClient, err := apiClient.GetFile(
		context.Background(),
		&pfs.GetFileRequest{
			Path: &pfs.Path{
				Commit: &pfs.Commit{
					Repository: &pfs.Repository{
						Name: repositoryName,
					},
					Id: commitID,
				},
				Path: path,
			},
		},
	)
	if err != nil {
		return nil, err
	}
	buffer := bytes.NewBuffer(nil)
	if err := protoutil.WriteFromStreamingBytesClient(apiGetFileClient, buffer); err != nil {
		return nil, err
	}
	return buffer, nil
}

func listFiles(apiClient pfs.ApiClient, repositoryName string, commitID string, path string, shardNum int, shardModulo int) (*pfs.ListFilesResponse, error) {
	return apiClient.ListFiles(
		context.Background(),
		&pfs.ListFilesRequest{
			Path: &pfs.Path{
				Commit: &pfs.Commit{
					Repository: &pfs.Repository{
						Name: repositoryName,
					},
					Id: commitID,
				},
				Path: path,
			},
			Shard: &pfs.Shard{
				Number: uint64(shardNum),
				Modulo: uint64(shardModulo),
			},
		},
	)
}

func commit(apiClient pfs.ApiClient, repositoryName string, commitID string) error {
	_, err := apiClient.Commit(
		context.Background(),
		&pfs.CommitRequest{
			Commit: &pfs.Commit{
				Repository: &pfs.Repository{
					Name: repositoryName,
				},
				Id: commitID,
			},
		},
	)
	return err
}

func getCommitInfo(apiClient pfs.ApiClient, repositoryName string, commitID string) (*pfs.GetCommitInfoResponse, error) {
	return apiClient.GetCommitInfo(
		context.Background(),
		&pfs.GetCommitInfoRequest{
			Commit: &pfs.Commit{
				Repository: &pfs.Repository{
					Name: repositoryName,
				},
				Id: commitID,
			},
		},
	)
}

func runTest(
	t *testing.T,
	driver drive.Driver,
	numShards int,
	f func(t *testing.T, apiClient pfs.ApiClient),
) {
	runGrpcTest(
		t,
		func(s *grpc.Server, address string) {
			combinedAPIServer :=
				server.NewCombinedAPIServer(
					route.NewSharder(
						numShards,
					),
					route.NewRouter(
						route.NewSingleAddresser(
							address,
							numShards,
						),
						route.NewDialer(),
						address,
					),
					driver,
				)
			pfs.RegisterApiServer(s, combinedAPIServer)
			pfs.RegisterInternalApiServer(s, combinedAPIServer)
		},
		func(t *testing.T, clientConn *grpc.ClientConn) {
			f(
				t,
				pfs.NewApiClient(
					clientConn,
				),
			)
		},
	)
}

func runGrpcTest(
	t *testing.T,
	registerFunc func(*grpc.Server, string),
	testFunc func(*testing.T, *grpc.ClientConn),
) {
	grpcSuite := &grpcSuite{
		registerFunc: registerFunc,
		testFunc:     testFunc,
	}
	suite.Run(t, grpcSuite)
}

type grpcSuite struct {
	suite.Suite
	registerFunc func(*grpc.Server, string)
	testFunc     func(*testing.T, *grpc.ClientConn)
	clientConn   *grpc.ClientConn
	server       *grpc.Server
	address      string
	errC         chan error
}

func (g *grpcSuite) SetupSuite() {
	port, err := freeport.Get()
	require.NoError(g.T(), err)
	g.address = fmt.Sprintf("0.0.0.0:%d", port)
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	require.NoError(g.T(), err)
	g.server = grpc.NewServer(grpc.MaxConcurrentStreams(math.MaxUint32))
	g.registerFunc(g.server, g.address)
	g.errC = make(chan error, 1)
	go func() {
		g.errC <- g.server.Serve(listener)
		close(g.errC)
	}()
	clientConn, err := grpc.Dial(g.address)
	if err != nil {
		g.server.Stop()
		<-g.errC
		require.NoError(g.T(), err)
	}
	g.clientConn = clientConn
}

func (g *grpcSuite) TearDownSuite() {
	g.server.Stop()
	<-g.errC
	_ = g.clientConn.Close()
}

func (g *grpcSuite) TestSuite() {
	g.testFunc(g.T(), g.clientConn)
}
<commit_msg>prepare for multi server pfs testing<commit_after>package pfstest

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net"
	"os"
	"strings"
	"sync/atomic"
	"testing"

	"golang.org/x/net/context"

	"google.golang.org/grpc"

	"github.com/facebookgo/freeport"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pfs/drive/btrfs"
	"github.com/pachyderm/pachyderm/src/pfs/route"
	"github.com/pachyderm/pachyderm/src/pfs/server"
	"github.com/pachyderm/pachyderm/src/pkg/executil"
	"github.com/pachyderm/pachyderm/src/pkg/protoutil"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
)

const (
	// TODO(pedge): large numbers of shards takes forever because
	// we are doing tons of btrfs operations on init, is there anything
	// we can do about that?
	testShardsPerServer = 4
	testNumServers      = 1
	testSize            = 10000
)

var (
	counter int32
)

func init() {
	executil.SetDebug(true)
}

func TestBtrfs(t *testing.T) {
	driver := btrfs.NewDriver(getBtrfsRootDir(t))
	runTest(t, driver, testSimple)
}

func getBtrfsRootDir(t *testing.T) string {
	// TODO(pedge)
	rootDir := os.Getenv("PFS_BTRFS_ROOT")
	if rootDir == "" {
		t.Fatal("PFS_BTRFS_ROOT not set")
	}
	return rootDir
}

func testSimple(t *testing.T, apiClient pfs.ApiClient) {
	repositoryName := testRepositoryName()

	err := initRepository(apiClient, repositoryName)
	require.NoError(t, err)

	getCommitInfoResponse, err := getCommitInfo(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Nil(t, getCommitInfoResponse.CommitInfo.ParentCommit)

	branchResponse, err := branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, branchResponse)
	newCommitID := branchResponse.Commit.Id

	getCommitInfoResponse, err = getCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	err = makeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = makeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)

	for i := 0; i < testSize; i++ {
		err = putFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
		require.NoError(t, err)
		err = putFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
		require.NoError(t, err)
	}

	err = commit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	getCommitInfoResponse, err = getCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	for i := 0; i < testSize; i++ {
		readStringer, err := getFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i))
		require.NoError(t, err)
		require.Equal(t, fmt.Sprintf("hello%d", i), readStringer.String())
		readStringer, err = getFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i))
		require.NoError(t, err)
		require.Equal(t, fmt.Sprintf("hello%d", i), readStringer.String())
	}

	listFilesResponse, err := listFiles(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))
	listFilesResponse, err = listFiles(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))
	listFilesResponse, err = listFiles(apiClient, repositoryName, newCommitID, "a/b", 0, 2)
	require.NoError(t, err)
	require.Equal(t, testSize/2, len(listFilesResponse.FileInfo))
	listFilesResponse, err = listFiles(apiClient, repositoryName, newCommitID, "a/c", 1, 2)
	require.NoError(t, err)
	require.Equal(t, testSize/2, len(listFilesResponse.FileInfo))
}

func testRepositoryName() string {
	return fmt.Sprintf("test-%d", atomic.AddInt32(&counter, 1))
}

func initRepository(apiClient pfs.ApiClient, repositoryName string) error {
	_, err := apiClient.InitRepository(
		context.Background(),
		&pfs.InitRepositoryRequest{
			Repository: &pfs.Repository{
				Name: repositoryName,
			},
		},
	)
	return err
}

func branch(apiClient pfs.ApiClient, repositoryName string, commitID string) (*pfs.BranchResponse, error) {
	return apiClient.Branch(
		context.Background(),
		&pfs.BranchRequest{
			Commit: &pfs.Commit{
				Repository: &pfs.Repository{
					Name: repositoryName,
				},
				Id: commitID,
			},
		},
	)
}

func makeDirectory(apiClient pfs.ApiClient, repositoryName string, commitID string, path string) error {
	_, err := apiClient.MakeDirectory(
		context.Background(),
		&pfs.MakeDirectoryRequest{
			Path: &pfs.Path{
				Commit: &pfs.Commit{
					Repository: &pfs.Repository{
						Name: repositoryName,
					},
					Id: commitID,
				},
				Path: path,
			},
		},
	)
	return err
}

func putFile(apiClient pfs.ApiClient, repositoryName string, commitID string, path string, reader io.Reader) error {
	value, err := ioutil.ReadAll(reader)
	if err != nil {
		return err
	}
	_, err = apiClient.PutFile(
		context.Background(),
		&pfs.PutFileRequest{
			Path: &pfs.Path{
				Commit: &pfs.Commit{
					Repository: &pfs.Repository{
						Name: repositoryName,
					},
					Id: commitID,
				},
				Path: path,
			},
			Value: value,
		},
	)
	return err
}

type readStringer interface {
	io.Reader
	fmt.Stringer
}

func getFile(apiClient pfs.ApiClient, repositoryName string, commitID string, path string) (readStringer, error) {
	apiGetFileClient, err := apiClient.GetFile(
		context.Background(),
		&pfs.GetFileRequest{
			Path: &pfs.Path{
				Commit: &pfs.Commit{
					Repository: &pfs.Repository{
						Name: repositoryName,
					},
					Id: commitID,
				},
				Path: path,
			},
		},
	)
	if err != nil {
		return nil, err
	}
	buffer := bytes.NewBuffer(nil)
	if err := protoutil.WriteFromStreamingBytesClient(apiGetFileClient, buffer); err != nil {
		return nil, err
	}
	return buffer, nil
}

func listFiles(apiClient pfs.ApiClient, repositoryName string, commitID string, path string, shardNum int, shardModulo int) (*pfs.ListFilesResponse, error) {
	return apiClient.ListFiles(
		context.Background(),
		&pfs.ListFilesRequest{
			Path: &pfs.Path{
				Commit: &pfs.Commit{
					Repository: &pfs.Repository{
						Name: repositoryName,
					},
					Id: commitID,
				},
				Path: path,
			},
			Shard: &pfs.Shard{
				Number: uint64(shardNum),
				Modulo: uint64(shardModulo),
			},
		},
	)
}

func commit(apiClient pfs.ApiClient, repositoryName string, commitID string) error {
	_, err := apiClient.Commit(
		context.Background(),
		&pfs.CommitRequest{
			Commit: &pfs.Commit{
				Repository: &pfs.Repository{
					Name: repositoryName,
				},
				Id: commitID,
			},
		},
	)
	return err
}

func getCommitInfo(apiClient pfs.ApiClient, repositoryName string, commitID string) (*pfs.GetCommitInfoResponse, error) {
	return apiClient.GetCommitInfo(
		context.Background(),
		&pfs.GetCommitInfoRequest{
			Commit: &pfs.Commit{
				Repository: &pfs.Repository{
					Name: repositoryName,
				},
				Id: commitID,
			},
		},
	)
}

func runTest(
	t *testing.T,
	driver drive.Driver,
	f func(t *testing.T, apiClient pfs.ApiClient),
) {
	runGrpcTest(
		t,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			for address, s := range servers {
				combinedAPIServer := server.NewCombinedAPIServer(
					route.NewSharder(
						testShardsPerServer*testNumServers,
					),
					route.NewRouter(
						route.NewSingleAddresser(
							address,
							testShardsPerServer,
						),
						route.NewDialer(),
						address,
					),
					driver,
				)
				pfs.RegisterApiServer(s, combinedAPIServer)
				pfs.RegisterInternalApiServer(s, combinedAPIServer)
			}
		},
		func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			f(
				t,
				pfs.NewApiClient(
					clientConn,
				),
			)
		},
	)
}

func runGrpcTest(
	t *testing.T,
	numServers int,
	registerFunc func(map[string]*grpc.Server),
	testFunc func(*testing.T, map[string]*grpc.ClientConn),
) {
	grpcSuite := &grpcSuite{
		numServers:   numServers,
		registerFunc: registerFunc,
		testFunc:     testFunc,
	}
	suite.Run(t, grpcSuite)
}

type grpcSuite struct {
	suite.Suite
	numServers   int
	registerFunc func(map[string]*grpc.Server)
	testFunc     func(*testing.T, map[string]*grpc.ClientConn)
	clientConns  map[string]*grpc.ClientConn
	servers      map[string]*grpc.Server
	errC         chan error
	done         chan bool
}

func (g *grpcSuite) SetupSuite() {
	g.servers = make(map[string]*grpc.Server)
	listeners := make(map[string]net.Listener)
	for i := 0; i < g.numServers; i++ {
		port, err := freeport.Get()
		require.NoError(g.T(), err)
		address := fmt.Sprintf("0.0.0.0:%d", port)
		g.servers[address] = grpc.NewServer(grpc.MaxConcurrentStreams(math.MaxUint32))
		listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
		require.NoError(g.T(), err)
		listeners[address] = listener
	}
	g.registerFunc(g.servers)
	g.errC = make(chan error, g.numServers)
	for address, server := range g.servers {
		address := address
		server := server
		go func() {
			g.errC <- server.Serve(listeners[address])
		}()
	}
	g.done = make(chan bool, 1)
	go func() {
		for j := 0; j < g.numServers; j++ {
			<-g.errC
		}
		g.done <- true
	}()
	g.clientConns = make(map[string]*grpc.ClientConn)
	for address := range g.servers {
		clientConn, err := grpc.Dial(address)
		if err != nil {
			g.TearDownSuite()
			require.NoError(g.T(), err)
		}
		g.clientConns[address] = clientConn
	}
}

func (g *grpcSuite) TearDownSuite() {
	for _, server := range g.servers {
		server.Stop()
	}
	<-g.done
	for _, clientConn := range g.clientConns {
		_ = clientConn.Close()
	}
}

func (g *grpcSuite) TestSuite() {
	g.testFunc(g.T(), g.clientConns)
}
<|endoftext|>


<commit_before>package store

import (
	"encoding/json"
	"net"
	"net/url"
	"strings"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libkv"
	kv "github.com/docker/libkv/store"
	"github.com/docker/libkv/store/consul"
	"github.com/docker/libkv/store/etcd"
	"github.com/k0kubun/pp"
	"github.com/luizbafilho/fusis/config"
	"github.com/luizbafilho/fusis/types"
	"github.com/pkg/errors"
)

func init() {
	registryStores()
}

func registryStores() {
	libkv.AddStore(kv.CONSUL, consul.New)
	libkv.AddStore(kv.ETCD, etcd.New)
}

type Store interface {
	GetServices() ([]types.Service, error)
	// GetService(serviceId string) (types.Service, error)
	AddService(svc *types.Service) error
	DeleteService(svc *types.Service) error
	SubscribeServices(ch chan []types.Service)
	WatchServices()

	GetDestinations() ([]types.Destination, error)
	AddDestination(svc *types.Service, dst *types.Destination) error
	DeleteDestination(svc *types.Service, dst *types.Destination) error
	SubscribeDestinations(ch chan []types.Destination)
	WatchDestinations()

	AddCheck(check types.CheckSpec) error
	DeleteCheck(check types.CheckSpec) error
	SubscribeChecks(ch chan []types.CheckSpec)
	WatchChecks()

	GetKV() kv.Store
}

var (
	ErrUnsupportedStore = errors.New("[store] Unsupported store")
)

type FusisStore struct {
	sync.Mutex
	kv kv.Store

	prefix string

	servicesChannels    []chan []types.Service
	destinationChannels []chan []types.Destination
	checksChannels      []chan []types.CheckSpec
}

func New(config *config.BalancerConfig) (Store, error) {
	u, err := url.Parse(config.StoreAddress)
	if err != nil {
		return nil, errors.Wrap(err, "error paring store address")
	}

	scheme := u.Scheme
	if scheme != "consul" && scheme != "etcd" {
		return nil, ErrUnsupportedStore
	}

	//Validating open connection
	_, err = net.Dial("tcp", u.Host)
	if err != nil {
		return nil, errors.Wrap(err, "Store connection failed. Make sure your store is up and running.")
	}

	pp.Println(scheme, u.Host)
	kv, err := libkv.NewStore(
		kv.Backend(scheme),
		[]string{u.Host},
		nil,
	)
	if err != nil {
		kv.Close()
		return nil, errors.Wrap(err, "Cannot create store consul")
	}

	svcsChs := []chan []types.Service{}
	dstsChs := []chan []types.Destination{}
	checksChs := []chan []types.CheckSpec{}

	fusisStore := &FusisStore{
		kv:                  kv,
		prefix:              config.StorePrefix,
		servicesChannels:    svcsChs,
		destinationChannels: dstsChs,
		checksChannels:      checksChs,
	}

	go fusisStore.WatchServices()
	go fusisStore.WatchDestinations()
	go fusisStore.WatchChecks()

	return fusisStore, nil
}

func (s *FusisStore) GetKV() kv.Store {
	return s.kv
}

// func (s *FusisStore) GetService(serviceId string) (types.Service, err) {
// }

func (s *FusisStore) GetServices() ([]types.Service, error) {
	svcs := []types.Service{}
	entries, err := s.kv.List(s.key("services"))
	if err != nil {
		if err == kv.ErrKeyNotFound {
			return svcs, nil
		}

		return nil, err
	}

	for _, pair := range entries {
		svc := types.Service{}
		if err := json.Unmarshal(pair.Value, &svc); err != nil {
			log.Error("[store] ", err)
		}

		svcs = append(svcs, svc)
	}

	return svcs, nil
}

func (s *FusisStore) GetDestinations() ([]types.Destination, error) {
	dsts := []types.Destination{}
	entries, err := s.kv.List("fusis/destinations")
	if err != nil {
		return nil, err
	}

	for _, pair := range entries {
		dst := types.Destination{}
		if err := json.Unmarshal(pair.Value, &dst); err != nil {
			log.Error("[store] ", err)
		}

		dsts = append(dsts, dst)
	}

	return dsts, nil
}

func (s *FusisStore) AddService(svc *types.Service) error {
	key := s.key("services", svc.GetId(), "config")

	value, err := json.Marshal(svc)
	if err != nil {
		return errors.Wrapf(err, "error marshaling service: %v", svc)
	}

	err = s.kv.Put(key, value, nil)
	if err != nil {
		return errors.Wrapf(err, "error sending service to store: %v", svc)
	}

	return nil
}

func (s *FusisStore) DeleteService(svc *types.Service) error {
	key := s.key("services", svc.GetId())

	err := s.kv.DeleteTree(key)
	if err != nil {
		return errors.Wrapf(err, "error trying to delete service: %v", svc)
	}

	return nil
}

func (s *FusisStore) SubscribeServices(updateCh chan []types.Service) {
	s.Lock()
	defer s.Unlock()
	s.servicesChannels = append(s.servicesChannels, updateCh)
}

func (s *FusisStore) WatchServices() {
	svcs := []types.Service{}

	stopCh := make(<-chan struct{})
	events, err := s.kv.WatchTree(s.key("services"), stopCh)
	if err != nil {
		log.Error("[store] ", err)
	}

	for {
		select {
		case entries := <-events:
			log.Debug("[store] Services received")

			for _, pair := range entries {
				svc := types.Service{}
				if err := json.Unmarshal(pair.Value, &svc); err != nil {
					log.Error("[store] ", err)
				}
				log.Debugf("[store] Got sevice: %#v ", svc)

				svcs = append(svcs, svc)
			}

			s.Lock()
			for _, ch := range s.servicesChannels {
				log.Debug("[store] Sending message to service ch")
				ch <- svcs
			}
			s.Unlock()

			//Cleaning up services slice
			svcs = []types.Service{}
		}
	}
}

func (s *FusisStore) AddDestination(svc *types.Service, dst *types.Destination) error {
	key := s.key("destinations", svc.GetId(), dst.GetId())

	value, err := json.Marshal(dst)
	if err != nil {
		return errors.Wrapf(err, "error marshaling destination: %v", dst)
	}

	err = s.kv.Put(key, value, nil)
	if err != nil {
		return errors.Wrapf(err, "error sending destination to store: %v", dst)
	}
	log.Debugf("[store] Added destination: %s with key: %s", value, key)

	return nil
}

func (s *FusisStore) DeleteDestination(svc *types.Service, dst *types.Destination) error {
	key := s.key("destinations", svc.GetId(), dst.GetId())
	// key := fmt.Sprintf("fusis/destinations/%s/%s", svc.GetId(), dst.GetId())

	err := s.kv.DeleteTree(key)
	if err != nil {
		return errors.Wrapf(err, "error trying to delete destination: %v", dst)
	}
	log.Debugf("[store] Deleted destination: %s", key)

	return nil
}

func (s *FusisStore) SubscribeDestinations(updateCh chan []types.Destination) {
	s.Lock()
	defer s.Unlock()
	s.destinationChannels = append(s.destinationChannels, updateCh)
}

func (s *FusisStore) WatchDestinations() {
	dsts := []types.Destination{}

	stopCh := make(<-chan struct{})
	events, err := s.kv.WatchTree(s.key("destinations"), stopCh)
	if err != nil {
		errors.Wrap(err, "failed watching fusis/destinations")
	}

	for {
		select {
		case entries := <-events:
			log.Debug("[store] Destinations received")

			for _, pair := range entries {
				dst := types.Destination{}
				if err := json.Unmarshal(pair.Value, &dst); err != nil {
					errors.Wrap(err, "failed unmarshall of destinations")
				}
				log.Debugf("[store] Got destination: %#v", dst)

				dsts = append(dsts, dst)
			}

			s.Lock()
			for _, ch := range s.destinationChannels {
				ch <- dsts
			}
			s.Unlock()

			//Cleaning up destinations slice
			dsts = []types.Destination{}
		}
	}
}

func (s *FusisStore) SubscribeChecks(updateCh chan []types.CheckSpec) {
	s.Lock()
	defer s.Unlock()
	s.checksChannels = append(s.checksChannels, updateCh)
}

func (s *FusisStore) AddCheck(spec types.CheckSpec) error {
	key := s.key("checks", spec.ServiceID)

	value, err := json.Marshal(spec)
	if err != nil {
		return errors.Wrapf(err, "error marshaling CheckSpec: %#v", spec)
	}

	err = s.kv.Put(key, value, nil)
	if err != nil {
		return errors.Wrapf(err, "error sending CheckSpec to store: %#v", spec)
	}

	return nil
}

func (s *FusisStore) DeleteCheck(spec types.CheckSpec) error {
	key := s.key("checks", spec.ServiceID)

	err := s.kv.DeleteTree(key)
	if err != nil {
		return errors.Wrapf(err, "error trying to delete check: %#v", spec)
	}
	log.Debugf("[store] Deleted check: %s", key)

	return nil
}

func (s *FusisStore) WatchChecks() {
	specs := []types.CheckSpec{}

	stopCh := make(<-chan struct{})
	events, err := s.kv.WatchTree(s.key("checks"), stopCh)
	if err != nil {
		log.Error(errors.Wrap(err, "failed watching fusis/checks"))
	}

	for {
		select {
		case entries := <-events:
			log.Debug("[store] Checks received")

			for _, pair := range entries {
				spec := types.CheckSpec{}
				if err := json.Unmarshal(pair.Value, &spec); err != nil {
					log.Error(errors.Wrap(err, "failed unmarshall of checks"))
				}
				log.Debugf("[store] Got Check: %#v", spec)

				specs = append(specs, spec)
			}

			s.Lock()
			for _, ch := range s.checksChannels {
				ch <- specs
			}
			s.Unlock()

			//Cleaning up destinations slice
			specs = []types.CheckSpec{}
		}
	}
}

func (s *FusisStore) key(keys ...string) string {
	return strings.Join(append([]string{s.prefix}, keys...), "/")
}
<commit_msg>Removing prints<commit_after>package store

import (
	"encoding/json"
	"net"
	"net/url"
	"strings"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libkv"
	kv "github.com/docker/libkv/store"
	"github.com/docker/libkv/store/consul"
	"github.com/docker/libkv/store/etcd"
	"github.com/luizbafilho/fusis/config"
	"github.com/luizbafilho/fusis/types"
	"github.com/pkg/errors"
)

func init() {
	registryStores()
}

func registryStores() {
	libkv.AddStore(kv.CONSUL, consul.New)
	libkv.AddStore(kv.ETCD, etcd.New)
}

type Store interface {
	GetServices() ([]types.Service, error)
	// GetService(serviceId string) (types.Service, error)
	AddService(svc *types.Service) error
	DeleteService(svc *types.Service) error
	SubscribeServices(ch chan []types.Service)
	WatchServices()

	GetDestinations() ([]types.Destination, error)
	AddDestination(svc *types.Service, dst *types.Destination) error
	DeleteDestination(svc *types.Service, dst *types.Destination) error
	SubscribeDestinations(ch chan []types.Destination)
	WatchDestinations()

	AddCheck(check types.CheckSpec) error
	DeleteCheck(check types.CheckSpec) error
	SubscribeChecks(ch chan []types.CheckSpec)
	WatchChecks()

	GetKV() kv.Store
}

var (
	ErrUnsupportedStore = errors.New("[store] Unsupported store")
)

type FusisStore struct {
	sync.Mutex
	kv kv.Store

	prefix string

	servicesChannels    []chan []types.Service
	destinationChannels []chan []types.Destination
	checksChannels      []chan []types.CheckSpec
}

func New(config *config.BalancerConfig) (Store, error) {
	u, err := url.Parse(config.StoreAddress)
	if err != nil {
		return nil, errors.Wrap(err, "error paring store address")
	}

	scheme := u.Scheme
	if scheme != "consul" && scheme != "etcd" {
		return nil, ErrUnsupportedStore
	}

	//Validating open connection
	_, err = net.Dial("tcp", u.Host)
	if err != nil {
		return nil, errors.Wrap(err, "Store connection failed. Make sure your store is up and running.")
	}

	kv, err := libkv.NewStore(
		kv.Backend(scheme),
		[]string{u.Host},
		nil,
	)
	if err != nil {
		kv.Close()
		return nil, errors.Wrap(err, "Cannot create store consul")
	}

	svcsChs := []chan []types.Service{}
	dstsChs := []chan []types.Destination{}
	checksChs := []chan []types.CheckSpec{}

	fusisStore := &FusisStore{
		kv:                  kv,
		prefix:              config.StorePrefix,
		servicesChannels:    svcsChs,
		destinationChannels: dstsChs,
		checksChannels:      checksChs,
	}

	go fusisStore.WatchServices()
	go fusisStore.WatchDestinations()
	go fusisStore.WatchChecks()

	return fusisStore, nil
}

func (s *FusisStore) GetKV() kv.Store {
	return s.kv
}

// func (s *FusisStore) GetService(serviceId string) (types.Service, err) {
// }

func (s *FusisStore) GetServices() ([]types.Service, error) {
	svcs := []types.Service{}
	entries, err := s.kv.List(s.key("services"))
	if err != nil {
		if err == kv.ErrKeyNotFound {
			return svcs, nil
		}

		return nil, err
	}

	for _, pair := range entries {
		svc := types.Service{}
		if err := json.Unmarshal(pair.Value, &svc); err != nil {
			log.Error("[store] ", err)
		}

		svcs = append(svcs, svc)
	}

	return svcs, nil
}

func (s *FusisStore) GetDestinations() ([]types.Destination, error) {
	dsts := []types.Destination{}
	entries, err := s.kv.List("fusis/destinations")
	if err != nil {
		return nil, err
	}

	for _, pair := range entries {
		dst := types.Destination{}
		if err := json.Unmarshal(pair.Value, &dst); err != nil {
			log.Error("[store] ", err)
		}

		dsts = append(dsts, dst)
	}

	return dsts, nil
}

func (s *FusisStore) AddService(svc *types.Service) error {
	key := s.key("services", svc.GetId(), "config")

	value, err := json.Marshal(svc)
	if err != nil {
		return errors.Wrapf(err, "error marshaling service: %v", svc)
	}

	err = s.kv.Put(key, value, nil)
	if err != nil {
		return errors.Wrapf(err, "error sending service to store: %v", svc)
	}

	return nil
}

func (s *FusisStore) DeleteService(svc *types.Service) error {
	key := s.key("services", svc.GetId())

	err := s.kv.DeleteTree(key)
	if err != nil {
		return errors.Wrapf(err, "error trying to delete service: %v", svc)
	}

	return nil
}

func (s *FusisStore) SubscribeServices(updateCh chan []types.Service) {
	s.Lock()
	defer s.Unlock()
	s.servicesChannels = append(s.servicesChannels, updateCh)
}

func (s *FusisStore) WatchServices() {
	svcs := []types.Service{}

	stopCh := make(<-chan struct{})
	events, err := s.kv.WatchTree(s.key("services"), stopCh)
	if err != nil {
		log.Error("[store] ", err)
	}

	for {
		select {
		case entries := <-events:
			log.Debug("[store] Services received")

			for _, pair := range entries {
				svc := types.Service{}
				if err := json.Unmarshal(pair.Value, &svc); err != nil {
					log.Error("[store] ", err)
				}
				log.Debugf("[store] Got sevice: %#v ", svc)

				svcs = append(svcs, svc)
			}

			s.Lock()
			for _, ch := range s.servicesChannels {
				log.Debug("[store] Sending message to service ch")
				ch <- svcs
			}
			s.Unlock()

			//Cleaning up services slice
			svcs = []types.Service{}
		}
	}
}

func (s *FusisStore) AddDestination(svc *types.Service, dst *types.Destination) error {
	key := s.key("destinations", svc.GetId(), dst.GetId())

	value, err := json.Marshal(dst)
	if err != nil {
		return errors.Wrapf(err, "error marshaling destination: %v", dst)
	}

	err = s.kv.Put(key, value, nil)
	if err != nil {
		return errors.Wrapf(err, "error sending destination to store: %v", dst)
	}
	log.Debugf("[store] Added destination: %s with key: %s", value, key)

	return nil
}

func (s *FusisStore) DeleteDestination(svc *types.Service, dst *types.Destination) error {
	key := s.key("destinations", svc.GetId(), dst.GetId())
	// key := fmt.Sprintf("fusis/destinations/%s/%s", svc.GetId(), dst.GetId())

	err := s.kv.DeleteTree(key)
	if err != nil {
		return errors.Wrapf(err, "error trying to delete destination: %v", dst)
	}
	log.Debugf("[store] Deleted destination: %s", key)

	return nil
}

func (s *FusisStore) SubscribeDestinations(updateCh chan []types.Destination) {
	s.Lock()
	defer s.Unlock()
	s.destinationChannels = append(s.destinationChannels, updateCh)
}

func (s *FusisStore) WatchDestinations() {
	dsts := []types.Destination{}

	stopCh := make(<-chan struct{})
	events, err := s.kv.WatchTree(s.key("destinations"), stopCh)
	if err != nil {
		errors.Wrap(err, "failed watching fusis/destinations")
	}

	for {
		select {
		case entries := <-events:
			log.Debug("[store] Destinations received")

			for _, pair := range entries {
				dst := types.Destination{}
				if err := json.Unmarshal(pair.Value, &dst); err != nil {
					errors.Wrap(err, "failed unmarshall of destinations")
				}
				log.Debugf("[store] Got destination: %#v", dst)

				dsts = append(dsts, dst)
			}

			s.Lock()
			for _, ch := range s.destinationChannels {
				ch <- dsts
			}
			s.Unlock()

			//Cleaning up destinations slice
			dsts = []types.Destination{}
		}
	}
}

func (s *FusisStore) SubscribeChecks(updateCh chan []types.CheckSpec) {
	s.Lock()
	defer s.Unlock()
	s.checksChannels = append(s.checksChannels, updateCh)
}

func (s *FusisStore) AddCheck(spec types.CheckSpec) error {
	key := s.key("checks", spec.ServiceID)

	value, err := json.Marshal(spec)
	if err != nil {
		return errors.Wrapf(err, "error marshaling CheckSpec: %#v", spec)
	}

	err = s.kv.Put(key, value, nil)
	if err != nil {
		return errors.Wrapf(err, "error sending CheckSpec to store: %#v", spec)
	}

	return nil
}

func (s *FusisStore) DeleteCheck(spec types.CheckSpec) error {
	key := s.key("checks", spec.ServiceID)

	err := s.kv.DeleteTree(key)
	if err != nil {
		return errors.Wrapf(err, "error trying to delete check: %#v", spec)
	}
	log.Debugf("[store] Deleted check: %s", key)

	return nil
}

func (s *FusisStore) WatchChecks() {
	specs := []types.CheckSpec{}

	stopCh := make(<-chan struct{})
	events, err := s.kv.WatchTree(s.key("checks"), stopCh)
	if err != nil {
		log.Error(errors.Wrap(err, "failed watching fusis/checks"))
	}

	for {
		select {
		case entries := <-events:
			log.Debug("[store] Checks received")

			for _, pair := range entries {
				spec := types.CheckSpec{}
				if err := json.Unmarshal(pair.Value, &spec); err != nil {
					log.Error(errors.Wrap(err, "failed unmarshall of checks"))
				}
				log.Debugf("[store] Got Check: %#v", spec)

				specs = append(specs, spec)
			}

			s.Lock()
			for _, ch := range s.checksChannels {
				ch <- specs
			}
			s.Unlock()

			//Cleaning up destinations slice
			specs = []types.CheckSpec{}
		}
	}
}

func (s *FusisStore) key(keys ...string) string {
	return strings.Join(append([]string{s.prefix}, keys...), "/")
}
<|endoftext|>


<commit_before>package tracker

import (
	"fmt"
	"time"
	"net/http"
	"net/url"
	"xd/lib/common"
	"xd/lib/network"

	"github.com/zeebo/bencode"
)

// http tracker
type HttpTracker struct {
	url string
	session network.Network
	// http client
	client *http.Client
	// next announce
	next time.Time
}

// create new http tracker from url
func NewHttpTracker(n network.Network, url string) *HttpTracker {
	return &HttpTracker{
		url: url,
		session: n,
		client: &http.Client{
			Transport: &http.Transport{
				Dial: n.Dial,
			},
		},
	}
}

// http compact response
type compactHttpAnnounceResponse struct {
	Peers []byte `bencode:"peers"`
	Interval int `bencode:"interval"`
}

func (t *HttpTracker) Name() string {
	u, err := url.Parse(t.url)
	if err == nil {
		return u.Host
	}
	return t.url
}

// send announce via http request
func (t *HttpTracker) Announce(req *Request) (resp *Response, err error) {
	interval := 30
	var u *url.URL
	u, err = url.Parse(t.url)
	if err == nil {

		// build query
		v := u.Query()
		v.Add("ip", req.IP.String())
		v.Add("infohash", string(req.Infohash.Bytes()))
		v.Add("peer_id", string(req.PeerID.Bytes()))
		v.Add("port", fmt.Sprintf("%d", req.Port))
		v.Add("numwant", fmt.Sprintf("%d", req.NumWant))
		if len(req.Event) > 0 {
			v.Add("event", req.Event)
		}
		v.Add("downloaded", fmt.Sprintf("%d", req.Downloaded))
		v.Add("uploaded", fmt.Sprintf("%d", req.Uploaded))

		// compact response
		if req.Compact {
			v.Add("compact", "1")
		}
		u.RawQuery = v.Encode()
		var r *http.Response
		r, err = t.client.Get(u.String())
		if err == nil {
			defer r.Body.Close()
			dec := bencode.NewDecoder(r.Body)
			resp = new(Response)
			if req.Compact {
				cresp := new(compactHttpAnnounceResponse)
				err = dec.Decode(cresp)
				if err == nil {
					l := len(cresp.Peers) / 32
					for l > 0 {
						p := new(common.Peer)
						// TODO: bounds check
						copy(p.Compact[:], cresp.Peers[(l-1)*32: l*32])
						resp.Peers = append(resp.Peers, p)
						l --
					}
				}
			} else {
				// decode non compact response
				err = dec.Decode(resp)
			}
			interval = resp.Interval
		}
	}
	t.next = time.Now().Add(time.Second * time.Duration(interval))
	return
}

func (t *HttpTracker) ShouldAnnounce() bool {
	return time.Now().After(t.next)
}
<commit_msg>fixes<commit_after>package tracker

import (
	"fmt"
	"time"
	"net/http"
	"net/url"
	"xd/lib/common"
	"xd/lib/network"
	"xd/lib/log"
	"github.com/zeebo/bencode"
)

// http tracker
type HttpTracker struct {
	url string
	session network.Network
	// http client
	client *http.Client
	// next announce
	next time.Time
	// announcing right now?
	announcing bool
}

// create new http tracker from url
func NewHttpTracker(n network.Network, url string) *HttpTracker {
	return &HttpTracker{
		url: url,
		session: n,
		client: &http.Client{
			Transport: &http.Transport{
				Dial: n.Dial,
			},
		},
	}
}

// http compact response
type compactHttpAnnounceResponse struct {
	Peers []byte `bencode:"peers"`
	Interval int `bencode:"interval"`
}

func (t *HttpTracker) Name() string {
	u, err := url.Parse(t.url)
	if err == nil {
		return u.Host
	}
	return t.url
}

// send announce via http request
func (t *HttpTracker) Announce(req *Request) (resp *Response, err error) {
	interval := 30
	var u *url.URL
	u, err = url.Parse(t.url)
	if err == nil {

		// build query
		v := u.Query()
		v.Add("ip", req.IP.String())
		v.Add("infohash", string(req.Infohash.Bytes()))
		v.Add("peer_id", string(req.PeerID.Bytes()))
		v.Add("port", fmt.Sprintf("%d", req.Port))
		v.Add("numwant", fmt.Sprintf("%d", req.NumWant))
		if len(req.Event) > 0 {
			v.Add("event", req.Event)
		}
		v.Add("downloaded", fmt.Sprintf("%d", req.Downloaded))
		v.Add("uploaded", fmt.Sprintf("%d", req.Uploaded))

		// compact response
		if req.Compact {
			v.Add("compact", "1")
		}
		u.RawQuery = v.Encode()
		var r *http.Response
		t.announcing = true
		log.Debugf("%s announcing", t.Name())
		r, err = t.client.Get(u.String())
		if err == nil {
			defer r.Body.Close()
			dec := bencode.NewDecoder(r.Body)
			resp = new(Response)
			if req.Compact {
				cresp := new(compactHttpAnnounceResponse)
				err = dec.Decode(cresp)
				if err == nil {
					interval = cresp.Interval
					l := len(cresp.Peers) / 32
					for l > 0 {
						p := new(common.Peer)
						// TODO: bounds check
						copy(p.Compact[:], cresp.Peers[(l-1)*32: l*32])
						resp.Peers = append(resp.Peers, p)
						l --
					}
				}
			} else {
				// decode non compact response
				err = dec.Decode(resp)
				interval = resp.Interval
			}
		}
	}
	t.next = time.Now().Add(time.Second * time.Duration(interval))
	log.Infof("%s next announce %s", t.Name(), t.next)
	t.announcing = false
	return
}

func (t *HttpTracker) ShouldAnnounce() bool {
	return time.Now().After(t.next) && !t.announcing
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ioutil implements some I/O utility functions.
package ioutil

import (
	"bytes"
	"io"
	"os"
	"sort"
)

// readAll reads from r until an error or EOF and returns the data it read
// from the internal buffer allocated with a specified capacity.
func readAll(r io.Reader, capacity int64) ([]byte, os.Error) {
	buf := bytes.NewBuffer(make([]byte, 0, capacity))
	_, err := buf.ReadFrom(r)
	return buf.Bytes(), err
}

// ReadAll reads from r until an error or EOF and returns the data it read.
func ReadAll(r io.Reader) ([]byte, os.Error) {
	return readAll(r, bytes.MinRead)
}

// ReadFile reads the file named by filename and returns the contents.
func ReadFile(filename string) ([]byte, os.Error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	// It's a good but not certain bet that FileInfo will tell us exactly how much to
	// read, so let's try it but be prepared for the answer to be wrong.
	fi, err := f.Stat()
	var n int64
	if err == nil && fi.Size < 2e9 { // Don't preallocate a huge buffer, just in case.
		n = fi.Size
	}
	// As initial capacity for readAll, use n + a little extra in case Size is zero,
	// and to avoid another allocation after Read has filled the buffer.  The readAll
	// call will read into its allocated internal buffer cheaply.  If the size was
	// wrong, we'll either waste some space off the end or reallocate as needed, but
	// in the overwhelmingly common case we'll get it just right.
	return readAll(f, n+bytes.MinRead)
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it with permissions perm;
// otherwise WriteFile truncates it before writing.
func WriteFile(filename string, data []byte, perm uint32) os.Error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	f.Close()
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	return err
}

// A fileInfoList implements sort.Interface.
type fileInfoList []*os.FileInfo

func (f fileInfoList) Len() int           { return len(f) }
func (f fileInfoList) Less(i, j int) bool { return f[i].Name < f[j].Name }
func (f fileInfoList) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }

// ReadDir reads the directory named by dirname and returns
// a list of sorted directory entries.
func ReadDir(dirname string) ([]*os.FileInfo, os.Error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	list, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	fi := make(fileInfoList, len(list))
	for i := range list {
		fi[i] = &list[i]
	}
	sort.Sort(fi)
	return fi, nil
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

// NopCloser returns a ReadCloser with a no-op Close method wrapping
// the provided Reader r.
func NopCloser(r io.Reader) io.ReadCloser {
	return nopCloser{r}
}

type devNull int

func (devNull) Write(p []byte) (int, os.Error) {
	return len(p), nil
}

var blackHole = make([]byte, 8192)

func (devNull) ReadFrom(r io.Reader) (n int64, err os.Error) {
	readSize := 0
	for {
		readSize, err = r.Read(blackHole)
		n += int64(readSize)
		if err != nil {
			if err == os.EOF {
				return n, nil
			}
			return
		}
	}
	panic("unreachable")
}

// Discard is an io.Writer on which all Write calls succeed
// without doing anything.
var Discard io.Writer = devNull(0)
<commit_msg>io/ioutil: add a comment on why devNull is a ReaderFrom<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ioutil implements some I/O utility functions.
package ioutil

import (
	"bytes"
	"io"
	"os"
	"sort"
)

// readAll reads from r until an error or EOF and returns the data it read
// from the internal buffer allocated with a specified capacity.
func readAll(r io.Reader, capacity int64) ([]byte, os.Error) {
	buf := bytes.NewBuffer(make([]byte, 0, capacity))
	_, err := buf.ReadFrom(r)
	return buf.Bytes(), err
}

// ReadAll reads from r until an error or EOF and returns the data it read.
func ReadAll(r io.Reader) ([]byte, os.Error) {
	return readAll(r, bytes.MinRead)
}

// ReadFile reads the file named by filename and returns the contents.
func ReadFile(filename string) ([]byte, os.Error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	// It's a good but not certain bet that FileInfo will tell us exactly how much to
	// read, so let's try it but be prepared for the answer to be wrong.
	fi, err := f.Stat()
	var n int64
	if err == nil && fi.Size < 2e9 { // Don't preallocate a huge buffer, just in case.
		n = fi.Size
	}
	// As initial capacity for readAll, use n + a little extra in case Size is zero,
	// and to avoid another allocation after Read has filled the buffer.  The readAll
	// call will read into its allocated internal buffer cheaply.  If the size was
	// wrong, we'll either waste some space off the end or reallocate as needed, but
	// in the overwhelmingly common case we'll get it just right.
	return readAll(f, n+bytes.MinRead)
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it with permissions perm;
// otherwise WriteFile truncates it before writing.
func WriteFile(filename string, data []byte, perm uint32) os.Error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
	if err != nil {
		return err
	}
	n, err := f.Write(data)
	f.Close()
	if err == nil && n < len(data) {
		err = io.ErrShortWrite
	}
	return err
}

// A fileInfoList implements sort.Interface.
type fileInfoList []*os.FileInfo

func (f fileInfoList) Len() int           { return len(f) }
func (f fileInfoList) Less(i, j int) bool { return f[i].Name < f[j].Name }
func (f fileInfoList) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }

// ReadDir reads the directory named by dirname and returns
// a list of sorted directory entries.
func ReadDir(dirname string) ([]*os.FileInfo, os.Error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	list, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}
	fi := make(fileInfoList, len(list))
	for i := range list {
		fi[i] = &list[i]
	}
	sort.Sort(fi)
	return fi, nil
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() os.Error { return nil }

// NopCloser returns a ReadCloser with a no-op Close method wrapping
// the provided Reader r.
func NopCloser(r io.Reader) io.ReadCloser {
	return nopCloser{r}
}

type devNull int

// devNull implements ReaderFrom as an optimization so io.Copy to
// ioutil.Discard can avoid doing unnecessary work.
var _ io.ReaderFrom = devNull(0)

func (devNull) Write(p []byte) (int, os.Error) {
	return len(p), nil
}

var blackHole = make([]byte, 8192)

func (devNull) ReadFrom(r io.Reader) (n int64, err os.Error) {
	readSize := 0
	for {
		readSize, err = r.Read(blackHole)
		n += int64(readSize)
		if err != nil {
			if err == os.EOF {
				return n, nil
			}
			return
		}
	}
	panic("unreachable")
}

// Discard is an io.Writer on which all Write calls succeed
// without doing anything.
var Discard io.Writer = devNull(0)
<|endoftext|>