<commit_before>package main

import (
	"fmt"

	"github.com/urfave/cli"
)

var (
	imagesFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "display only image IDs",
		},
		cli.BoolFlag{
			Name:  "noheading, n",
			Usage: "do not print column headings",
		},
		cli.BoolFlag{
			Name:  "notruncate",
			Usage: "do not truncate output",
		},
	}
	imagesDescription = "Lists locally stored images."
	imagesCommand     = cli.Command{
		Name:        "images",
		Usage:       "List images in local storage",
		Description: imagesDescription,
		Flags:       imagesFlags,
		Action:      imagesCmd,
		ArgsUsage:   " ",
	}
)

func imagesCmd(c *cli.Context) error {
	store, err := getStore(c)
	if err != nil {
		return err
	}

	quiet := false
	if c.IsSet("quiet") {
		quiet = c.Bool("quiet")
	}
	noheading := false
	if c.IsSet("noheading") {
		noheading = c.Bool("noheading")
	}
	truncate := true
	if c.IsSet("notruncate") {
		truncate = !c.Bool("notruncate")
	}
	images, err := store.Images()
	if err != nil {
		return fmt.Errorf("error reading images: %v", err)
	}

	if len(images) > 0 && !noheading && !quiet {
		if truncate {
			fmt.Printf("%-12s %s\n", "IMAGE ID", "IMAGE NAME")
		} else {
			fmt.Printf("%-64s %s\n", "IMAGE ID", "IMAGE NAME")
		}
	}
	for _, image := range images {
		if quiet {
			fmt.Printf("%s\n", image.ID)
		} else {
			names := []string{""}
			if len(image.Names) > 0 {
				names = image.Names
			}
			for _, name := range names {
				if truncate {
					fmt.Printf("%-12.12s %s\n", image.ID, name)
				} else {
					fmt.Printf("%-64s %s\n", image.ID, name)
				}
			}
		}
	}

	return nil
}
<commit_msg>Small cleanup on buildah rmi to make code cleaner<commit_after>package main

import (
	"fmt"

	"github.com/urfave/cli"
)

var (
	imagesFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "display only image IDs",
		},
		cli.BoolFlag{
			Name:  "noheading, n",
			Usage: "do not print column headings",
		},
		cli.BoolFlag{
			Name:  "notruncate",
			Usage: "do not truncate output",
		},
	}
	imagesDescription = "Lists locally stored images."
	imagesCommand     = cli.Command{
		Name:        "images",
		Usage:       "List images in local storage",
		Description: imagesDescription,
		Flags:       imagesFlags,
		Action:      imagesCmd,
		ArgsUsage:   " ",
	}
)

func imagesCmd(c *cli.Context) error {
	store, err := getStore(c)
	if err != nil {
		return err
	}

	quiet := false
	if c.IsSet("quiet") {
		quiet = c.Bool("quiet")
	}
	noheading := false
	if c.IsSet("noheading") {
		noheading = c.Bool("noheading")
	}
	truncate := true
	if c.IsSet("notruncate") {
		truncate = !c.Bool("notruncate")
	}
	images, err := store.Images()
	if err != nil {
		return fmt.Errorf("error reading images: %v", err)
	}

	if len(images) > 0 && !noheading && !quiet {
		if truncate {
			fmt.Printf("%-12s %s\n", "IMAGE ID", "IMAGE NAME")
		} else {
			fmt.Printf("%-64s %s\n", "IMAGE ID", "IMAGE NAME")
		}
	}
	for _, image := range images {
		if quiet {
			fmt.Printf("%s\n", image.ID)
			continue
		}
		names := []string{""}
		if len(image.Names) > 0 {
			names = image.Names
		}
		for _, name := range names {
			if truncate {
				fmt.Printf("%-12.12s %s\n", image.ID, name)
			} else {
				fmt.Printf("%-64s %s\n", image.ID, name)
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Package react returns a random reaction parsed from a json file, see example json file for structure
package react

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"regexp"
	"time"

	"github.com/fsnotify/fsnotify"
	jfile "github.com/nboughton/go-utils/json/file"
)

// Item is the match text and possible responses for a reacion
type Item struct {
	Text string
	Resp []string
}

// React contains reactions from a reactions.json file
type React struct {
	Items []Item
}

var (
	// Reactions is the struct that the reactions.json file is parsed into
	Reactions  React
	reactRegex *regexp.Regexp
	reactFile  *string
)

func init() {
	rand.Seed(time.Now().UnixNano())
	reactFile = flag.String("react", "reactions.json", "Path to reactions file")
	flag.Parse()

	readFile()
	genRegex()

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Panicln(err)
	}

	err = watcher.Add(*reactFile)
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		for {
			select {
			case ev := <-watcher.Events:
				// if modified reload
				if ev.Op == fsnotify.Write {
					readFile()
					genRegex()
				}

			case err := <-watcher.Errors:
				log.Println("error:", err)

			}
		}
	}()
}

// Match tests to see if a string matches a listed string
func Match(s string) bool {
	return reactRegex.MatchString(s)
}

// Respond returns a random response from the first reaction to match the generated regex
func Respond(s string) (string, error) {
	str := reactRegex.FindAllString(s, 1)[0]
	for _, r := range Reactions.Items {
		if r.Text == str {
			return randS(r.Resp), nil
		}
	}
	return "", fmt.Errorf("No response found")
}

func randS(s []string) string {
	if len(s) > 1 {
		return s[rand.Intn(len(s)-1)]
	}

	return ""
}

func readFile() error {
	err := jfile.Scan(*reactFile, &Reactions)
	if err != nil {
		return err
	}
	return nil
}

func genRegex() {
	s := "("
	for idx, i := range Reactions.Items {
		s += regexp.QuoteMeta(i.Text)
		if idx < len(Reactions.Items)-1 {
			s += "|"
		}
	}
	s += ")"

	reactRegex = regexp.MustCompile(s)
}
<commit_msg>fix reaction so that single item arrays produce a result<commit_after>// Package react returns a random reaction parsed from a json file, see example json file for structure
package react

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"regexp"
	"time"

	"github.com/fsnotify/fsnotify"
	jfile "github.com/nboughton/go-utils/json/file"
)

// Item is the match text and possible responses for a reacion
type Item struct {
	Text string
	Resp []string
}

// React contains reactions from a reactions.json file
type React struct {
	Items []Item
}

var (
	// Reactions is the struct that the reactions.json file is parsed into
	Reactions  React
	reactRegex *regexp.Regexp
	reactFile  *string
)

func init() {
	rand.Seed(time.Now().UnixNano())
	reactFile = flag.String("react", "reactions.json", "Path to reactions file")
	flag.Parse()

	readFile()
	genRegex()

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Panicln(err)
	}

	err = watcher.Add(*reactFile)
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		for {
			select {
			case ev := <-watcher.Events:
				// if modified reload
				if ev.Op == fsnotify.Write {
					readFile()
					genRegex()
				}

			case err := <-watcher.Errors:
				log.Println("error:", err)

			}
		}
	}()
}

// Match tests to see if a string matches a listed string
func Match(s string) bool {
	return reactRegex.MatchString(s)
}

// Respond returns a random response from the first reaction to match the generated regex
func Respond(s string) (string, error) {
	str := reactRegex.FindAllString(s, 1)[0]
	for _, r := range Reactions.Items {
		if r.Text == str {
			return randS(r.Resp), nil
		}
	}
	return "", fmt.Errorf("No response found")
}

func randS(s []string) string {
	if len(s) > 1 {
		return s[rand.Intn(len(s)-1)]
	}

	return s[0]
}

func readFile() error {
	err := jfile.Scan(*reactFile, &Reactions)
	if err != nil {
		return err
	}
	return nil
}

func genRegex() {
	s := "("
	for idx, i := range Reactions.Items {
		s += regexp.QuoteMeta(i.Text)
		if idx < len(Reactions.Items)-1 {
			s += "|"
		}
	}
	s += ")"

	reactRegex = regexp.MustCompile(s)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"golang.org/x/sync/errgroup"
	"google.golang.org/api/option"
)

const (
	uploadURL     = "https://golang.org/dl/upload"
	projectID     = "999119582588"
	storageBucket = "golang"
)

// File represents a file on the golang.org downloads page.
// It should be kept in sync with the download code in x/tools/godoc/dl.
type File struct {
	Filename       string
	OS             string
	Arch           string
	Version        string
	ChecksumSHA256 string
	Size           int64
	Kind           string // "archive", "installer", "source"
}

// fileRe matches the files created by the release tool, such as:
//   go1.5beta2.src.tar.gz
//   go1.5.1.linux-386.tar.gz
//   go1.5.windows-amd64.msi
var fileRe = regexp.MustCompile(`^(go[a-z0-9-.]+)\.(src|([a-z0-9]+)-([a-z0-9]+)(?:-([a-z0-9.]+))?)\.(tar\.gz|zip|pkg|msi)(.asc)?$`)

func upload(files []string) error {
	ctx := context.Background()
	c, err := storageClient(ctx)
	if err != nil {
		return err
	}
	defer c.Close()

	var sitePayloads []*File
	var uploaded []string
	for _, name := range files {
		fileBytes, err := ioutil.ReadFile(name)
		if err != nil {
			return fmt.Errorf("ioutil.ReadFile(%q): %v", name, err)
		}
		base := filepath.Base(name)
		log.Printf("Uploading %v to GCS ...", base)
		m := fileRe.FindStringSubmatch(base)
		if m == nil {
			return fmt.Errorf("unrecognized file: %q", base)
		}
		checksum := fmt.Sprintf("%x", sha256.Sum256(fileBytes))

		// Upload file to Google Cloud Storage.
		if err := putObject(ctx, c, base, fileBytes); err != nil {
			return fmt.Errorf("uploading %q: %v", name, err)
		}
		uploaded = append(uploaded, base)

		if strings.HasSuffix(base, ".asc") {
			// Don't add asc files to the download page, just upload it.
			continue
		}

		// Upload file.sha256.
		fname := base + ".sha256"
		if err := putObject(ctx, c, fname, []byte(checksum)); err != nil {
			return fmt.Errorf("uploading %q: %v", base+".sha256", err)
		}
		uploaded = append(uploaded, fname)

		var kind string
		switch {
		case m[2] == "src":
			kind = "source"
		case strings.HasSuffix(base, ".tar.gz"), strings.HasSuffix(base, ".zip"):
			kind = "archive"
		case strings.HasSuffix(base, ".msi"), strings.HasSuffix(base, ".pkg"):
			kind = "installer"
		}
		f := &File{
			Filename:       base,
			Version:        m[1],
			OS:             m[3],
			Arch:           m[4],
			ChecksumSHA256: checksum,
			Size:           int64(len(fileBytes)),
			Kind:           kind,
		}
		sitePayloads = append(sitePayloads, f)
	}

	log.Println("Waiting for edge cache ...")
	if err := waitForEdgeCache(uploaded); err != nil {
		return fmt.Errorf("waitForEdgeCache(%+v): %v", uploaded, err)
	}

	log.Println("Uploading payloads to golang.org ...")
	for _, f := range sitePayloads {
		if err := updateSite(f); err != nil {
			return fmt.Errorf("updateSite(%+v): %v", f, err)
		}
	}
	return nil
}

func waitForEdgeCache(uploaded []string) error {
	var g errgroup.Group
	for _, u := range uploaded {
		fname := u
		g.Go(func() error {
			// Add some jitter so that dozens of requests are not hitting the
			// endpoint at once.
			time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
			t := time.Tick(5 * time.Second)
			var retries int
			for {
				url := "https://redirector.gvt1.com/edgedl/go/" + fname
				resp, err := http.Head(url)
				if err != nil {
					if retries < 3 {
						retries++
						<-t
						continue
					}
					return fmt.Errorf("http.Head(%q): %v", url, err)
				}
				retries = 0
				defer resp.Body.Close()
				if resp.StatusCode == http.StatusOK {
					log.Printf("%s is ready to go!", url)
					break
				}
				<-t
			}
			return nil
		})
	}
	return g.Wait()
}

func updateSite(f *File) error {
	// Post file details to golang.org.
	req, err := json.Marshal(f)
	if err != nil {
		return err
	}
	v := url.Values{"user": {*user}, "key": []string{userToken()}}
	u := fmt.Sprintf("%s?%s", uploadURL, v.Encode())
	resp, err := http.Post(u, "application/json", bytes.NewReader(req))
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		b, _ := ioutil.ReadAll(resp.Body)
		return fmt.Errorf("upload failed: %v\n%s", resp.Status, b)
	}
	return nil
}

func putObject(ctx context.Context, c *storage.Client, name string, body []byte) error {
	wr := c.Bucket(storageBucket).Object(name).NewWriter(ctx)
	wr.ACL = []storage.ACLRule{
		{Entity: storage.AllUsers, Role: storage.RoleReader},
		// If you don't give the owners access, the web UI seems to
		// have a bug and doesn't have access to see that it's public,
		// so won't render the "Shared Publicly" link. So we do that,
		// even though it's dumb and unnecessary otherwise:
		{Entity: storage.ACLEntity("project-owners-" + projectID), Role: storage.RoleOwner},
	}
	if _, err := wr.Write(body); err != nil {
		return err
	}
	return wr.Close()
}

func storageClient(ctx context.Context) (*storage.Client, error) {
	file := filepath.Join(os.Getenv("HOME"), "keys", "golang-org.service.json")
	blob, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	config, err := google.JWTConfigFromJSON(blob, storage.ScopeReadWrite)
	if err != nil {
		return nil, err
	}
	return storage.NewClient(ctx, option.WithTokenSource(config.TokenSource(ctx)))
}
<commit_msg>cmd/release: allow uploading artifacts from GCS<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"golang.org/x/sync/errgroup"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
)

const (
	uploadURL     = "https://golang.org/dl/upload"
	projectID     = "999119582588"
	storageBucket = "golang"
)

var publicACL = []storage.ACLRule{
	{Entity: storage.AllUsers, Role: storage.RoleReader},
	// If you don't give the owners access, the web UI seems to
	// have a bug and doesn't have access to see that it's public,
	// so won't render the "Shared Publicly" link. So we do that,
	// even though it's dumb and unnecessary otherwise:
	{Entity: storage.ACLEntity("project-owners-" + projectID), Role: storage.RoleOwner},
}

// File represents a file on the golang.org downloads page.
// It should be kept in sync with the download code in x/tools/godoc/dl.
type File struct {
	Filename       string
	OS             string
	Arch           string
	Version        string
	ChecksumSHA256 string
	Size           int64
	Kind           string // "archive", "installer", "source"
}

// fileRe matches the files created by the release tool, such as:
//   go1.5beta2.src.tar.gz
//   go1.5.1.linux-386.tar.gz
//   go1.5.windows-amd64.msi
var fileRe = regexp.MustCompile(`^(go[a-z0-9-.]+)\.(src|([a-z0-9]+)-([a-z0-9]+)(?:-([a-z0-9.]+))?)\.(tar\.gz|zip|pkg|msi)(.asc)?$`)

func upload(files []string) error {
	ctx := context.Background()
	c, err := storageClient(ctx)
	if err != nil {
		return err
	}
	defer c.Close()

	files, err = expandFiles(ctx, c, files)
	if err != nil {
		return err
	}

	files = chooseBestFiles(files)

	var sitePayloads []*File
	var uploaded []string
	for _, name := range files {
		base := filepath.Base(name)
		log.Printf("Uploading %v to GCS ...", base)
		m := fileRe.FindStringSubmatch(base)
		if m == nil {
			return fmt.Errorf("unrecognized file: %q", base)
		}

		checksum, size, err := uploadArtifact(ctx, c, name)
		if err != nil {
			return fmt.Errorf("uploading %q: %v", name, err)
		}

		uploaded = append(uploaded, base)

		if strings.HasSuffix(base, ".asc") {
			// Don't add asc files to the download page, just upload it.
			continue
		}

		// Upload file.sha256.
		fname := base + ".sha256"
		if err := putObject(ctx, c, fname, []byte(checksum)); err != nil {
			return fmt.Errorf("uploading %q: %v", base+".sha256", err)
		}
		uploaded = append(uploaded, fname)

		var kind string
		switch {
		case m[2] == "src":
			kind = "source"
		case strings.HasSuffix(base, ".tar.gz"), strings.HasSuffix(base, ".zip"):
			kind = "archive"
		case strings.HasSuffix(base, ".msi"), strings.HasSuffix(base, ".pkg"):
			kind = "installer"
		}
		f := &File{
			Filename:       base,
			Version:        m[1],
			OS:             m[3],
			Arch:           m[4],
			ChecksumSHA256: checksum,
			Size:           size,
			Kind:           kind,
		}
		sitePayloads = append(sitePayloads, f)
	}

	log.Println("Waiting for edge cache ...")
	if err := waitForEdgeCache(uploaded); err != nil {
		return fmt.Errorf("waitForEdgeCache(%+v): %v", uploaded, err)
	}

	log.Println("Uploading payloads to golang.org ...")
	for _, f := range sitePayloads {
		if err := updateSite(f); err != nil {
			return fmt.Errorf("updateSite(%+v): %v", f, err)
		}
	}
	return nil
}

func waitForEdgeCache(uploaded []string) error {
	var g errgroup.Group
	for _, u := range uploaded {
		fname := u
		g.Go(func() error {
			// Add some jitter so that dozens of requests are not hitting the
			// endpoint at once.
			time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
			t := time.Tick(5 * time.Second)
			var retries int
			for {
				url := "https://redirector.gvt1.com/edgedl/go/" + fname
				resp, err := http.Head(url)
				if err != nil {
					if retries < 3 {
						retries++
						<-t
						continue
					}
					return fmt.Errorf("http.Head(%q): %v", url, err)
				}
				retries = 0
				defer resp.Body.Close()
				if resp.StatusCode == http.StatusOK {
					log.Printf("%s is ready to go!", url)
					break
				}
				<-t
			}
			return nil
		})
	}
	return g.Wait()
}

func updateSite(f *File) error {
	// Post file details to golang.org.
	req, err := json.Marshal(f)
	if err != nil {
		return err
	}
	v := url.Values{"user": {*user}, "key": []string{userToken()}}
	u := fmt.Sprintf("%s?%s", uploadURL, v.Encode())
	resp, err := http.Post(u, "application/json", bytes.NewReader(req))
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		b, _ := ioutil.ReadAll(resp.Body)
		return fmt.Errorf("upload failed: %v\n%s", resp.Status, b)
	}
	return nil
}

func putObject(ctx context.Context, c *storage.Client, name string, body []byte) error {
	wr := c.Bucket(storageBucket).Object(name).NewWriter(ctx)
	wr.ACL = publicACL
	if _, err := wr.Write(body); err != nil {
		return err
	}
	return wr.Close()
}

func storageClient(ctx context.Context) (*storage.Client, error) {
	file := filepath.Join(os.Getenv("HOME"), "keys", "golang-org.service.json")
	blob, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	config, err := google.JWTConfigFromJSON(blob, storage.ScopeReadWrite)
	if err != nil {
		return nil, err
	}
	return storage.NewClient(ctx, option.WithTokenSource(config.TokenSource(ctx)))
}

// expandFiles expands any "/..." paths in GCS URIs to include files in its subtree.
func expandFiles(ctx context.Context, storageClient *storage.Client, files []string) ([]string, error) {
	var expanded []string
	for _, f := range files {
		if !(strings.HasPrefix(f, "gs://") && strings.HasSuffix(f, "/...")) {
			expanded = append(expanded, f)
			continue
		}
		bucket, path := gcsParts(f)

		iter := storageClient.Bucket(bucket).Objects(ctx, &storage.Query{
			Prefix: strings.TrimSuffix(path, "..."), // Retain trailing "/" (if present).
		})
		for {
			attrs, err := iter.Next()
			if err == iterator.Done {
				break
			}
			if err != nil {
				return nil, err
			}
			if filepath.Ext(attrs.Name) == ".sha256" {
				// Ignore sha256 files.
				continue
			}
			expanded = append(expanded, fmt.Sprintf("gs://%s/%s", attrs.Bucket, attrs.Name))
		}
	}
	return expanded, nil
}

// gcsParts splits a GCS URI (e.g., "gs://bucket/path/to/object") into its bucket and path parts:
// ("bucket", "path/to/object")
//
// It assumes its input a well-formed GCS URI.
func gcsParts(uri string) (bucket, path string) {
	parts := strings.SplitN(strings.TrimPrefix(uri, "gs://"), "/", 2)
	return parts[0], parts[1]
}

func chooseBestFiles(files []string) []string {
	// map from basename to filepath/GCS URI.
	best := make(map[string]string)
	for _, f := range files {
		base := filepath.Base(f)
		if _, ok := best[base]; !ok {
			best[base] = f
			continue
		}

		// Overwrite existing only if the new entry is signed.
		if strings.HasPrefix(f, "gs://") && strings.Contains(f, "/signed/") {
			best[base] = f
		}
	}

	var out []string
	for _, path := range best {
		out = append(out, path)
	}
	sort.Strings(out) // for prettier printing.
	return out
}

func uploadArtifact(ctx context.Context, storageClient *storage.Client, path string) (checksum string, size int64, err error) {
	if strings.HasPrefix(path, "gs://") {
		return uploadArtifactGCS(ctx, storageClient, path)
	}
	return uploadArtifactLocal(ctx, storageClient, path)
}

func uploadArtifactGCS(ctx context.Context, storageClient *storage.Client, path string) (checksum string, size int64, err error) {
	bucket, path := gcsParts(path)
	base := filepath.Base(path)
	src := storageClient.Bucket(bucket).Object(path)
	dst := storageClient.Bucket(storageBucket).Object(base)

	r, err := storageClient.Bucket(bucket).Object(path + ".sha256").NewReader(ctx)
	if err != nil {
		return "", -1, fmt.Errorf("could not get sha256: %v", err)
	}
	checksumBytes, err := ioutil.ReadAll(r)
	if err != nil {
		return "", -1, fmt.Errorf("could not get sha256: %v", err)
	}
	copier := dst.CopierFrom(src)
	copier.ACL = publicACL
	attrs, err := copier.Run(ctx)
	if err != nil {
		return "", -1, err
	}
	return string(checksumBytes), attrs.Size, nil
}

func uploadArtifactLocal(ctx context.Context, storageClient *storage.Client, path string) (checksum string, size int64, err error) {
	base := filepath.Base(path)

	fileBytes, err := ioutil.ReadFile(path)
	if err != nil {
		return "", -1, fmt.Errorf("ioutil.ReadFile: %v", err)
	}
	// Upload file to Google Cloud Storage.
	if err := putObject(ctx, storageClient, base, fileBytes); err != nil {
		return "", -1, err
	}
	checksum = fmt.Sprintf("%x", sha256.Sum256(fileBytes))
	return checksum, int64(len(fileBytes)), nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/containers/image/docker"
	"github.com/containers/image/manifest"
	"github.com/urfave/cli"
)

// inspectOutput is the output format of (skopeo inspect), primarily so that we can format it with a simple json.MarshalIndent.
type inspectOutput struct {
	Name          string `json:",omitempty"`
	Tag           string `json:",omitempty"`
	Digest        string
	RepoTags      []string
	Created       time.Time
	DockerVersion string
	Labels        map[string]string
	Architecture  string
	Os            string
	Layers        []string
}

var inspectCmd = cli.Command{
	Name:      "inspect",
	Usage:     "Inspect image IMAGE-NAME",
	ArgsUsage: "IMAGE-NAME",
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "raw",
			Usage: "output raw manifest",
		},
	},
	Action: func(c *cli.Context) error {
		img, err := parseImage(c)
		if err != nil {
			return err
		}
		rawManifest, _, err := img.Manifest()
		if err != nil {
			return err
		}
		if c.Bool("raw") {
			fmt.Fprintln(c.App.Writer, string(rawManifest))
			return nil
		}
		imgInspect, err := img.Inspect()
		if err != nil {
			return err
		}
		outputData := inspectOutput{
			Name: "", // Possibly overridden for a docker.Image.
			Tag:  imgInspect.Tag,
			// Digest is set below.
			RepoTags:      []string{}, // Possibly overriden for a docker.Image.
			Created:       imgInspect.Created,
			DockerVersion: imgInspect.DockerVersion,
			Labels:        imgInspect.Labels,
			Architecture:  imgInspect.Architecture,
			Os:            imgInspect.Os,
			Layers:        imgInspect.Layers,
		}
		outputData.Digest, err = manifest.Digest(rawManifest)
		if err != nil {
			return fmt.Errorf("Error computing manifest digest: %v", err)
		}
		if dockerImg, ok := img.(*docker.Image); ok {
			outputData.Name = dockerImg.SourceRefFullName()
			outputData.RepoTags, err = dockerImg.GetRepositoryTags()
			if err != nil {
				return fmt.Errorf("Error determining repository tags: %v", err)
			}
		}
		out, err := json.MarshalIndent(outputData, "", "    ")
		if err != nil {
			return err
		}
		fmt.Fprintln(c.App.Writer, string(out))
		return nil
	},
}
<commit_msg>Output the original raw manifest in (skopeo inspect --raw)<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/containers/image/docker"
	"github.com/containers/image/manifest"
	"github.com/urfave/cli"
)

// inspectOutput is the output format of (skopeo inspect), primarily so that we can format it with a simple json.MarshalIndent.
type inspectOutput struct {
	Name          string `json:",omitempty"`
	Tag           string `json:",omitempty"`
	Digest        string
	RepoTags      []string
	Created       time.Time
	DockerVersion string
	Labels        map[string]string
	Architecture  string
	Os            string
	Layers        []string
}

var inspectCmd = cli.Command{
	Name:      "inspect",
	Usage:     "Inspect image IMAGE-NAME",
	ArgsUsage: "IMAGE-NAME",
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "raw",
			Usage: "output raw manifest",
		},
	},
	Action: func(c *cli.Context) error {
		img, err := parseImage(c)
		if err != nil {
			return err
		}
		rawManifest, _, err := img.Manifest()
		if err != nil {
			return err
		}
		if c.Bool("raw") {
			_, err := c.App.Writer.Write(rawManifest)
			if err != nil {
				return fmt.Errorf("Error writing manifest to standard output: %v", err)
			}
			return nil
		}
		imgInspect, err := img.Inspect()
		if err != nil {
			return err
		}
		outputData := inspectOutput{
			Name: "", // Possibly overridden for a docker.Image.
			Tag:  imgInspect.Tag,
			// Digest is set below.
			RepoTags:      []string{}, // Possibly overriden for a docker.Image.
			Created:       imgInspect.Created,
			DockerVersion: imgInspect.DockerVersion,
			Labels:        imgInspect.Labels,
			Architecture:  imgInspect.Architecture,
			Os:            imgInspect.Os,
			Layers:        imgInspect.Layers,
		}
		outputData.Digest, err = manifest.Digest(rawManifest)
		if err != nil {
			return fmt.Errorf("Error computing manifest digest: %v", err)
		}
		if dockerImg, ok := img.(*docker.Image); ok {
			outputData.Name = dockerImg.SourceRefFullName()
			outputData.RepoTags, err = dockerImg.GetRepositoryTags()
			if err != nil {
				return fmt.Errorf("Error determining repository tags: %v", err)
			}
		}
		out, err := json.MarshalIndent(outputData, "", "    ")
		if err != nil {
			return err
		}
		fmt.Fprintln(c.App.Writer, string(out))
		return nil
	},
}
<|endoftext|>


<commit_before>package kafka

import (
	"crypto/tls"
	"fmt"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/outputs"
	"github.com/influxdata/telegraf/plugins/serializers"

	"github.com/Shopify/sarama"
)

type Kafka struct {
	// Kafka brokers to send metrics to
	Brokers []string
	// Kafka topic
	Topic string
	// Routing Key Tag
	RoutingTag string `toml:"routing_tag"`
	// Compression Codec Tag
	CompressionCodec int
	// RequiredAcks Tag
	RequiredAcks int
	// MaxRetry Tag
	MaxRetry int

	// Legacy SSL config options
	// TLS client certificate
	Certificate string
	// TLS client key
	Key string
	// TLS certificate authority
	CA string

	// Path to CA file
	SSLCA string `toml:"ssl_ca"`
	// Path to host cert file
	SSLCert string `toml:"ssl_cert"`
	// Path to cert key file
	SSLKey string `toml:"ssl_key"`

	// Skip SSL verification
	InsecureSkipVerify bool

	tlsConfig tls.Config
	producer  sarama.SyncProducer

	serializer serializers.Serializer
}

var sampleConfig = `
  ## URLs of kafka brokers
  brokers = ["localhost:9092"]
  ## Kafka topic for producer messages
  topic = "telegraf"
  ## Telegraf tag to use as a routing key
  ##  ie, if this tag exists, it's value will be used as the routing key
  routing_tag = "host"

  ## CompressionCodec represents the various compression codecs recognized by
  ## Kafka in messages.
  ##  0 : No compression
  ##  1 : Gzip compression
  ##  2 : Snappy compression
  compression_codec = 0

  ##  RequiredAcks is used in Produce Requests to tell the broker how many
  ##  replica acknowledgements it must see before responding
  ##   0 : the producer never waits for an acknowledgement from the broker.
  ##       This option provides the lowest latency but the weakest durability
  ##       guarantees (some data will be lost when a server fails).
  ##   1 : the producer gets an acknowledgement after the leader replica has
  ##       received the data. This option provides better durability as the
  ##       client waits until the server acknowledges the request as successful
  ##       (only messages that were written to the now-dead leader but not yet
  ##       replicated will be lost).
  ##   -1: the producer gets an acknowledgement after all in-sync replicas have
  ##       received the data. This option provides the best durability, we
  ##       guarantee that no messages will be lost as long as at least one in
  ##       sync replica remains.
  required_acks = -1

  ##  The total number of times to retry sending a message
  max_retry = 3

  ## Optional SSL Config
  # ssl_ca = "/etc/telegraf/ca.pem"
  # ssl_cert = "/etc/telegraf/cert.pem"
  # ssl_key = "/etc/telegraf/key.pem"
  ## Use SSL but skip chain & host verification
  # insecure_skip_verify = false

  ## Data format to output.
  ## Each data format has it's own unique set of configuration options, read
  ## more about them here:
  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
  data_format = "influx"
`

func (k *Kafka) SetSerializer(serializer serializers.Serializer) {
	k.serializer = serializer
}

func (k *Kafka) Connect() error {
	config := sarama.NewConfig()

	config.Producer.RequiredAcks = sarama.RequiredAcks(k.RequiredAcks)
	config.Producer.Compression = sarama.CompressionCodec(k.CompressionCodec)
	config.Producer.Retry.Max = k.MaxRetry
	config.Producer.Return.Successes = true

	// Legacy support ssl config
	if k.Certificate != "" {
		k.SSLCert = k.Certificate
		k.SSLCA = k.CA
		k.SSLKey = k.Key
	}

	tlsConfig, err := internal.GetTLSConfig(
		k.SSLCert, k.SSLKey, k.SSLCA, k.InsecureSkipVerify)
	if err != nil {
		return err
	}

	if tlsConfig != nil {
		config.Net.TLS.Config = tlsConfig
		config.Net.TLS.Enable = true
	}

	producer, err := sarama.NewSyncProducer(k.Brokers, config)
	if err != nil {
		return err
	}
	k.producer = producer
	return nil
}

func (k *Kafka) Close() error {
	return k.producer.Close()
}

func (k *Kafka) SampleConfig() string {
	return sampleConfig
}

func (k *Kafka) Description() string {
	return "Configuration for the Kafka server to send metrics to"
}

func (k *Kafka) Write(metrics []telegraf.Metric) error {
	if len(metrics) == 0 {
		return nil
	}

	for _, metric := range metrics {
		buf, err := k.serializer.Serialize(metric)
		if err != nil {
			return err
		}

		m := &sarama.ProducerMessage{
			Topic: k.Topic,
			Value: sarama.ByteEncoder(buf),
		}
		if h, ok := metric.Tags()[k.RoutingTag]; ok {
			m.Key = sarama.StringEncoder(h)
		}

		_, _, err = k.producer.SendMessage(m)

		if err != nil {
			return fmt.Errorf("FAILED to send kafka message: %s\n", err)
		}
	}
	return nil
}

func init() {
	outputs.Add("kafka", func() telegraf.Output {
		return &Kafka{
			MaxRetry:     3,
			RequiredAcks: -1,
		}
	})
}
<commit_msg>it's -> its (#2729)<commit_after>package kafka

import (
	"crypto/tls"
	"fmt"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/outputs"
	"github.com/influxdata/telegraf/plugins/serializers"

	"github.com/Shopify/sarama"
)

type Kafka struct {
	// Kafka brokers to send metrics to
	Brokers []string
	// Kafka topic
	Topic string
	// Routing Key Tag
	RoutingTag string `toml:"routing_tag"`
	// Compression Codec Tag
	CompressionCodec int
	// RequiredAcks Tag
	RequiredAcks int
	// MaxRetry Tag
	MaxRetry int

	// Legacy SSL config options
	// TLS client certificate
	Certificate string
	// TLS client key
	Key string
	// TLS certificate authority
	CA string

	// Path to CA file
	SSLCA string `toml:"ssl_ca"`
	// Path to host cert file
	SSLCert string `toml:"ssl_cert"`
	// Path to cert key file
	SSLKey string `toml:"ssl_key"`

	// Skip SSL verification
	InsecureSkipVerify bool

	tlsConfig tls.Config
	producer  sarama.SyncProducer

	serializer serializers.Serializer
}

var sampleConfig = `
  ## URLs of kafka brokers
  brokers = ["localhost:9092"]
  ## Kafka topic for producer messages
  topic = "telegraf"
  ## Telegraf tag to use as a routing key
  ##  ie, if this tag exists, its value will be used as the routing key
  routing_tag = "host"

  ## CompressionCodec represents the various compression codecs recognized by
  ## Kafka in messages.
  ##  0 : No compression
  ##  1 : Gzip compression
  ##  2 : Snappy compression
  compression_codec = 0

  ##  RequiredAcks is used in Produce Requests to tell the broker how many
  ##  replica acknowledgements it must see before responding
  ##   0 : the producer never waits for an acknowledgement from the broker.
  ##       This option provides the lowest latency but the weakest durability
  ##       guarantees (some data will be lost when a server fails).
  ##   1 : the producer gets an acknowledgement after the leader replica has
  ##       received the data. This option provides better durability as the
  ##       client waits until the server acknowledges the request as successful
  ##       (only messages that were written to the now-dead leader but not yet
  ##       replicated will be lost).
  ##   -1: the producer gets an acknowledgement after all in-sync replicas have
  ##       received the data. This option provides the best durability, we
  ##       guarantee that no messages will be lost as long as at least one in
  ##       sync replica remains.
  required_acks = -1

  ##  The total number of times to retry sending a message
  max_retry = 3

  ## Optional SSL Config
  # ssl_ca = "/etc/telegraf/ca.pem"
  # ssl_cert = "/etc/telegraf/cert.pem"
  # ssl_key = "/etc/telegraf/key.pem"
  ## Use SSL but skip chain & host verification
  # insecure_skip_verify = false

  ## Data format to output.
  ## Each data format has its own unique set of configuration options, read
  ## more about them here:
  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
  data_format = "influx"
`

func (k *Kafka) SetSerializer(serializer serializers.Serializer) {
	k.serializer = serializer
}

func (k *Kafka) Connect() error {
	config := sarama.NewConfig()

	config.Producer.RequiredAcks = sarama.RequiredAcks(k.RequiredAcks)
	config.Producer.Compression = sarama.CompressionCodec(k.CompressionCodec)
	config.Producer.Retry.Max = k.MaxRetry
	config.Producer.Return.Successes = true

	// Legacy support ssl config
	if k.Certificate != "" {
		k.SSLCert = k.Certificate
		k.SSLCA = k.CA
		k.SSLKey = k.Key
	}

	tlsConfig, err := internal.GetTLSConfig(
		k.SSLCert, k.SSLKey, k.SSLCA, k.InsecureSkipVerify)
	if err != nil {
		return err
	}

	if tlsConfig != nil {
		config.Net.TLS.Config = tlsConfig
		config.Net.TLS.Enable = true
	}

	producer, err := sarama.NewSyncProducer(k.Brokers, config)
	if err != nil {
		return err
	}
	k.producer = producer
	return nil
}

func (k *Kafka) Close() error {
	return k.producer.Close()
}

func (k *Kafka) SampleConfig() string {
	return sampleConfig
}

func (k *Kafka) Description() string {
	return "Configuration for the Kafka server to send metrics to"
}

func (k *Kafka) Write(metrics []telegraf.Metric) error {
	if len(metrics) == 0 {
		return nil
	}

	for _, metric := range metrics {
		buf, err := k.serializer.Serialize(metric)
		if err != nil {
			return err
		}

		m := &sarama.ProducerMessage{
			Topic: k.Topic,
			Value: sarama.ByteEncoder(buf),
		}
		if h, ok := metric.Tags()[k.RoutingTag]; ok {
			m.Key = sarama.StringEncoder(h)
		}

		_, _, err = k.producer.SendMessage(m)

		if err != nil {
			return fmt.Errorf("FAILED to send kafka message: %s\n", err)
		}
	}
	return nil
}

func init() {
	outputs.Add("kafka", func() telegraf.Output {
		return &Kafka{
			MaxRetry:     3,
			RequiredAcks: -1,
		}
	})
}
<|endoftext|>


<commit_before>package extension

import (
	"errors"
	"fmt"
	"github.com/kayex/sirius"
	"github.com/kayex/sirius/config"
)

type StaticLoader struct {
	config config.AppConfig
}

func NewStaticLoader(cfg config.AppConfig) *StaticLoader {
	return &StaticLoader{
		config: cfg,
	}
}

func (l *StaticLoader) Load(eid sirius.EID) (sirius.Extension, error) {
	switch eid {
	case "thumbs_up":
		return &ThumbsUp{}, nil
	case "ripperino":
		return &Ripperino{}, nil
	case "replacer":
		return &Replacer{}, nil
	case "quotes":
		return &Quotes{}, nil
	case "ip_lookup":
		return &IPLookup{}, nil
	case "geocode":
		return &Geocode{
			APIKey: l.config.Maps.APIKey,
		}, nil
	}

	return nil, errors.New(fmt.Sprintf("Invalid eid: %v", eid))
}
<commit_msg>Shorten variable name<commit_after>package extension

import (
	"errors"
	"fmt"
	"github.com/kayex/sirius"
	"github.com/kayex/sirius/config"
)

type StaticLoader struct {
	cfg config.AppConfig
}

func NewStaticLoader(cfg config.AppConfig) *StaticLoader {
	return &StaticLoader{
		cfg: cfg,
	}
}

func (l *StaticLoader) Load(eid sirius.EID) (sirius.Extension, error) {
	switch eid {
	case "thumbs_up":
		return &ThumbsUp{}, nil
	case "ripperino":
		return &Ripperino{}, nil
	case "replacer":
		return &Replacer{}, nil
	case "quotes":
		return &Quotes{}, nil
	case "ip_lookup":
		return &IPLookup{}, nil
	case "geocode":
		return &Geocode{
			APIKey: l.cfg.Maps.APIKey,
		}, nil
	}

	return nil, errors.New(fmt.Sprintf("Invalid eid: %v", eid))
}
<|endoftext|>


<commit_before>// (C) 2017 by Ricardo Branco
//
// MIT License

package main

type word uint64

const (
	bitsPerWord = 64
)

type Bitset struct {
	words []word
	count int // Number of elements inserted
	max   int // Keep track of the maximum number ever inserted
}

// Returns a new Bitset. Set argument to the expected maximum number or -1.
func NewBitset(max int) *Bitset {
	words := 1
	if max >= 0 {
		words = max / bitsPerWord
		if max%bitsPerWord != 0 {
			words++
		}
	}
	bs := new(Bitset)
	bs.words = make([]word, words)
	bs.max = max
	return bs
}

func (bs *Bitset) Add(i int) {
	if i >= len(bs.words)*bitsPerWord {
		var newSet = make([]word, len(bs.words)+i/bitsPerWord+1)
		copy(newSet, bs.words)
		bs.words = newSet
	}
	bs.words[i/bitsPerWord] |= 1 << uint(i%bitsPerWord)
	bs.count++
	if i > bs.max {
		bs.max = i
	}
}

func (bs *Bitset) Del(i int) {
	bs.words[i/bitsPerWord] &= ^(1 << uint(i%bitsPerWord))
	bs.count--
}

func (bs *Bitset) Test(i int) bool {
	return (bs.words[i/bitsPerWord] & (1 << uint(i%bitsPerWord))) != 0
}

func (bs *Bitset) SetAll() {
	for i := range bs.words {
		bs.words[i] = ^word(0)
	}
	bs.count = len(bs.words) * bitsPerWord
	if bs.max >= 0 {
		bs.count = bs.max + 1
	}
}

func (bs *Bitset) ClearAll() {
	for i := range bs.words {
		bs.words[i] = 0
	}
	bs.count = 0
}

func (bs *Bitset) GetCount() int {
	return bs.count
}

// Returns a slice of all numbers in the set
func (bs *Bitset) GetAll() (s []int) {
	if bs.count == 0 {
		return
	}
	s = make([]int, 0, bs.count)
	for i, w := range bs.words {
		for {
			if w == 0 {
				break
			}
			bit := ffs(w)
			num := i*bitsPerWord + int(bit)
			if num > bs.max {
				return
			}
			s = append(s, num)
			w &= ^(1 << bit)
		}
	}
	return
}

func ffs(w word) (bit uint) {
	if w&0xffffffff == 0 {
		bit += 32
		w >>= 32
	}
	if w&0xffff == 0 {
		bit += 16
		w >>= 16
	}
	if w&0xff == 0 {
		bit += 8
		w >>= 8
	}
	if w&0xf == 0 {
		bit += 4
		w >>= 4
	}
	if w&0x3 == 0 {
		bit += 2
		w >>= 2
	}
	if w&0x1 == 0 {
		bit++
	}
	return
}
<commit_msg>Use a simple bitset with only one word<commit_after>// (C) 2017 by Ricardo Branco
//
// MIT License

package main

type word uint64

const (
	bitsPerWord = 64
)

type Bitset struct {
	word
	count int // Number of elements inserted
	max   int // Keep track of the maximum number ever inserted
}

// Returns a new Bitset. Set argument to the expected maximum number or -1.
func NewBitset(max int) *Bitset {
	bs := new(Bitset)
	bs.max = max
	return bs
}

func (bs *Bitset) Add(i int) {
	bs.word |= 1 << uint(i%bitsPerWord)
	bs.count++
	if i > bs.max {
		bs.max = i
	}
}

func (bs *Bitset) Del(i int) {
	bs.word &= ^(1 << uint(i%bitsPerWord))
	bs.count--
}

func (bs *Bitset) Test(i int) bool {
	return (bs.word & (1 << uint(i%bitsPerWord))) != 0
}

func (bs *Bitset) SetAll() {
	bs.word = ^word(0)
	if bs.max >= 0 {
		bs.count = bs.max + 1
	} else {
		bs.count = bitsPerWord
	}
}

func (bs *Bitset) ClearAll() {
	bs.word = 0
	bs.count = 0
}

func (bs *Bitset) GetCount() int {
	return bs.count
}

// Returns a slice of all numbers in the set
func (bs *Bitset) GetAll() (s []int) {
	if bs.count == 0 {
		return
	}
	s = make([]int, 0, bs.count)
	w := bs.word
	for {
		if w == 0 {
			break
		}
		bit := ffs(w)
		num := int(bit)
		if num > bs.max {
			return
		}
		s = append(s, num)
		w &= ^(1 << bit)
	}
	return
}

func ffs(w word) (bit uint) {
	if w&0xffffffff == 0 {
		bit += 32
		w >>= 32
	}
	if w&0xffff == 0 {
		bit += 16
		w >>= 16
	}
	if w&0xff == 0 {
		bit += 8
		w >>= 8
	}
	if w&0xf == 0 {
		bit += 4
		w >>= 4
	}
	if w&0x3 == 0 {
		bit += 2
		w >>= 2
	}
	if w&0x1 == 0 {
		bit++
	}
	return
}
<|endoftext|>


<commit_before>// Copyright © 2017 Karl Hepworth <Karl.Hepworth@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

// solrCmd represents the solr command
var solrCmd = &cobra.Command{
	Use:   "solr",
	Short: "A brief description of your command",
	Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:

Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("solr called")
	},
}

func init() {
	RootCmd.AddCommand(solrCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// solrCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// solrCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
<commit_msg>Remove solr command - out of project scope.<commit_after><|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build libfuzzer

package runtime

import "unsafe"

func libfuzzerCallWithTwoByteBuffers(fn, start, end *byte)
func libfuzzerCallTraceIntCmp(fn *byte, arg0, arg1, fakePC uintptr)
func libfuzzerCall4(fn *byte, fakePC uintptr, s1, s2 unsafe.Pointer, result uintptr)

// Keep in sync with the definition of ret_sled in src/runtime/libfuzzer_amd64.s
const retSledSize = 512

func libfuzzerTraceCmp1(arg0, arg1 uint8, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_cmp1, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

func libfuzzerTraceCmp2(arg0, arg1 uint16, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_cmp2, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

func libfuzzerTraceCmp4(arg0, arg1 uint32, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_cmp4, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

func libfuzzerTraceCmp8(arg0, arg1 uint64, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_cmp8, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

func libfuzzerTraceConstCmp1(arg0, arg1 uint8, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_const_cmp1, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

func libfuzzerTraceConstCmp2(arg0, arg1 uint16, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_const_cmp2, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

func libfuzzerTraceConstCmp4(arg0, arg1 uint32, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_const_cmp4, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

func libfuzzerTraceConstCmp8(arg0, arg1 uint64, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_const_cmp8, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

var pcTables []byte

func init() {
	libfuzzerCallWithTwoByteBuffers(&__sanitizer_cov_8bit_counters_init, &__start___sancov_cntrs, &__stop___sancov_cntrs)
	start := unsafe.Pointer(&__start___sancov_cntrs)
	end := unsafe.Pointer(&__stop___sancov_cntrs)

	// PC tables are arrays of ptr-sized integers representing pairs [PC,PCFlags] for every instrumented block.
	// The number of PCs and PCFlags is the same as the number of 8-bit counters. Each PC table entry has
	// the size of two ptr-sized integers. We allocate one more byte than what we actually need so that we can
	// get a pointer representing the end of the PC table array.
	size := (uintptr(end)-uintptr(start))*unsafe.Sizeof(uintptr(0))*2 + 1
	pcTables = make([]byte, size)
	libfuzzerCallWithTwoByteBuffers(&__sanitizer_cov_pcs_init, &pcTables[0], &pcTables[size-1])
}

// We call libFuzzer's __sanitizer_weak_hook_strcmp function which takes the
// following four arguments:
//
//  1. caller_pc: location of string comparison call site
//  2. s1: first string used in the comparison
//  3. s2: second string used in the comparison
//  4. result: an integer representing the comparison result. 0 indicates
//     equality (comparison will ignored by libfuzzer), non-zero indicates a
//     difference (comparison will be taken into consideration).
func libfuzzerHookStrCmp(s1, s2 string, fakePC int) {
	if s1 != s2 {
		libfuzzerCall4(&__sanitizer_weak_hook_strcmp, uintptr(fakePC), cstring(s1), cstring(s2), uintptr(1))
	}
	// if s1 == s2 we could call the hook with a last argument of 0 but this is unnecessary since this case will be then
	// ignored by libfuzzer
}

// This function has now the same implementation as libfuzzerHookStrCmp because we lack better checks
// for case-insensitive string equality in the runtime package.
func libfuzzerHookEqualFold(s1, s2 string, fakePC int) {
	if s1 != s2 {
		libfuzzerCall4(&__sanitizer_weak_hook_strcmp, uintptr(fakePC), cstring(s1), cstring(s2), uintptr(1))
	}
}

//go:linkname __sanitizer_cov_trace_cmp1 __sanitizer_cov_trace_cmp1
//go:cgo_import_static __sanitizer_cov_trace_cmp1
var __sanitizer_cov_trace_cmp1 byte

//go:linkname __sanitizer_cov_trace_cmp2 __sanitizer_cov_trace_cmp2
//go:cgo_import_static __sanitizer_cov_trace_cmp2
var __sanitizer_cov_trace_cmp2 byte

//go:linkname __sanitizer_cov_trace_cmp4 __sanitizer_cov_trace_cmp4
//go:cgo_import_static __sanitizer_cov_trace_cmp4
var __sanitizer_cov_trace_cmp4 byte

//go:linkname __sanitizer_cov_trace_cmp8 __sanitizer_cov_trace_cmp8
//go:cgo_import_static __sanitizer_cov_trace_cmp8
var __sanitizer_cov_trace_cmp8 byte

//go:linkname __sanitizer_cov_trace_const_cmp1 __sanitizer_cov_trace_const_cmp1
//go:cgo_import_static __sanitizer_cov_trace_const_cmp1
var __sanitizer_cov_trace_const_cmp1 byte

//go:linkname __sanitizer_cov_trace_const_cmp2 __sanitizer_cov_trace_const_cmp2
//go:cgo_import_static __sanitizer_cov_trace_const_cmp2
var __sanitizer_cov_trace_const_cmp2 byte

//go:linkname __sanitizer_cov_trace_const_cmp4 __sanitizer_cov_trace_const_cmp4
//go:cgo_import_static __sanitizer_cov_trace_const_cmp4
var __sanitizer_cov_trace_const_cmp4 byte

//go:linkname __sanitizer_cov_trace_const_cmp8 __sanitizer_cov_trace_const_cmp8
//go:cgo_import_static __sanitizer_cov_trace_const_cmp8
var __sanitizer_cov_trace_const_cmp8 byte

//go:linkname __sanitizer_cov_8bit_counters_init __sanitizer_cov_8bit_counters_init
//go:cgo_import_static __sanitizer_cov_8bit_counters_init
var __sanitizer_cov_8bit_counters_init byte

//go:linkname __start___sancov_cntrs __start___sancov_cntrs
//go:cgo_import_static __start___sancov_cntrs
var __start___sancov_cntrs byte

//go:linkname __stop___sancov_cntrs __stop___sancov_cntrs
//go:cgo_import_static __stop___sancov_cntrs
var __stop___sancov_cntrs byte

//go:linkname __sanitizer_cov_pcs_init __sanitizer_cov_pcs_init
//go:cgo_import_static __sanitizer_cov_pcs_init
var __sanitizer_cov_pcs_init byte

//go:linkname __sanitizer_weak_hook_strcmp __sanitizer_weak_hook_strcmp
//go:cgo_import_static __sanitizer_weak_hook_strcmp
var __sanitizer_weak_hook_strcmp byte
<commit_msg>runtime: fix stack split at bad time when fuzzing<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build libfuzzer

package runtime

import "unsafe"

func libfuzzerCallWithTwoByteBuffers(fn, start, end *byte)
func libfuzzerCallTraceIntCmp(fn *byte, arg0, arg1, fakePC uintptr)
func libfuzzerCall4(fn *byte, fakePC uintptr, s1, s2 unsafe.Pointer, result uintptr)

// Keep in sync with the definition of ret_sled in src/runtime/libfuzzer_amd64.s
const retSledSize = 512

// In libFuzzer mode, the compiler inserts calls to libfuzzerTraceCmpN and libfuzzerTraceConstCmpN
// (where N can be 1, 2, 4, or 8) for encountered integer comparisons in the code to be instrumented.
// This may result in these functions having callers that are nosplit. That is why they must be nosplit.
//
//go:nosplit
func libfuzzerTraceCmp1(arg0, arg1 uint8, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_cmp1, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

//go:nosplit
func libfuzzerTraceCmp2(arg0, arg1 uint16, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_cmp2, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

//go:nosplit
func libfuzzerTraceCmp4(arg0, arg1 uint32, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_cmp4, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

//go:nosplit
func libfuzzerTraceCmp8(arg0, arg1 uint64, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_cmp8, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

//go:nosplit
func libfuzzerTraceConstCmp1(arg0, arg1 uint8, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_const_cmp1, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

//go:nosplit
func libfuzzerTraceConstCmp2(arg0, arg1 uint16, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_const_cmp2, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

//go:nosplit
func libfuzzerTraceConstCmp4(arg0, arg1 uint32, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_const_cmp4, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

//go:nosplit
func libfuzzerTraceConstCmp8(arg0, arg1 uint64, fakePC int) {
	fakePC = fakePC % retSledSize
	libfuzzerCallTraceIntCmp(&__sanitizer_cov_trace_const_cmp8, uintptr(arg0), uintptr(arg1), uintptr(fakePC))
}

var pcTables []byte

func init() {
	libfuzzerCallWithTwoByteBuffers(&__sanitizer_cov_8bit_counters_init, &__start___sancov_cntrs, &__stop___sancov_cntrs)
	start := unsafe.Pointer(&__start___sancov_cntrs)
	end := unsafe.Pointer(&__stop___sancov_cntrs)

	// PC tables are arrays of ptr-sized integers representing pairs [PC,PCFlags] for every instrumented block.
	// The number of PCs and PCFlags is the same as the number of 8-bit counters. Each PC table entry has
	// the size of two ptr-sized integers. We allocate one more byte than what we actually need so that we can
	// get a pointer representing the end of the PC table array.
	size := (uintptr(end)-uintptr(start))*unsafe.Sizeof(uintptr(0))*2 + 1
	pcTables = make([]byte, size)
	libfuzzerCallWithTwoByteBuffers(&__sanitizer_cov_pcs_init, &pcTables[0], &pcTables[size-1])
}

// We call libFuzzer's __sanitizer_weak_hook_strcmp function which takes the
// following four arguments:
//
//  1. caller_pc: location of string comparison call site
//  2. s1: first string used in the comparison
//  3. s2: second string used in the comparison
//  4. result: an integer representing the comparison result. 0 indicates
//     equality (comparison will ignored by libfuzzer), non-zero indicates a
//     difference (comparison will be taken into consideration).
func libfuzzerHookStrCmp(s1, s2 string, fakePC int) {
	if s1 != s2 {
		libfuzzerCall4(&__sanitizer_weak_hook_strcmp, uintptr(fakePC), cstring(s1), cstring(s2), uintptr(1))
	}
	// if s1 == s2 we could call the hook with a last argument of 0 but this is unnecessary since this case will be then
	// ignored by libfuzzer
}

// This function has now the same implementation as libfuzzerHookStrCmp because we lack better checks
// for case-insensitive string equality in the runtime package.
func libfuzzerHookEqualFold(s1, s2 string, fakePC int) {
	if s1 != s2 {
		libfuzzerCall4(&__sanitizer_weak_hook_strcmp, uintptr(fakePC), cstring(s1), cstring(s2), uintptr(1))
	}
}

//go:linkname __sanitizer_cov_trace_cmp1 __sanitizer_cov_trace_cmp1
//go:cgo_import_static __sanitizer_cov_trace_cmp1
var __sanitizer_cov_trace_cmp1 byte

//go:linkname __sanitizer_cov_trace_cmp2 __sanitizer_cov_trace_cmp2
//go:cgo_import_static __sanitizer_cov_trace_cmp2
var __sanitizer_cov_trace_cmp2 byte

//go:linkname __sanitizer_cov_trace_cmp4 __sanitizer_cov_trace_cmp4
//go:cgo_import_static __sanitizer_cov_trace_cmp4
var __sanitizer_cov_trace_cmp4 byte

//go:linkname __sanitizer_cov_trace_cmp8 __sanitizer_cov_trace_cmp8
//go:cgo_import_static __sanitizer_cov_trace_cmp8
var __sanitizer_cov_trace_cmp8 byte

//go:linkname __sanitizer_cov_trace_const_cmp1 __sanitizer_cov_trace_const_cmp1
//go:cgo_import_static __sanitizer_cov_trace_const_cmp1
var __sanitizer_cov_trace_const_cmp1 byte

//go:linkname __sanitizer_cov_trace_const_cmp2 __sanitizer_cov_trace_const_cmp2
//go:cgo_import_static __sanitizer_cov_trace_const_cmp2
var __sanitizer_cov_trace_const_cmp2 byte

//go:linkname __sanitizer_cov_trace_const_cmp4 __sanitizer_cov_trace_const_cmp4
//go:cgo_import_static __sanitizer_cov_trace_const_cmp4
var __sanitizer_cov_trace_const_cmp4 byte

//go:linkname __sanitizer_cov_trace_const_cmp8 __sanitizer_cov_trace_const_cmp8
//go:cgo_import_static __sanitizer_cov_trace_const_cmp8
var __sanitizer_cov_trace_const_cmp8 byte

//go:linkname __sanitizer_cov_8bit_counters_init __sanitizer_cov_8bit_counters_init
//go:cgo_import_static __sanitizer_cov_8bit_counters_init
var __sanitizer_cov_8bit_counters_init byte

//go:linkname __start___sancov_cntrs __start___sancov_cntrs
//go:cgo_import_static __start___sancov_cntrs
var __start___sancov_cntrs byte

//go:linkname __stop___sancov_cntrs __stop___sancov_cntrs
//go:cgo_import_static __stop___sancov_cntrs
var __stop___sancov_cntrs byte

//go:linkname __sanitizer_cov_pcs_init __sanitizer_cov_pcs_init
//go:cgo_import_static __sanitizer_cov_pcs_init
var __sanitizer_cov_pcs_init byte

//go:linkname __sanitizer_weak_hook_strcmp __sanitizer_weak_hook_strcmp
//go:cgo_import_static __sanitizer_weak_hook_strcmp
var __sanitizer_weak_hook_strcmp byte
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"github.com/boltdb/bolt"
	"github.com/codegangsta/cli"
	kval "github.com/kval-access-language/kval-boltdb"
	"github.com/olekukonko/tablewriter"
)

func main() {
	var file string
	var bucket string

	cli.AppHelpTemplate = `NAME:
  {{.Name}} - {{.Usage}}

VERSION:
  {{.Version}}

USAGE:
  {{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}

GLOBAL OPTIONS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
AUTHOR:
  {{range .Authors}}{{ . }}{{end}}
`
	app := cli.NewApp()
	app.Name = "bolter"
	app.Usage = "view boltdb file in your terminal"
	app.Version = "1.0.0"
	app.Author = "Hasit Mistry"
	app.Email = "hasitnm@gmail.com"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "file, f",
			Usage:       "boltdb `FILE` to view",
			Destination: &file,
		},
		cli.StringFlag{
			Name:        "bucket, b",
			Usage:       "boltdb `BUCKET` to view",
			Destination: &bucket,
		},
	}
	app.Action = func(c *cli.Context) error {
		if file == "" {
			cli.ShowAppHelp(c)
			return nil
		}

		var i impl
		i = impl{fmt: &tableFormatter{}}
		if _, err := os.Stat(file); os.IsNotExist(err) {
			log.Fatal(err)
			return err
		}
		i.initDB(file)
		defer i.DB.Close()

		i.readInput()

		return nil
	}
	app.Run(os.Args)
}

func (i *impl) readInput() {
	fmt.Fprint(os.Stdout, "DB Layout:\n\n")
	i.listBuckets()
	scanner := bufio.NewScanner(os.Stdin)
	fmt.Fprint(os.Stdout, "\nEnter bucket to explore (q to quit, m to print):\n\n")
	for scanner.Scan() {
		bucket := scanner.Text()
		fmt.Fprintln(os.Stdout, "")
		switch(bucket) {
		case "q":
			fallthrough
		case "quit":
			return
		case "":
			i.listBuckets()
		default:
			i.listBucketItems(bucket)
		}
	}
}

type formatter interface {
	DumpBuckets([]bucket)
	DumpBucketItems(string, []item)
}

type impl struct {
	KV  kval.Kvalboltdb
	DB  *bolt.DB
	fmt formatter
	loc string		// where we are in the structure
}

type item struct {
	Key   string `json:"Key"`
	Value string `json:"Value"`
}

type bucket struct {
	Name string `json:"Name"`
}

func (i *impl) initDB(file string) {
	var err error
	// Read-only permission, should be equiv. bolt.Open(file, 0400, nil)
	i.KV, err = kval.Connect(file)
	i.DB = kval.GetBolt(i.KV)
	if err != nil {
		log.Fatal(err)
	}
}

func (i *impl) updateLoc(bucket string) string {
	if i.loc == "" {
		i.loc = "GET " + bucket
	} else {
		i.loc = i.loc + " >> " + bucket
	}
	return i.loc
}

func (i *impl) listBucketItems(bucket string) {
	items := []item{}
	getItems := i.updateLoc(bucket)
	fmt.Fprintf(os.Stderr, "Query: " + i.loc + "\n\n")
	res, err := kval.Query(i.KV, getItems)
	if err != nil {
		if err.Error() != "Cannot GOTO bucket, bucket not found" {
			log.Fatal(err)
		} else {
			fmt.Fprintln(os.Stderr, "Bucket not found")
			fmt.Println(getItems)
		}
	}
	for k, v := range res.Result {
		if v == kval.Nestedbucket {
			k = k + "*"
			v = ""
		}
		items = append(items, item{Key: string(k), Value: string(v)})
	}

	i.fmt.DumpBucketItems(bucket, items)
}

func (i *impl) listBuckets() {
	buckets := []bucket{}
	err := i.DB.View(func(tx *bolt.Tx) error {
		return tx.ForEach(func(bucketname []byte, _ *bolt.Bucket) error {
			buckets = append(buckets, bucket{Name: string(bucketname)})
			return nil
		})
	})
	if err != nil {
		log.Fatal(err)
	}
	i.fmt.DumpBuckets(buckets)
}

type tableFormatter struct{}

func (tf tableFormatter) DumpBuckets(buckets []bucket) {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Buckets"})
	for _, b := range buckets {
		row := []string{b.Name}
		table.Append(row)
	}
	table.Render()
	fmt.Println()
}

func (tf tableFormatter) DumpBucketItems(bucket string, items []item) {
	fmt.Printf("Bucket: %s\n", bucket)
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Key", "Value"})
	for _, item := range items {
		row := []string{item.Key, item.Value}
		table.Append(row)
	}
	table.Render()
	fmt.Println()
}

<commit_msg>[feature] Navigate backwards.<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/boltdb/bolt"
	"github.com/codegangsta/cli"
	kval "github.com/kval-access-language/kval-boltdb"
	"github.com/olekukonko/tablewriter"
	"log"
	"os"
	"strings"
)

func main() {
	var file string
	var bucket string

	cli.AppHelpTemplate = `NAME:
  {{.Name}} - {{.Usage}}

VERSION:
  {{.Version}}

USAGE:
  {{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}

GLOBAL OPTIONS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
AUTHOR:
  {{range .Authors}}{{ . }}{{end}}
`
	app := cli.NewApp()
	app.Name = "bolter"
	app.Usage = "view boltdb file in your terminal"
	app.Version = "1.0.0"
	app.Author = "Hasit Mistry"
	app.Email = "hasitnm@gmail.com"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "file, f",
			Usage:       "boltdb `FILE` to view",
			Destination: &file,
		},
		cli.StringFlag{
			Name:        "bucket, b",
			Usage:       "boltdb `BUCKET` to view",
			Destination: &bucket,
		},
	}
	app.Action = func(c *cli.Context) error {
		if file == "" {
			cli.ShowAppHelp(c)
			return nil
		}

		var i impl
		i = impl{fmt: &tableFormatter{}}
		if _, err := os.Stat(file); os.IsNotExist(err) {
			log.Fatal(err)
			return err
		}
		i.initDB(file)
		defer i.DB.Close()

		i.readInput()

		return nil
	}
	app.Run(os.Args)
}

func (i *impl) readInput() {
	i.listBuckets()
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		bucket := scanner.Text()
		fmt.Fprintln(os.Stdout, "")
		switch bucket {
		case "q":
			fallthrough
		case "quit":
			return
		case " ":
			// TODO: Change KVAL to get first record...
			if !strings.Contains(i.loc, "GET") || !strings.Contains(i.loc, ">>") {
				fmt.Println("Going back...")
				i.loc = ""
				i.listBuckets()
			} else {
				i.listBucketItems(bucket, true)
			}
		case "":
			i.listBuckets()
		default:
			i.listBucketItems(bucket, false)
		}
		bucket = ""
	}
}

type formatter interface {
	DumpBuckets([]bucket)
	DumpBucketItems(string, []item)
}

type impl struct {
	KV  kval.Kvalboltdb
	DB  *bolt.DB
	fmt formatter
	loc string // where we are in the structure
}

type item struct {
	Key   string `json:"Key"`
	Value string `json:"Value"`
}

type bucket struct {
	Name string `json:"Name"`
}

func (i *impl) initDB(file string) {
	var err error
	// Read-only permission, should be equiv. bolt.Open(file, 0400, nil)
	i.KV, err = kval.Connect(file)
	i.DB = kval.GetBolt(i.KV)
	if err != nil {
		log.Fatal(err)
	}
}

func (i *impl) updateLoc(bucket string, goBack bool) string {

	// handle goback
	if goBack {
		s := strings.Split(i.loc, ">>")
		i.loc = strings.Join(s[:len(s)-1], " ")
		fmt.Println(i.loc)
		return i.loc
	}

	// handle loc on merit...
	if i.loc == "" {
		i.loc = "GET " + bucket
	} else {
		i.loc = i.loc + " >> " + bucket
	}
	return i.loc
}

func (i *impl) listBucketItems(bucket string, goBack bool) {
	items := []item{}
	getItems := i.updateLoc(bucket, goBack)
	fmt.Fprintf(os.Stderr, "Query: "+i.loc+"\n\n")
	res, err := kval.Query(i.KV, getItems)
	if err != nil {
		if err.Error() != "Cannot GOTO bucket, bucket not found" {
			log.Fatal(err)
		} else {
			fmt.Fprintln(os.Stderr, "Bucket not found")
			fmt.Println(getItems)
		}
	}
	for k, v := range res.Result {
		if v == kval.Nestedbucket {
			k = k + "*"
			v = ""
		}
		items = append(items, item{Key: string(k), Value: string(v)})
	}
	i.fmt.DumpBucketItems(bucket, items)
	fmt.Fprint(os.Stdout, "Enter bucket to explore (q to quit, SPACE to go back, ENTER to reset):\n\n")
}

func (i *impl) listBuckets() {
	buckets := []bucket{}
	err := i.DB.View(func(tx *bolt.Tx) error {
		return tx.ForEach(func(bucketname []byte, _ *bolt.Bucket) error {
			buckets = append(buckets, bucket{Name: string(bucketname)})
			return nil
		})
	})
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprint(os.Stdout, "DB Layout:\n\n")
	i.fmt.DumpBuckets(buckets)
	fmt.Fprint(os.Stdout, "Enter bucket to explore (q to quit, SPACE to go back, ENTER to reset):\n\n")
}

type tableFormatter struct{}

func (tf tableFormatter) DumpBuckets(buckets []bucket) {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Buckets"})
	for _, b := range buckets {
		row := []string{b.Name}
		table.Append(row)
	}
	table.Render()
	fmt.Println()
}

func (tf tableFormatter) DumpBucketItems(bucket string, items []item) {
	fmt.Printf("Bucket: %s\n", bucket)
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Key", "Value"})
	for _, item := range items {
		row := []string{item.Key, item.Value}
		table.Append(row)
	}
	table.Render()
	fmt.Println()
}
<|endoftext|>


<commit_before>package client

import (
	"github.com/dedis/prifi/prifi-lib/net"
	"gopkg.in/dedis/onet.v2/log"
)

/*
* Received_REL_ALL_DISRUPTION_REVEAL handles REL_ALL_DISRUPTION_REVEAL messages.
* The method calls a function from the DCNet to regenerate the bits from roundID in position BitPos
* The result is sent to the relay.
 */
func (p *PriFiLibClientInstance) Received_REL_ALL_DISRUPTION_REVEAL(msg net.REL_ALL_DISRUPTION_REVEAL) error {
	log.Lvl1("Disruption Phase 1: Received de-anonymization query for round", msg.RoundID, "bit pos", msg.BitPos)

	// TODO: check the NIZK

	bitMap := p.clientState.DCNet.GetBitsOfRound(int32(msg.RoundID), int32(msg.BitPos))
	//send the data to the relay
	toSend := &net.CLI_REL_DISRUPTION_REVEAL{
		ClientID: p.clientState.ID,
		Bits:     bitMap,
	}

	//LB->CV: if ForceDisruptionOnRound, continue lying here!
	if true && p.clientState.ID == 0 {
		log.Lvl1("Disruption: Malicious client cheating again, old value", bitMap, "(new value right below)")
		trusteeToAccuse := 0
		// pretend the PRG told me to output a 1, and the trustee is lying with its 0
		bitMap[trusteeToAccuse] = 1
	}
	log.Lvl1("Disruption: Sending previous round to relay (Round: ", msg.RoundID, ", bit position:", msg.BitPos, "), value", bitMap)

	p.messageSender.SendToRelayWithLog(toSend, "")
	return nil
}

/*
* Received_REL_ALL_REVEAL_SHARED_SECRETS handles REL_ALL_REVEAL_SHARED_SECRETS messages.
* The method gets the shared secret and sends it to the relay.
 */
func (p *PriFiLibClientInstance) Received_REL_ALL_REVEAL_SHARED_SECRETS(msg net.REL_ALL_REVEAL_SHARED_SECRETS) error {
	log.Lvl1("Disruption Phase 2: Received a reveal secret message for trustee", msg.EntityID)
	// CARLOS TODO: NIZK
	// TODO: check that the relay asks for the correct entity, and not a honest entity. There should be a signature check on the TRU_REL_DISRUPTION_REVEAL the relay received (and forwarded to the client)
	secret := p.clientState.sharedSecrets[msg.EntityID]
	toSend := &net.CLI_REL_SHARED_SECRET{
		ClientID:  p.clientState.ID,
		TrusteeID: msg.EntityID,
		Secret:    secret,
		NIZK:      make([]byte, 0)}
	p.messageSender.SendToRelayWithLog(toSend, "Sent secret to relay")
	log.Lvl1("Reveling secret with trustee", msg.EntityID)
	return nil
}
<commit_msg>Add pause to see a successful check of SECRET_REVEAL message<commit_after>package client

import (
	"github.com/dedis/prifi/prifi-lib/net"
	"gopkg.in/dedis/onet.v2/log"
	"time"
)

/*
* Received_REL_ALL_DISRUPTION_REVEAL handles REL_ALL_DISRUPTION_REVEAL messages.
* The method calls a function from the DCNet to regenerate the bits from roundID in position BitPos
* The result is sent to the relay.
 */
func (p *PriFiLibClientInstance) Received_REL_ALL_DISRUPTION_REVEAL(msg net.REL_ALL_DISRUPTION_REVEAL) error {
	log.Lvl1("Disruption Phase 1: Received de-anonymization query for round", msg.RoundID, "bit pos", msg.BitPos)

	// TODO: check the NIZK

	bitMap := p.clientState.DCNet.GetBitsOfRound(int32(msg.RoundID), int32(msg.BitPos))
	//send the data to the relay
	toSend := &net.CLI_REL_DISRUPTION_REVEAL{
		ClientID: p.clientState.ID,
		Bits:     bitMap,
	}

	//if ForceDisruptionOnRound, continue lying here!
	//LB->CV: have this as a param in prifi.toml
	ForceDisruptionOnRound := 3
	if ForceDisruptionOnRound > -1 && p.clientState.ID == 0 {
		log.Lvl1("Disruption: Malicious client cheating again, old value", bitMap, "(new value right below)")
		trusteeToAccuse := 0
		// pretend the PRG told me to output a 1, and the trustee is lying with its 0
		bitMap[trusteeToAccuse] = 1
	}
	log.Lvl1("Disruption: Sending previous round to relay (Round: ", msg.RoundID, ", bit position:", msg.BitPos, "), value", bitMap)

	p.messageSender.SendToRelayWithLog(toSend, "")
	return nil
}

/*
* Received_REL_ALL_REVEAL_SHARED_SECRETS handles REL_ALL_REVEAL_SHARED_SECRETS messages.
* The method gets the shared secret and sends it to the relay.
 */
func (p *PriFiLibClientInstance) Received_REL_ALL_REVEAL_SHARED_SECRETS(msg net.REL_ALL_REVEAL_SHARED_SECRETS) error {
	log.Lvl1("Disruption Phase 2: Received a reveal secret message for trustee", msg.EntityID)
	// CARLOS TODO: NIZK
	// TODO: check that the relay asks for the correct entity, and not a honest entity. There should be a signature check on the TRU_REL_DISRUPTION_REVEAL the relay received (and forwarded to the client)
	secret := p.clientState.sharedSecrets[msg.EntityID]
	toSend := &net.CLI_REL_SHARED_SECRET{
		ClientID:  p.clientState.ID,
		TrusteeID: msg.EntityID,
		Secret:    secret,
		NIZK:      make([]byte, 0)}

	//LB->CV: have this as a param in prifi.toml
	ForceDisruptionOnRound := 3
	if ForceDisruptionOnRound > -1 && p.clientState.ID == 0 {
		//this client is hesitant to answer as he will get caught
		time.Sleep(1 * time.Second)
		// this is just to let the honest trustee answer and see what happens
	}

	p.messageSender.SendToRelayWithLog(toSend, "Sent secret to relay")
	log.Lvl1("Reveling secret with trustee", msg.EntityID)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package eval

import (
	"go/token"
	"log"
	"reflect"
)

/*
 * Type bridging
 */

var (
	evalTypes   = make(map[reflect.Type]Type)
	nativeTypes = make(map[Type]reflect.Type)
)

// TypeFromNative converts a regular Go type into a the corresponding
// interpreter Type.
func TypeFromNative(t reflect.Type) Type {
	if et, ok := evalTypes[t]; ok {
		return et
	}

	var nt *NamedType
	if t.Name() != "" {
		name := t.PkgPath() + "·" + t.Name()
		nt = &NamedType{token.NoPos, name, nil, true, make(map[string]Method)}
		evalTypes[t] = nt
	}

	var et Type
	switch t.Kind() {
	case reflect.Bool:
		et = BoolType

	case reflect.Float32:
		et = Float32Type
	case reflect.Float64:
		et = Float64Type

	case reflect.Int16:
		et = Int16Type
	case reflect.Int32:
		et = Int32Type
	case reflect.Int64:
		et = Int64Type
	case reflect.Int8:
		et = Int8Type
	case reflect.Int:
		et = IntType

	case reflect.Uint16:
		et = Uint16Type
	case reflect.Uint32:
		et = Uint32Type
	case reflect.Uint64:
		et = Uint64Type
	case reflect.Uint8:
		et = Uint8Type
	case reflect.Uint:
		et = UintType
	case reflect.Uintptr:
		et = UintptrType

	case reflect.String:
		et = StringType
	case reflect.Array:
		et = NewArrayType(int64(t.Len()), TypeFromNative(t.Elem()))
	case reflect.Chan:
		log.Panicf("%T not implemented", t)
	case reflect.Func:
		nin := t.NumIn()
		// Variadic functions have DotDotDotType at the end
		variadic := t.IsVariadic()
		if variadic {
			nin--
		}
		in := make([]Type, nin)
		for i := range in {
			in[i] = TypeFromNative(t.In(i))
		}
		out := make([]Type, t.NumOut())
		for i := range out {
			out[i] = TypeFromNative(t.Out(i))
		}
		et = NewFuncType(in, variadic, out)
	case reflect.Interface:
		log.Panicf("%T not implemented", t)
	case reflect.Map:
		log.Panicf("%T not implemented", t)
	case reflect.Ptr:
		et = NewPtrType(TypeFromNative(t.Elem()))
	case reflect.Slice:
		et = NewSliceType(TypeFromNative(t.Elem()))
	case reflect.Struct:
		n := t.NumField()
		fields := make([]StructField, n)
		for i := 0; i < n; i++ {
			sf := t.Field(i)
			// TODO(austin) What to do about private fields?
			fields[i].Name = sf.Name
			fields[i].Type = TypeFromNative(sf.Type)
			fields[i].Anonymous = sf.Anonymous
		}
		et = NewStructType(fields)
	case reflect.UnsafePointer:
		log.Panicf("%T not implemented", t)
	default:
		log.Panicf("unexpected reflect.Type: %T", t)
	}

	if nt != nil {
		if _, ok := et.(*NamedType); !ok {
			nt.Complete(et)
			et = nt
		}
	}

	nativeTypes[et] = t
	evalTypes[t] = et

	return et
}

// TypeOfNative returns the interpreter Type of a regular Go value.
func TypeOfNative(v interface{}) Type { return TypeFromNative(reflect.TypeOf(v)) }

/*
 * Function bridging
 */

type nativeFunc struct {
	fn      func(*Thread, []Value, []Value)
	in, out int
}

func (f *nativeFunc) NewFrame() *Frame {
	vars := make([]Value, f.in+f.out)
	return &Frame{nil, vars}
}

func (f *nativeFunc) Call(t *Thread) { f.fn(t, t.f.Vars[0:f.in], t.f.Vars[f.in:f.in+f.out]) }

// FuncFromNative creates an interpreter function from a native
// function that takes its in and out arguments as slices of
// interpreter Value's.  While somewhat inconvenient, this avoids
// value marshalling.
func FuncFromNative(fn func(*Thread, []Value, []Value), t *FuncType) FuncValue {
	return &funcV{&nativeFunc{fn, len(t.In), len(t.Out)}}
}

// FuncFromNativeTyped is like FuncFromNative, but constructs the
// function type from a function pointer using reflection.  Typically,
// the type will be given as a nil pointer to a function with the
// desired signature.
func FuncFromNativeTyped(fn func(*Thread, []Value, []Value), t interface{}) (*FuncType, FuncValue) {
	ft := TypeOfNative(t).(*FuncType)
	return ft, FuncFromNative(fn, ft)
}
<commit_msg>bridge: first stab at map<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package eval

import (
	"go/token"
	"log"
	"reflect"
)

/*
 * Type bridging
 */

var (
	evalTypes   = make(map[reflect.Type]Type)
	nativeTypes = make(map[Type]reflect.Type)
)

// TypeFromNative converts a regular Go type into a the corresponding
// interpreter Type.
func TypeFromNative(t reflect.Type) Type {
	if et, ok := evalTypes[t]; ok {
		return et
	}

	var nt *NamedType
	if t.Name() != "" {
		name := t.PkgPath() + "·" + t.Name()
		nt = &NamedType{token.NoPos, name, nil, true, make(map[string]Method)}
		evalTypes[t] = nt
	}

	var et Type
	switch t.Kind() {
	case reflect.Bool:
		et = BoolType

	case reflect.Float32:
		et = Float32Type
	case reflect.Float64:
		et = Float64Type

	case reflect.Int16:
		et = Int16Type
	case reflect.Int32:
		et = Int32Type
	case reflect.Int64:
		et = Int64Type
	case reflect.Int8:
		et = Int8Type
	case reflect.Int:
		et = IntType

	case reflect.Uint16:
		et = Uint16Type
	case reflect.Uint32:
		et = Uint32Type
	case reflect.Uint64:
		et = Uint64Type
	case reflect.Uint8:
		et = Uint8Type
	case reflect.Uint:
		et = UintType
	case reflect.Uintptr:
		et = UintptrType

	case reflect.String:
		et = StringType
	case reflect.Array:
		et = NewArrayType(int64(t.Len()), TypeFromNative(t.Elem()))
	case reflect.Chan:
		log.Panicf("%T not implemented", t)
	case reflect.Func:
		nin := t.NumIn()
		// Variadic functions have DotDotDotType at the end
		variadic := t.IsVariadic()
		if variadic {
			nin--
		}
		in := make([]Type, nin)
		for i := range in {
			in[i] = TypeFromNative(t.In(i))
		}
		out := make([]Type, t.NumOut())
		for i := range out {
			out[i] = TypeFromNative(t.Out(i))
		}
		et = NewFuncType(in, variadic, out)
	case reflect.Interface:
		log.Panicf("%T not implemented", t)
	case reflect.Map:
		et = NewMapType(TypeFromNative(t.Key()), TypeFromNative(t.Elem()))
	case reflect.Ptr:
		et = NewPtrType(TypeFromNative(t.Elem()))
	case reflect.Slice:
		et = NewSliceType(TypeFromNative(t.Elem()))
	case reflect.Struct:
		n := t.NumField()
		fields := make([]StructField, n)
		for i := 0; i < n; i++ {
			sf := t.Field(i)
			// TODO(austin) What to do about private fields?
			fields[i].Name = sf.Name
			fields[i].Type = TypeFromNative(sf.Type)
			fields[i].Anonymous = sf.Anonymous
		}
		et = NewStructType(fields)
	case reflect.UnsafePointer:
		log.Panicf("%T not implemented", t)
	default:
		log.Panicf("unexpected reflect.Type: %T", t)
	}

	if nt != nil {
		if _, ok := et.(*NamedType); !ok {
			nt.Complete(et)
			et = nt
		}
	}

	nativeTypes[et] = t
	evalTypes[t] = et

	return et
}

// TypeOfNative returns the interpreter Type of a regular Go value.
func TypeOfNative(v interface{}) Type { return TypeFromNative(reflect.TypeOf(v)) }

/*
 * Function bridging
 */

type nativeFunc struct {
	fn      func(*Thread, []Value, []Value)
	in, out int
}

func (f *nativeFunc) NewFrame() *Frame {
	vars := make([]Value, f.in+f.out)
	return &Frame{nil, vars}
}

func (f *nativeFunc) Call(t *Thread) { f.fn(t, t.f.Vars[0:f.in], t.f.Vars[f.in:f.in+f.out]) }

// FuncFromNative creates an interpreter function from a native
// function that takes its in and out arguments as slices of
// interpreter Value's.  While somewhat inconvenient, this avoids
// value marshalling.
func FuncFromNative(fn func(*Thread, []Value, []Value), t *FuncType) FuncValue {
	return &funcV{&nativeFunc{fn, len(t.In), len(t.Out)}}
}

// FuncFromNativeTyped is like FuncFromNative, but constructs the
// function type from a function pointer using reflection.  Typically,
// the type will be given as a nil pointer to a function with the
// desired signature.
func FuncFromNativeTyped(fn func(*Thread, []Value, []Value), t interface{}) (*FuncType, FuncValue) {
	ft := TypeOfNative(t).(*FuncType)
	return ft, FuncFromNative(fn, ft)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"encoding/json"
	"fmt"
	"net/http"

	"appengine"
	"appengine/datastore"
)

func init() {
	http.HandleFunc("/updatebenchmark", updateBenchmark)
}

func updateBenchmark(w http.ResponseWriter, r *http.Request) {
	if !appengine.IsDevAppServer() {
		fmt.Fprint(w, "Update must not run on real server.")
		return
	}

	if r.Method != "POST" {
		fmt.Fprintf(w, "bad request method")
		return
	}

	c := contextForRequest(r)
	if !validKey(c, r.FormValue("key"), r.FormValue("builder")) {
		fmt.Fprintf(w, "bad builder/key")
		return
	}

	defer r.Body.Close()
	var hashes []string
	if err := json.NewDecoder(r.Body).Decode(&hashes); err != nil {
		fmt.Fprintf(w, "failed to decode request: %v", err)
		return
	}

	ncommit := 0
	nrun := 0
	tx := func(c appengine.Context) error {
		var cr *CommitRun
		for _, hash := range hashes {
			// Update Commit.
			com := &Commit{Hash: hash}
			err := datastore.Get(c, com.Key(c), com)
			if err != nil && err != datastore.ErrNoSuchEntity {
				return fmt.Errorf("fetching Commit: %v", err)
			}
			if err == datastore.ErrNoSuchEntity {
				continue
			}
			com.NeedsBenchmarking = true
			if err := putCommit(c, com); err != nil {
				return err
			}
			ncommit++

			// create PerfResult
			res := &PerfResult{CommitHash: com.Hash, CommitNum: com.Num}
			err = datastore.Get(c, res.Key(c), res)
			if err != nil && err != datastore.ErrNoSuchEntity {
				return fmt.Errorf("fetching PerfResult: %v", err)
			}
			if err == datastore.ErrNoSuchEntity {
				if _, err := datastore.Put(c, res.Key(c), res); err != nil {
					return fmt.Errorf("putting PerfResult: %v", err)
				}
			}

			// Update CommitRun.
			if cr != nil && cr.StartCommitNum != com.Num/PerfRunLength*PerfRunLength {
				if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
					return fmt.Errorf("putting CommitRun: %v", err)
				}
				nrun++
				cr = nil
			}
			if cr == nil {
				var err error
				cr, err = GetCommitRun(c, com.Num)
				if err != nil {
					return fmt.Errorf("getting CommitRun: %v", err)
				}
			}
			if com.Num < cr.StartCommitNum || com.Num >= cr.StartCommitNum+PerfRunLength {
				return fmt.Errorf("commit num %v out of range [%v, %v)", com.Num, cr.StartCommitNum, cr.StartCommitNum+PerfRunLength)
			}
			idx := com.Num - cr.StartCommitNum
			cr.Hash[idx] = com.Hash
			cr.User[idx] = shortDesc(com.User)
			cr.Desc[idx] = shortDesc(com.Desc)
			cr.Time[idx] = com.Time
			cr.NeedsBenchmarking[idx] = com.NeedsBenchmarking
		}
		if cr != nil {
			if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
				return fmt.Errorf("putting CommitRun: %v", err)
			}
			nrun++
		}
		return nil
	}
	if err := datastore.RunInTransaction(c, tx, nil); err != nil {
		fmt.Fprintf(w, "failed to execute tx: %v", err)
		return
	}
	fmt.Fprintf(w, "OK (updated %v commits and %v commit runs)", ncommit, nrun)
}
<commit_msg>dashboard: update update script to delete Commit.PerfResult This is required to repair perf data in the datastore. Update golang/go#8930.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"encoding/json"
	"fmt"
	"net/http"

	"appengine"
	"appengine/datastore"
)

func init() {
	http.HandleFunc("/updatebenchmark", updateBenchmark)
}

func updateBenchmark(w http.ResponseWriter, r *http.Request) {
	if !appengine.IsDevAppServer() {
		fmt.Fprint(w, "Update must not run on real server.")
		return
	}

	if r.Method != "POST" {
		fmt.Fprintf(w, "bad request method")
		return
	}

	c := contextForRequest(r)
	if !validKey(c, r.FormValue("key"), r.FormValue("builder")) {
		fmt.Fprintf(w, "bad builder/key")
		return
	}

	defer r.Body.Close()
	var hashes []string
	if err := json.NewDecoder(r.Body).Decode(&hashes); err != nil {
		fmt.Fprintf(w, "failed to decode request: %v", err)
		return
	}

	ncommit := 0
	nrun := 0
	tx := func(c appengine.Context) error {
		var cr *CommitRun
		for _, hash := range hashes {
			// Update Commit.
			com := &Commit{Hash: hash}
			err := datastore.Get(c, com.Key(c), com)
			if err != nil && err != datastore.ErrNoSuchEntity {
				return fmt.Errorf("fetching Commit: %v", err)
			}
			if err == datastore.ErrNoSuchEntity {
				continue
			}
			com.NeedsBenchmarking = true
			com.ResultData = nil
			if err := putCommit(c, com); err != nil {
				return err
			}
			ncommit++

			// create PerfResult
			res := &PerfResult{CommitHash: com.Hash, CommitNum: com.Num}
			err = datastore.Get(c, res.Key(c), res)
			if err != nil && err != datastore.ErrNoSuchEntity {
				return fmt.Errorf("fetching PerfResult: %v", err)
			}
			if err == datastore.ErrNoSuchEntity {
				if _, err := datastore.Put(c, res.Key(c), res); err != nil {
					return fmt.Errorf("putting PerfResult: %v", err)
				}
			}

			// Update CommitRun.
			if cr != nil && cr.StartCommitNum != com.Num/PerfRunLength*PerfRunLength {
				if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
					return fmt.Errorf("putting CommitRun: %v", err)
				}
				nrun++
				cr = nil
			}
			if cr == nil {
				var err error
				cr, err = GetCommitRun(c, com.Num)
				if err != nil {
					return fmt.Errorf("getting CommitRun: %v", err)
				}
			}
			if com.Num < cr.StartCommitNum || com.Num >= cr.StartCommitNum+PerfRunLength {
				return fmt.Errorf("commit num %v out of range [%v, %v)", com.Num, cr.StartCommitNum, cr.StartCommitNum+PerfRunLength)
			}
			idx := com.Num - cr.StartCommitNum
			cr.Hash[idx] = com.Hash
			cr.User[idx] = shortDesc(com.User)
			cr.Desc[idx] = shortDesc(com.Desc)
			cr.Time[idx] = com.Time
			cr.NeedsBenchmarking[idx] = com.NeedsBenchmarking
		}
		if cr != nil {
			if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
				return fmt.Errorf("putting CommitRun: %v", err)
			}
			nrun++
		}
		return nil
	}
	if err := datastore.RunInTransaction(c, tx, nil); err != nil {
		fmt.Fprintf(w, "failed to execute tx: %v", err)
		return
	}
	fmt.Fprintf(w, "OK (updated %v commits and %v commit runs)", ncommit, nrun)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/gorilla/mux"
)

var ROOT = ""

type TypeMapJSON struct {
	MIME        string `json:"mime"`
	IsDirectory bool   `json:"is_directory"`
	IsHidden    bool   `json:"is_hidden"`
	IsAudio     bool   `json:"is_audio"`
	IsImage     bool   `json:"is_image"`
	IsVideo     bool   `json:"is_video"`
}

type FileInfoJSON struct {
	Name       string      `json:"name"`
	Size       int64       `json:"size"`
	ModifiedAt string      `json:"modified_at"`
	Type       TypeMapJSON `json:"type"`
}

// returns a pair of (filename, MIME type) strings given a `file` output line
func parseMIMEType(fileOutputLine string) (string, string, error) {
	// parse the file program output into a bare MIME type
	mimeString := strings.TrimSpace(fileOutputLine)
	splitIndex := strings.LastIndex(mimeString, ":")

	if len(fileOutputLine) <= 1 || splitIndex <= 0 {
		return "", "", fmt.Errorf("Invalid MIME string: '%s'", fileOutputLine)
	}

	return mimeString[0:splitIndex], strings.TrimSpace(mimeString[splitIndex+1:]), nil
}

// given a path, returns a map of child name to MIME type
func getMIMETypes(root string, files []os.FileInfo) map[string]string {
	// build the command to get all the MIME types at once, for efficiency
	args := []string{"--mime-type", "--dereference", "--preserve-date"}
	for _, file := range files {
		args = append(args, path.Join(root, file.Name()))
	}

	// call `file` for a newline-delimited string of "filename: MIME-type" pairs
	result := make(map[string]string, len(files))
	fileOutput, err := exec.Command("file", args...).Output()
	if err != nil {
		return result
	}

	for _, line := range strings.Split(string(fileOutput), "\n") {
		fileName, mimeType, err := parseMIMEType(line)
		if err == nil {
			// use the full path, so we can handle multiple directories unambiguously
			result[fileName] = mimeType
		}
	}

	return result
}

func getFileOrDirectory(w http.ResponseWriter, r *http.Request) {
	// disable caching since we'll want to keep this listing up-to-date
	w.Header().Add("Cache-Control", "no-cache")

	// ensure the file/directory actually exists
	// TODO: see if it's a file or a directory, don't just assume a directory!
	requestPath, err := normalizePathToRoot(ROOT, mux.Vars(r)["path"])
	fmt.Println("requestPath:", requestPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	children, err := ioutil.ReadDir(requestPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	// get a map of all the MIME types for the given files
	mimeTypes := getMIMETypes(requestPath, children)
	fmt.Println(mimeTypes)

	// list the directory to a JSON response
	var files []FileInfoJSON
	for _, file := range children {
		fileName := file.Name()
		filePath := path.Join(requestPath, fileName)
		mimeType, _ := mimeTypes[filePath]
		isHidden := strings.HasPrefix(fileName, ".")

		// TODO: determine if it's one of these types!
		isAudio, isImage, isVideo := false, false, false

		files = append(files, FileInfoJSON{
			fileName,
			file.Size(),
			file.ModTime().Format("2006-01-02T15:04:05Z"), // ISO 8601
			TypeMapJSON{
				mimeType,
				file.IsDir(),
				isHidden,
				isAudio,
				isImage,
				isVideo,
			},
		})
	}

	json, err := json.Marshal(files)
	if err != nil {
		http.Error(w, "Failed to generate JSON response", 500)
		return
	}

	w.Header().Add("Content-Type", "application/json")
	w.Write(json)
}

// normalizes the path using a root, and returns it. if the path exits the root
// or is otherwise invalid, returns an error.
func normalizePathToRoot(root, child string) (string, error) {
	// clean the path, resolving any ".."s in it
	requestPath := path.Clean(path.Join(root, child))

	// if the path exited the root directory, fail
	relPath, err := filepath.Rel(root, requestPath)
	if err != nil || strings.Index(relPath, "..") >= 0 {
		// keep things vague since someone's probably trying to be sneaky anyway
		return "", fmt.Errorf("Invalid path")
	}

	return requestPath, nil
}

// retrieves/caches/updates a thumbnail file given a path, or returns an error
// if no thumbnail could be geneated.
func getThumbnail(w http.ResponseWriter, r *http.Request) {
	// TODO:
	// * look up the existing file to get its modtime and ensure it exists
	// * see if we have a cached file with the same modtime
	// ** if so, use it
	// ** otherwise, generate a preview
	// *** use graphicsmagick/ffmpeg to generate a preview thumbnail
	// *** store the new file to a mirroed path with the filename plus the modtime
	// * read the cached file and return its contents

	// cache preview thumbnails for a good while to lower load on this tiny
	// server, even if we are caching the preview thumbnails on-disk too.
	w.Header().Add("Cache-Control", "max-age=3600")
	w.Header().Add("Content-Type", "image/jpeg")
}

func main() {
	_, err := exec.LookPath("file")
	if err != nil {
		panic("The `file` executable could not be found; make sure it's installed!")
	}

	if len(os.Args) <= 1 {
		panic("A root directory argument is required")
	}

	ROOT = os.Args[1]

	fmt.Printf("Serving '%s'...\n", ROOT)

	r := mux.NewRouter()
	r.HandleFunc("/files/{path:.*}", getFileOrDirectory)
	r.HandleFunc("/thumbnails/{path:.*}", getThumbnail)

	http.ListenAndServe(":3000", r)
}
<commit_msg>Implement file download<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/gorilla/mux"
)

var ROOT = ""

type FileTypeMapJSON struct {
	MIME        string `json:"mime"`
	IsDirectory bool   `json:"is_directory"`
	IsHidden    bool   `json:"is_hidden"`
	IsAudio     bool   `json:"is_audio"`
	IsImage     bool   `json:"is_image"`
	IsVideo     bool   `json:"is_video"`
}

type FileInfoJSON struct {
	Name       string          `json:"name"`
	Size       int64           `json:"size"`
	ModifiedAt string          `json:"modified_at"`
	Type       FileTypeMapJSON `json:"type"`
}

// returns a pair of (filename, MIME type) strings given a `file` output line
func parseMIMEType(fileOutputLine string) (string, string, error) {
	// parse the file program output into a bare MIME type
	mimeString := strings.TrimSpace(fileOutputLine)
	splitIndex := strings.LastIndex(mimeString, ":")

	if len(fileOutputLine) <= 1 || splitIndex <= 0 {
		return "", "", fmt.Errorf("Invalid MIME string: '%s'", fileOutputLine)
	}

	return mimeString[0:splitIndex], strings.TrimSpace(mimeString[splitIndex+1:]), nil
}

// given a path, returns a map of child name to MIME type
func getMIMETypes(root string, files ...os.FileInfo) map[string]string {
	// build the command to get all the MIME types at once, for efficiency
	args := []string{"--mime-type", "--dereference", "--preserve-date"}
	for _, file := range files {
		args = append(args, path.Join(root, file.Name()))
	}

	// call `file` for a newline-delimited string of "filename: MIME-type" pairs
	result := make(map[string]string, len(files))
	fileOutput, err := exec.Command("file", args...).Output()
	if err != nil {
		return result
	}

	for _, line := range strings.Split(string(fileOutput), "\n") {
		fileName, mimeType, err := parseMIMEType(line)
		if err == nil {
			// use the full path, so we can handle multiple directories unambiguously
			result[fileName] = mimeType
		}
	}

	return result
}

// normalizes the path using a root, and returns it. if the path exits the root
// or is otherwise invalid, returns an error.
func normalizePathToRoot(root, child string) (string, error) {
	// clean the path, resolving any ".."s in it
	requestPath := path.Clean(path.Join(root, child))

	// if the path exited the root directory, fail
	relPath, err := filepath.Rel(root, requestPath)
	if err != nil || strings.Index(relPath, "..") >= 0 {
		// keep things vague since someone's probably trying to be sneaky anyway
		return "", fmt.Errorf("Invalid path")
	}

	return requestPath, nil
}

func getFile(w http.ResponseWriter, r *http.Request) {
	// make sure our path is valid
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathToRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// stat the file so we can set appropriate response headers, and so we can
	// ensure it's a regular file and not a directory.
	fileInfo, err := os.Stat(normalizedPath)
	if err != nil || fileInfo.IsDir() {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	file, err := os.Open(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}
	defer file.Close()

	mimeType := getMIMETypes(filepath.Base(normalizedPath), fileInfo)[normalizedPath]
	w.Header().Add("Content-Type", mimeType)
	w.Header().Add("Cache-Control", "no-cache")

	http.ServeFile(w, r, normalizedPath)
}

func getDirectory(w http.ResponseWriter, r *http.Request) {
	// disable caching since we'll want to keep this listing up-to-date
	w.Header().Add("Cache-Control", "no-cache")

	// ensure the directory actually exists
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathToRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	children, err := ioutil.ReadDir(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	// get a map of all the MIME types for the given files
	mimeTypes := getMIMETypes(normalizedPath, children...)

	// list the directory to a JSON response
	var files []FileInfoJSON
	for _, file := range children {
		fileName := file.Name()
		filePath := path.Join(normalizedPath, fileName)
		mimeType, _ := mimeTypes[filePath]
		isHidden := strings.HasPrefix(fileName, ".")

		// TODO: determine if it's one of these types!
		isAudio, isImage, isVideo := false, false, false

		files = append(files, FileInfoJSON{
			fileName,
			file.Size(),
			file.ModTime().Format("2006-01-02T15:04:05Z"), // ISO 8601
			FileTypeMapJSON{
				mimeType,
				file.IsDir(),
				isHidden,
				isAudio,
				isImage,
				isVideo,
			},
		})
	}

	json, err := json.Marshal(files)
	if err != nil {
		http.Error(w, "Failed to generate JSON response", 500)
		return
	}

	w.Header().Add("Content-Type", "application/json")
	w.Write(json)
}

// retrieves/caches/updates a thumbnail file given a path, or returns an error
// if no thumbnail could be geneated.
func getThumbnail(w http.ResponseWriter, r *http.Request) {
	// TODO:
	// * look up the existing file to get its modtime and ensure it exists
	// * see if we have a cached file with the same modtime
	// ** if so, use it
	// ** otherwise, generate a preview
	// *** use graphicsmagick/ffmpeg to generate a preview thumbnail
	// *** store the new file to a mirroed path with the filename plus the modtime
	// * read the cached file and return its contents

	// cache preview thumbnails for a good while to lower load on this tiny
	// server, even if we are caching the preview thumbnails on-disk too.
	w.Header().Add("Cache-Control", "max-age=3600")
	w.Header().Add("Content-Type", "image/jpeg")
}

func main() {
	// ensure we have all the binaries we need
	requiredBinaries := []string{"file"}
	for _, binary := range requiredBinaries {
		if _, err := exec.LookPath(binary); err != nil {
			log.Panicf("'%s' must be installed and in the PATH\n", binary)
		}
	}

	if len(os.Args) <= 1 {
		panic("A root directory argument is required")
	}

	ROOT = os.Args[1]

	fmt.Printf("Serving '%s'...\n", ROOT)

	r := mux.NewRouter()

	// anything with a trailing `/` indicates a directory; anything that ends
	// without a trailing slash indicates a file.
	r.HandleFunc("/files{path:.*}/", getDirectory)
	r.HandleFunc("/files/{path:.*[^/]$}", getFile)
	r.HandleFunc("/thumbnails/{path:.*[^/]$}", getThumbnail)

	http.ListenAndServe(":3000", r)
}
<|endoftext|>