<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

func main() {
}
<commit_msg>Proof of concept implementation of the `snapshot` command<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

type Hash interface {
	Function() string
	HexContents() string
	ObjectName(parentDir string) (dir string, name string)
	Equal(Hash) bool
	fmt.Stringer
}

type sha256Hash struct {
	Contents string
}

func (h *sha256Hash) Function() string {
	return "sha256"
}

func (h *sha256Hash) HexContents() string {
	return h.Contents
}

func (h *sha256Hash) Equal(other Hash) bool {
	return h.Function() == other.Function() && h.HexContents() == other.HexContents()
}

func (h *sha256Hash) String() string {
	return h.Function() + ":" + h.Contents
}

func (h *sha256Hash) ObjectName(parentDir string) (dir string, name string) {
	return filepath.Join(parentDir, h.Function(), h.Contents[0:2], h.Contents[2:4]), h.Contents[4:]
}

type Path string
type Tree map[Path]Hash

func (t Tree) String() string {
	var lines []string
	for p, h := range t {
		encodedFileName := base64.RawStdEncoding.EncodeToString([]byte(p))
		lines = append(lines, encodedFileName+" "+h.String())
	}
	sort.Strings(lines)
	return strings.Join(lines, "\n")
}

type File struct {
	Mode     string
	Contents Hash
	Parents  []Hash
}

func (f *File) IsDir() bool {
	return strings.HasPrefix(f.Mode, "d")
}

func (f *File) String() string {
	lines := []string{f.Mode, f.Contents.String()}
	for _, parent := range f.Parents {
		lines = append(lines, parent.String())
	}
	return strings.Join(lines, "\n")
}

func ParseFile(reader io.Reader) (*File, error) {
	encoded, err := io.ReadAll(reader)
	if err != nil {
		return nil, fmt.Errorf("failure reading file metadata from the reader: %v", err)
	}
	lines := strings.Split(string(encoded), "\n")
	if len(lines) < 2 {
		return nil, fmt.Errorf("malformed file metadata: %q", encoded)
	}
	var hashes []Hash
	for _, line := range lines[1:] {
		if !strings.HasPrefix(line, "sha256:") {
			return nil, fmt.Errorf("unsupported hash function for %q", line)
		}
		hashes = append(hashes, &sha256Hash{strings.TrimPrefix(line, "sha256:")})
	}
	f := &File{
		Mode:     lines[0],
		Contents: hashes[0],
		Parents:  hashes[1:],
	}
	return f, nil
}

type Archiver interface {
	Exclude(Path) bool
	StoreObject(context.Context, io.Reader) (Hash, error)
	ReadObject(context.Context, Hash) (io.ReadCloser, error)
	StoreFile(context.Context, Path, *File) (Hash, error)
	ReadFile(context.Context, Path) (Hash, *File, error)
}

func SnapshotFileMetadata(ctx context.Context, a Archiver, p Path, info os.FileInfo, contentsHash Hash) (Hash, error) {
	modeLine := info.Mode().String()
	prevFileHash, prev, err := a.ReadFile(ctx, p)
	if err != nil && !os.IsNotExist(err) {
		return nil, fmt.Errorf("failure looking up the previous file snapshot: %v", err)
	}
	if prev != nil && prev.Contents.String() == contentsHash.String() && prev.Mode == modeLine {
		// The file is unchanged from the last snapshot...
		return prevFileHash, nil
	}
	f := &File{
		Contents: contentsHash,
		Mode:     modeLine,
	}
	if prev != nil {
		f.Parents = []Hash{prevFileHash}
	}
	h, err := a.StoreFile(ctx, p, f)
	if err != nil {
		return nil, fmt.Errorf("failure saving the latest file metadata for %q: %v", p, err)
	}
	return h, nil
}

func SnapshotRegularFile(ctx context.Context, a Archiver, p Path, info os.FileInfo, contents io.Reader) (Hash, error) {
	h, err := a.StoreObject(ctx, contents)
	if err != nil {
		return nil, fmt.Errorf("failure storing an object: %v", err)
	}
	return SnapshotFileMetadata(ctx, a, p, info, h)
}

func SnapshotDirectory(ctx context.Context, a Archiver, p Path, info os.FileInfo, contents *os.File) (Hash, error) {
	entries, err := contents.ReadDir(0)
	if err != nil {
		return nil, fmt.Errorf("failure reading the filesystem contents of the directory %q: %v", p, err)
	}
	childHashes := make(Tree)
	for _, entry := range entries {
		childPath := Path(filepath.Join(string(p), entry.Name()))
		if a.Exclude(childPath) {
			continue
		}
		childHash, err := Snapshot(ctx, a, childPath)
		if err != nil {
			return nil, fmt.Errorf("failure hashing the child dir %q: %v", childPath, err)
		}
		childHashes[Path(entry.Name())] = childHash
	}
	contentsJson := []byte(childHashes.String())
	contentsHash, err := a.StoreObject(ctx, bytes.NewReader(contentsJson))
	return SnapshotFileMetadata(ctx, a, p, info, contentsHash)
}

func Snapshot(ctx context.Context, a Archiver, p Path) (Hash, error) {
	contents, err := os.Open(string(p))
	if err != nil {
		return nil, fmt.Errorf("failure reading the file %q: %v", p, err)
	}
	defer contents.Close()

	info, err := contents.Stat()
	if err != nil {
		return nil, fmt.Errorf("failure reading the filesystem metadata for %q: %v", p, err)
	}
	if info.IsDir() {
		return SnapshotDirectory(ctx, a, p, info, contents)
	} else {
		return SnapshotRegularFile(ctx, a, p, info, contents)
	}
}

type archiver struct {
	ArchiveDir string
}

func (a *archiver) Exclude(p Path) bool {
	return p == Path(a.ArchiveDir)
}

func (a *archiver) StoreObject(ctx context.Context, reader io.Reader) (h Hash, err error) {
	tmp, err := os.CreateTemp("", "archiver")
	if err != nil {
		return nil, fmt.Errorf("failure creating a temp file: %v", err)
	}
	defer func() {
		if err != nil {
			os.Remove(tmp.Name())
		}
	}()
	reader = io.TeeReader(reader, tmp)
	sum := sha256.New()
	if _, err := io.Copy(sum, reader); err != nil {
		return nil, fmt.Errorf("failure hashing an object: %v", err)
	}
	tmp.Close()
	h = &sha256Hash{fmt.Sprintf("%x", sum.Sum(nil))}
	objPath, objName := h.ObjectName(filepath.Join(a.ArchiveDir, "objects"))
	if err := os.MkdirAll(objPath, os.FileMode(0700)); err != nil {
		return nil, fmt.Errorf("failure creating the object dir for %q: %v", h, err)
	}
	if err := os.Rename(tmp.Name(), filepath.Join(objPath, objName)); err != nil {
		return nil, fmt.Errorf("failure writing the object file for %q: %v", h, err)
	}
	return h, nil
}

func (a *archiver) ReadObject(ctx context.Context, h Hash) (io.ReadCloser, error) {
	objPath, objName := h.ObjectName(filepath.Join(a.ArchiveDir, "objects"))
	return os.Open(filepath.Join(objPath, objName))
}

func (a *archiver) pathHashFile(p Path) (dir string, name string) {
	pathHash := &sha256Hash{fmt.Sprintf("%x", sha256.Sum256([]byte(p)))}
	return pathHash.ObjectName(filepath.Join(a.ArchiveDir, "paths"))
}

func (a *archiver) StoreFile(ctx context.Context, p Path, f *File) (Hash, error) {
	bs := []byte(f.String())
	h, err := a.StoreObject(ctx, bytes.NewReader(bs))
	if err != nil {
		return nil, fmt.Errorf("failure saving file metadata for %+v: %v", f, err)
	}
	pathHashDir, pathHashFile := a.pathHashFile(p)
	if err := os.MkdirAll(pathHashDir, 0700); err != nil {
		return nil, fmt.Errorf("failure creating the paths dir for %q: %v", p, err)
	}
	if err := os.WriteFile(filepath.Join(pathHashDir, pathHashFile), []byte(h.String()), 0600); err != nil {
		return nil, fmt.Errorf("failure writing the hash for path %q: %v", p, err)
	}
	return h, nil
}

func (a *archiver) ReadFile(ctx context.Context, p Path) (Hash, *File, error) {
	pathHashDir, pathHashFile := a.pathHashFile(p)
	bs, err := os.ReadFile(filepath.Join(pathHashDir, pathHashFile))
	if err != nil {
		return nil, nil, err
	}
	fileHashStr := string(bs)
	if !strings.HasPrefix(fileHashStr, "sha256:") {
		return nil, nil, fmt.Errorf("unsupported hash function for %q", fileHashStr)
	}
	h := &sha256Hash{strings.TrimPrefix(fileHashStr, "sha256:")}
	reader, err := a.ReadObject(ctx, h)
	if err != nil {
		return nil, nil, fmt.Errorf("failure looking up the file snapshot for %q: %v", p, err)
	}
	defer reader.Close()
	f, err := ParseFile(reader)
	if err != nil {
		return nil, nil, fmt.Errorf("failure parsing the file snapshot for %q: %v", p, err)
	}
	return h, f, nil
}

func main() {
	if len(os.Args) < 2 || os.Args[1] != "snapshot" {
		fmt.Printf("Usage: %s snapshot [<PATH>]\n", os.Args[0])
		os.Exit(1)
	}
	var path string
	if len(os.Args) > 2 {
		path = os.Args[2]
	} else {
		wd, err := os.Getwd()
		if err != nil {
			log.Fatalf("Error determining the current working directory: %v\n", err)
		}
		path = wd
	}
	abs, err := filepath.Abs(path)
	if err != nil {
		log.Fatalf("Failure resolving the absolute path of %q: %v", path, err)
	}
	path = abs

	home, err := os.UserHomeDir()
	if err != nil {
		log.Fatalf("Failure resolving the user's home dir: %v\n", err)
	}
	a := &archiver{filepath.Join(home, ".archive")}
	ctx := context.Background()
	if h, err := Snapshot(ctx, a, Path(path)); err != nil {
		log.Fatalf("Failure snapshotting the directory %q: %v\n", path, err)
	} else {
		fmt.Printf("Snapshotted %q to %q\n", path, h)
	}
}
<|endoftext|>


<commit_before>// cmd_measurements.go

package main

import (
	"fmt"
	"github.com/keltia/ripe-atlas"
	"github.com/urfave/cli"
	"log"
	"os"
	"strconv"
)

// init injects our measurement-related commands
func init() {
	cliCommands = append(cliCommands, cli.Command{
		Name: "measurements",
		Aliases: []string{
			"measures",
			"m",
		},
		Usage:       "measurements-related keywords",
		Description: "All the commands for measurements",
		Subcommands: []cli.Command{
			{
				Name:        "list",
				Aliases:     []string{"ls"},
				Usage:       "lists all measurements",
				Description: "displays all measurements",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:        "country, c",
						Usage:       "filter on country",
						Destination: &fCountry,
					},
					cli.StringFlag{
						Name:        "asn",
						Usage:       "filter on asn",
						Value:       "",
						Destination: &fAsn,
					},
					cli.BoolFlag{
						Name:        "A, all",
						Usage:       "all measurements even inactive ones",
						Destination: &fAllMeasurements,
					},
					cli.BoolFlag{
						Name:        "is-anchor",
						Usage:       "select anchor measurements",
						Destination: &fIsAnchor,
					},
					cli.StringFlag{
						Name:        "t, type",
						Usage:       "filter on measurement type",
						Value:       "",
						Destination: &fMeasureType,
					},
				},
				Action: measurementsList,
			},
			{
				Name:        "info",
				Usage:       "info for one measurement",
				Description: "gives info for one measurement",
				Action:      measurementInfo,
			},
			{
				Name:        "results",
				Aliases:     []string{"r", "res"},
				Usage:       "results for one measurement",
				Description: "returns results for one measurement",
				Action:      measurementResults,
			},
			{
				Name:        "delete",
				Aliases:     []string{"rm", "del", "destroy"},
				Usage:       "info for one measurement",
				Description: "stops one measurement (or all)",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:        "A",
						Usage:       "select all measurements",
						Destination: &fAllMeasurements,
					},
				},
				Action: measurementDelete,
			},
		},
	})
}

// displayMeasurement returns a string with one measurement
func displayMeasurement(m *atlas.Measurement, verbose bool) (res string) {
	if verbose {
		res = fmt.Sprintf("%v\n", m)
	} else {
		res = fmt.Sprintf("ID: %d type: %s description: %s Target: %s\n", m.ID, m.Type, m.Description, m.Target)
	}
	return
}

// displayAllMeasurements returns a string with all measurements
func displayAllMeasurements(ml *[]atlas.Measurement, verbose bool) (res string) {
	res = ""
	for _, m := range *ml {
		res += displayMeasurement(&m, verbose)
	}
	return
}

// measurementsList returns a list of measurements according to parameters
func measurementsList(c *cli.Context) error {
	opts := make(map[string]string)

	if fCountry != "" {
		opts["country_code"] = fCountry
	}

	if fAsn != "" {
		opts["asn"] = fAsn
	}

	if fMeasureType != "" {
		opts["type"] = fMeasureType
	}

	// Check global parameters
	opts = checkGlobalFlags(opts)

	if fVerbose {
		displayOptions(opts)
	}

	q, err := client.GetMeasurements(opts)
	if err != nil {
		log.Printf("GetMeasurements err: %v - q:%v", err, q)
		os.Exit(1)
	}
	log.Printf("Got %d measurements with %v\n", len(q), opts)
	fmt.Print(displayAllMeasurements(&q, fVerbose))

	return nil
}

// measurementInfo is for one measurement only
func measurementInfo(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a measurement ID!")
	}

	id, _ := strconv.Atoi(args[0])

	p, err := client.GetMeasurement(id)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}
	fmt.Print(displayMeasurement(p, fVerbose))

	return nil
}

// measurementResults returns the result part of the measurement
func measurementResults(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a measurement ID!")
	}

	id, _ := strconv.Atoi(args[0])

	m, err := client.GetMeasurement(id)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}

	// m.Result is an URI pointing to results, fetch it
	if m.Result == "" {
		fmt.Println("Empty result")
	}

	resp, err := client.FetchResult(m.Result)
	if err != nil {
		err = fmt.Errorf("bad net/http answer for %s: %v", m.Result, err)
		return err
	}

	fmt.Print(resp)
	return nil
}

func measurementCreate(c *cli.Context) error {
	return nil
}

func measurementDelete(c *cli.Context) (err error) {

	if fAllMeasurements {
		opts := make(map[string]string)

		// Check global parameters
		opts = checkGlobalFlags(opts)

		if fVerbose {
			displayOptions(opts)
		}

		list, err := client.GetMeasurements(opts)
		if err != nil {
			err = fmt.Errorf("Delete all failed: %v", err)
		} else {
			for _, m := range list {
				err = client.DeleteMeasurement(m.ID)
				if err != nil {
					err = fmt.Errorf("Error: can not delete measurement %d", m.ID)
				}
			}
			fmt.Println("All measurements stopped.")
		}
	} else {
		args := c.Args()
		if args[0] == "" {
			log.Fatal("Error: you must specify a measurement ID!")
		}

		id, _ := strconv.Atoi(args[0])
		err = client.DeleteMeasurement(id)
	}

	return
}
<commit_msg>Add the "stop" alias which reflects the real fact.<commit_after>// cmd_measurements.go

package main

import (
	"fmt"
	"github.com/keltia/ripe-atlas"
	"github.com/urfave/cli"
	"log"
	"os"
	"strconv"
)

// init injects our measurement-related commands
func init() {
	cliCommands = append(cliCommands, cli.Command{
		Name: "measurements",
		Aliases: []string{
			"measures",
			"m",
		},
		Usage:       "measurements-related keywords",
		Description: "All the commands for measurements",
		Subcommands: []cli.Command{
			{
				Name:        "list",
				Aliases:     []string{"ls"},
				Usage:       "lists all measurements",
				Description: "displays all measurements",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:        "country, c",
						Usage:       "filter on country",
						Destination: &fCountry,
					},
					cli.StringFlag{
						Name:        "asn",
						Usage:       "filter on asn",
						Value:       "",
						Destination: &fAsn,
					},
					cli.BoolFlag{
						Name:        "A, all",
						Usage:       "all measurements even inactive ones",
						Destination: &fAllMeasurements,
					},
					cli.BoolFlag{
						Name:        "is-anchor",
						Usage:       "select anchor measurements",
						Destination: &fIsAnchor,
					},
					cli.StringFlag{
						Name:        "t, type",
						Usage:       "filter on measurement type",
						Value:       "",
						Destination: &fMeasureType,
					},
				},
				Action: measurementsList,
			},
			{
				Name:        "info",
				Usage:       "info for one measurement",
				Description: "gives info for one measurement",
				Action:      measurementInfo,
			},
			{
				Name:        "results",
				Aliases:     []string{"r", "res"},
				Usage:       "results for one measurement",
				Description: "returns results for one measurement",
				Action:      measurementResults,
			},
			{
				Name:        "delete",
				Aliases:     []string{"rm", "del", "destroy", "stop"},
				Usage:       "info for one measurement",
				Description: "stops one measurement (or all)",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:        "A",
						Usage:       "select all measurements",
						Destination: &fAllMeasurements,
					},
				},
				Action: measurementDelete,
			},
		},
	})
}

// displayMeasurement returns a string with one measurement
func displayMeasurement(m *atlas.Measurement, verbose bool) (res string) {
	if verbose {
		res = fmt.Sprintf("%v\n", m)
	} else {
		res = fmt.Sprintf("ID: %d type: %s description: %s Target: %s\n", m.ID, m.Type, m.Description, m.Target)
	}
	return
}

// displayAllMeasurements returns a string with all measurements
func displayAllMeasurements(ml *[]atlas.Measurement, verbose bool) (res string) {
	res = ""
	for _, m := range *ml {
		res += displayMeasurement(&m, verbose)
	}
	return
}

// measurementsList returns a list of measurements according to parameters
func measurementsList(c *cli.Context) error {
	opts := make(map[string]string)

	if fCountry != "" {
		opts["country_code"] = fCountry
	}

	if fAsn != "" {
		opts["asn"] = fAsn
	}

	if fMeasureType != "" {
		opts["type"] = fMeasureType
	}

	// Check global parameters
	opts = checkGlobalFlags(opts)

	if fVerbose {
		displayOptions(opts)
	}

	q, err := client.GetMeasurements(opts)
	if err != nil {
		log.Printf("GetMeasurements err: %v - q:%v", err, q)
		os.Exit(1)
	}
	log.Printf("Got %d measurements with %v\n", len(q), opts)
	fmt.Print(displayAllMeasurements(&q, fVerbose))

	return nil
}

// measurementInfo is for one measurement only
func measurementInfo(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a measurement ID!")
	}

	id, _ := strconv.Atoi(args[0])

	p, err := client.GetMeasurement(id)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}
	fmt.Print(displayMeasurement(p, fVerbose))

	return nil
}

// measurementResults returns the result part of the measurement
func measurementResults(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a measurement ID!")
	}

	id, _ := strconv.Atoi(args[0])

	m, err := client.GetMeasurement(id)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}

	// m.Result is an URI pointing to results, fetch it
	if m.Result == "" {
		fmt.Println("Empty result")
	}

	resp, err := client.FetchResult(m.Result)
	if err != nil {
		err = fmt.Errorf("bad net/http answer for %s: %v", m.Result, err)
		return err
	}

	fmt.Print(resp)
	return nil
}

func measurementCreate(c *cli.Context) error {
	return nil
}

func measurementDelete(c *cli.Context) (err error) {

	if fAllMeasurements {
		opts := make(map[string]string)

		// Check global parameters
		opts = checkGlobalFlags(opts)

		if fVerbose {
			displayOptions(opts)
		}

		list, err := client.GetMeasurements(opts)
		if err != nil {
			err = fmt.Errorf("Delete all failed: %v", err)
		} else {
			for _, m := range list {
				err = client.DeleteMeasurement(m.ID)
				if err != nil {
					err = fmt.Errorf("Error: can not delete measurement %d", m.ID)
				}
			}
			fmt.Println("All measurements stopped.")
		}
	} else {
		args := c.Args()
		if args[0] == "" {
			log.Fatal("Error: you must specify a measurement ID!")
		}

		id, _ := strconv.Atoi(args[0])
		err = client.DeleteMeasurement(id)
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"

	"code.cloudfoundry.org/guardian/guardiancmd"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/localip"
	"github.com/concourse/atc"
	"github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
)

type Certs struct {
	Dir string `long:"certs-dir" description:"Directory to use when creating the resource certificates volume."`
}

type GardenBackend struct {
	guardiancmd.ServerCommand

	DNS DNSConfig `group:"DNS Proxy Configuration" namespace:"dns-proxy"`
}

func (cmd WorkerCommand) lessenRequirements(prefix string, command *flags.Command) {
	command.FindOptionByLongName(prefix + "garden-bind-port").Default = []string{"7777"}

	// configured as work-dir/depot
	command.FindOptionByLongName(prefix + "garden-depot").Required = false

	// un-configure graph (default /var/gdn/graph)
	command.FindOptionByLongName(prefix + "garden-graph").Required = false
	command.FindOptionByLongName(prefix + "garden-graph").Default = []string{}

	// these are provided as assets embedded in the 'concourse' binary
	command.FindOptionByLongName(prefix + "garden-runtime-plugin").Required = false
	command.FindOptionByLongName(prefix + "garden-dadoo-bin").Required = false
	command.FindOptionByLongName(prefix + "garden-init-bin").Required = false
	command.FindOptionByLongName(prefix + "garden-nstar-bin").Required = false
	command.FindOptionByLongName(prefix + "garden-tar-bin").Required = false

	// configured as work-dir/volumes
	command.FindOptionByLongName(prefix + "baggageclaim-volumes").Required = false
}

func (cmd *WorkerCommand) gardenRunner(logger lager.Logger, hasAssets bool) (atc.Worker, ifrit.Runner, error) {
	err := cmd.checkRoot()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	depotDir := filepath.Join(cmd.WorkDir.Path(), "depot")

	// must be readable by other users so unprivileged containers can run their
	// own `initc' process
	err = os.MkdirAll(depotDir, 0755)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	cmd.Garden.Server.BindIP = guardiancmd.IPFlag(cmd.BindIP.IP)
	cmd.Garden.Containers.Dir = depotDir

	cmd.Garden.Network.AllowHostAccess = true

	worker := cmd.Worker.Worker()
	worker.Platform = "linux"
	worker.CertsPath = &cmd.Certs.Dir

	if hasAssets {
		cmd.Garden.Runtime.Plugin = cmd.assetPath("bin", "runc")
		cmd.Garden.Bin.Dadoo = guardiancmd.FileFlag(cmd.assetPath("bin", "dadoo"))
		cmd.Garden.Bin.Init = guardiancmd.FileFlag(cmd.assetPath("bin", "init"))
		cmd.Garden.Bin.NSTar = guardiancmd.FileFlag(cmd.assetPath("bin", "nstar"))
		cmd.Garden.Bin.Tar = guardiancmd.FileFlag(cmd.assetPath("bin", "tar"))

		iptablesDir := cmd.assetPath("iptables")
		cmd.Garden.Bin.IPTables = guardiancmd.FileFlag(filepath.Join(iptablesDir, "sbin", "iptables"))

		worker.ResourceTypes, err = cmd.extractResources(logger.Session("extract-resources"))
		if err != nil {
			return atc.Worker{}, nil, err
		}
	}

	worker.Name, err = cmd.workerName()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	members := grouper.Members{
		{
			Name:   "garden-runc",
			Runner: &cmd.Garden.ServerCommand,
		},
	}

	if cmd.Garden.DNS.Enable {
		dnsProxyRunner, err := cmd.dnsProxyRunner(logger.Session("dns-proxy"))
		if err != nil {
			return atc.Worker{}, nil, err
		}

		lip, err := localip.LocalIP()
		if err != nil {
			return atc.Worker{}, nil, err
		}

		cmd.Garden.Network.AdditionalDNSServers = append(
			cmd.Garden.Network.AdditionalDNSServers,
			guardiancmd.IPFlag(net.ParseIP(lip)),
		)

		members = append(members, grouper.Member{
			Name:   "dns-proxy",
			Runner: dnsProxyRunner,
		})
	}

	return worker, grouper.NewParallel(os.Interrupt, members), nil
}

func (cmd *WorkerCommand) extractResources(logger lager.Logger) ([]atc.WorkerResourceType, error) {
	var resourceTypes []atc.WorkerResourceType

	resourcesDir := cmd.assetPath("resources")

	infos, err := ioutil.ReadDir(resourcesDir)
	if err != nil {
		logger.Error("failed-to-list-resource-assets", err)
		return nil, err
	}

	for _, info := range infos {
		resourceType := info.Name()

		workerResourceType, err := cmd.extractResource(
			logger.Session("extract", lager.Data{"resource-type": resourceType}),
			resourcesDir,
			resourceType,
		)
		if err != nil {
			logger.Error("failed-to-extract-resource", err)
			return nil, err
		}

		resourceTypes = append(resourceTypes, workerResourceType)
	}

	return resourceTypes, nil
}

func (cmd *WorkerCommand) extractResource(
	logger lager.Logger,
	resourcesDir string,
	resourceType string,
) (atc.WorkerResourceType, error) {
	resourceImagesDir := cmd.assetPath("resource-images")
	tarBin := cmd.assetPath("bin", "tar")

	archive := filepath.Join(resourcesDir, resourceType, "rootfs.tar.gz")

	extractedDir := filepath.Join(resourceImagesDir, resourceType)

	rootfsDir := filepath.Join(extractedDir, "rootfs")
	okMarker := filepath.Join(extractedDir, "ok")

	var version string
	versionFile, err := os.Open(filepath.Join(resourcesDir, resourceType, "version"))
	if err != nil {
		logger.Error("failed-to-read-version", err)
		return atc.WorkerResourceType{}, err
	}

	_, err = fmt.Fscanf(versionFile, "%s", &version)
	if err != nil {
		logger.Error("failed-to-parse-version", err)
		return atc.WorkerResourceType{}, err
	}

	defer versionFile.Close()

	var privileged bool
	_, err = os.Stat(filepath.Join(resourcesDir, resourceType, "privileged"))
	if err == nil {
		privileged = true
	}

	_, err = os.Stat(okMarker)
	if err == nil {
		logger.Info("already-extracted")
	} else {
		logger.Info("extracting")

		err := os.RemoveAll(rootfsDir)
		if err != nil {
			logger.Error("failed-to-clear-out-existing-rootfs", err)
			return atc.WorkerResourceType{}, err
		}

		err = os.MkdirAll(rootfsDir, 0755)
		if err != nil {
			logger.Error("failed-to-create-rootfs-dir", err)
			return atc.WorkerResourceType{}, err
		}

		tar := exec.Command(tarBin, "-zxf", archive, "-C", rootfsDir)

		output, err := tar.CombinedOutput()
		if err != nil {
			logger.Error("failed-to-extract-resource", err, lager.Data{
				"output": string(output),
			})
			return atc.WorkerResourceType{}, err
		}

		ok, err := os.Create(okMarker)
		if err != nil {
			logger.Error("failed-to-create-ok-marker", err)
			return atc.WorkerResourceType{}, err
		}

		err = ok.Close()
		if err != nil {
			logger.Error("failed-to-close-ok-marker", err)
			return atc.WorkerResourceType{}, err
		}
	}

	return atc.WorkerResourceType{
		Type:       resourceType,
		Image:      rootfsDir,
		Version:    version,
		Privileged: privileged,
	}, nil
}

var ErrNotRoot = errors.New("worker must be run as root")

func (cmd *WorkerCommand) checkRoot() error {
	currentUser, err := user.Current()
	if err != nil {
		return err
	}

	if currentUser.Uid != "0" {
		return ErrNotRoot
	}

	return nil
}

func (cmd *WorkerCommand) dnsProxyRunner(logger lager.Logger) (ifrit.Runner, error) {
	server, err := cmd.Garden.DNS.Server()
	if err != nil {
		return nil, err
	}

	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		server.NotifyStartedFunc = func() {
			close(ready)
			logger.Info("started")
		}

		serveErr := make(chan error, 1)

		go func() {
			serveErr <- server.ListenAndServe()
		}()

		for {
			select {
			case err := <-serveErr:
				return err
			case <-signals:
				server.Shutdown()
			}
		}
	}), nil
}
<commit_msg>Update guardian properties to match new gdn version<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"

	"code.cloudfoundry.org/guardian/guardiancmd"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/localip"
	"github.com/concourse/atc"
	"github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
)

type Certs struct {
	Dir string `long:"certs-dir" description:"Directory to use when creating the resource certificates volume."`
}

type GardenBackend struct {
	guardiancmd.ServerCommand

	DNS DNSConfig `group:"DNS Proxy Configuration" namespace:"dns-proxy"`
}

func (cmd WorkerCommand) lessenRequirements(prefix string, command *flags.Command) {
	command.FindOptionByLongName(prefix + "garden-bind-port").Default = []string{"7777"}

	// configured as work-dir/depot
	command.FindOptionByLongName(prefix + "garden-depot").Required = false

	// these are provided as assets embedded in the 'concourse' binary
	command.FindOptionByLongName(prefix + "garden-runtime-plugin").Required = false
	command.FindOptionByLongName(prefix + "garden-dadoo-bin").Required = false
	command.FindOptionByLongName(prefix + "garden-init-bin").Required = false
	command.FindOptionByLongName(prefix + "garden-nstar-bin").Required = false
	command.FindOptionByLongName(prefix + "garden-tar-bin").Required = false

	// configured as work-dir/volumes
	command.FindOptionByLongName(prefix + "baggageclaim-volumes").Required = false
}

func (cmd *WorkerCommand) gardenRunner(logger lager.Logger, hasAssets bool) (atc.Worker, ifrit.Runner, error) {
	err := cmd.checkRoot()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	depotDir := filepath.Join(cmd.WorkDir.Path(), "depot")

	// must be readable by other users so unprivileged containers can run their
	// own `initc' process
	err = os.MkdirAll(depotDir, 0755)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	cmd.Garden.Server.BindIP = guardiancmd.IPFlag(cmd.BindIP.IP)
	cmd.Garden.Containers.Dir = depotDir

	cmd.Garden.Network.AllowHostAccess = true
	cmd.Garden.Image.NoPlugin = true

	worker := cmd.Worker.Worker()
	worker.Platform = "linux"
	worker.CertsPath = &cmd.Certs.Dir

	if hasAssets {
		cmd.Garden.Runtime.Plugin = cmd.assetPath("bin", "runc")
		cmd.Garden.Bin.Dadoo = guardiancmd.FileFlag(cmd.assetPath("bin", "dadoo"))
		cmd.Garden.Bin.Init = guardiancmd.FileFlag(cmd.assetPath("bin", "init"))
		cmd.Garden.Bin.NSTar = guardiancmd.FileFlag(cmd.assetPath("bin", "nstar"))
		cmd.Garden.Bin.Tar = guardiancmd.FileFlag(cmd.assetPath("bin", "tar"))

		iptablesDir := cmd.assetPath("iptables")
		cmd.Garden.Bin.IPTables = guardiancmd.FileFlag(filepath.Join(iptablesDir, "sbin", "iptables"))

		worker.ResourceTypes, err = cmd.extractResources(logger.Session("extract-resources"))
		if err != nil {
			return atc.Worker{}, nil, err
		}
	}

	worker.Name, err = cmd.workerName()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	members := grouper.Members{
		{
			Name:   "garden-runc",
			Runner: &cmd.Garden.ServerCommand,
		},
	}

	if cmd.Garden.DNS.Enable {
		dnsProxyRunner, err := cmd.dnsProxyRunner(logger.Session("dns-proxy"))
		if err != nil {
			return atc.Worker{}, nil, err
		}

		lip, err := localip.LocalIP()
		if err != nil {
			return atc.Worker{}, nil, err
		}

		cmd.Garden.Network.AdditionalDNSServers = append(
			cmd.Garden.Network.AdditionalDNSServers,
			guardiancmd.IPFlag(net.ParseIP(lip)),
		)

		members = append(members, grouper.Member{
			Name:   "dns-proxy",
			Runner: dnsProxyRunner,
		})
	}

	return worker, grouper.NewParallel(os.Interrupt, members), nil
}

func (cmd *WorkerCommand) extractResources(logger lager.Logger) ([]atc.WorkerResourceType, error) {
	var resourceTypes []atc.WorkerResourceType

	resourcesDir := cmd.assetPath("resources")

	infos, err := ioutil.ReadDir(resourcesDir)
	if err != nil {
		logger.Error("failed-to-list-resource-assets", err)
		return nil, err
	}

	for _, info := range infos {
		resourceType := info.Name()

		workerResourceType, err := cmd.extractResource(
			logger.Session("extract", lager.Data{"resource-type": resourceType}),
			resourcesDir,
			resourceType,
		)
		if err != nil {
			logger.Error("failed-to-extract-resource", err)
			return nil, err
		}

		resourceTypes = append(resourceTypes, workerResourceType)
	}

	return resourceTypes, nil
}

func (cmd *WorkerCommand) extractResource(
	logger lager.Logger,
	resourcesDir string,
	resourceType string,
) (atc.WorkerResourceType, error) {
	resourceImagesDir := cmd.assetPath("resource-images")
	tarBin := cmd.assetPath("bin", "tar")

	archive := filepath.Join(resourcesDir, resourceType, "rootfs.tar.gz")

	extractedDir := filepath.Join(resourceImagesDir, resourceType)

	rootfsDir := filepath.Join(extractedDir, "rootfs")
	okMarker := filepath.Join(extractedDir, "ok")

	var version string
	versionFile, err := os.Open(filepath.Join(resourcesDir, resourceType, "version"))
	if err != nil {
		logger.Error("failed-to-read-version", err)
		return atc.WorkerResourceType{}, err
	}

	_, err = fmt.Fscanf(versionFile, "%s", &version)
	if err != nil {
		logger.Error("failed-to-parse-version", err)
		return atc.WorkerResourceType{}, err
	}

	defer versionFile.Close()

	var privileged bool
	_, err = os.Stat(filepath.Join(resourcesDir, resourceType, "privileged"))
	if err == nil {
		privileged = true
	}

	_, err = os.Stat(okMarker)
	if err == nil {
		logger.Info("already-extracted")
	} else {
		logger.Info("extracting")

		err := os.RemoveAll(rootfsDir)
		if err != nil {
			logger.Error("failed-to-clear-out-existing-rootfs", err)
			return atc.WorkerResourceType{}, err
		}

		err = os.MkdirAll(rootfsDir, 0755)
		if err != nil {
			logger.Error("failed-to-create-rootfs-dir", err)
			return atc.WorkerResourceType{}, err
		}

		tar := exec.Command(tarBin, "-zxf", archive, "-C", rootfsDir)

		output, err := tar.CombinedOutput()
		if err != nil {
			logger.Error("failed-to-extract-resource", err, lager.Data{
				"output": string(output),
			})
			return atc.WorkerResourceType{}, err
		}

		ok, err := os.Create(okMarker)
		if err != nil {
			logger.Error("failed-to-create-ok-marker", err)
			return atc.WorkerResourceType{}, err
		}

		err = ok.Close()
		if err != nil {
			logger.Error("failed-to-close-ok-marker", err)
			return atc.WorkerResourceType{}, err
		}
	}

	return atc.WorkerResourceType{
		Type:       resourceType,
		Image:      rootfsDir,
		Version:    version,
		Privileged: privileged,
	}, nil
}

var ErrNotRoot = errors.New("worker must be run as root")

func (cmd *WorkerCommand) checkRoot() error {
	currentUser, err := user.Current()
	if err != nil {
		return err
	}

	if currentUser.Uid != "0" {
		return ErrNotRoot
	}

	return nil
}

func (cmd *WorkerCommand) dnsProxyRunner(logger lager.Logger) (ifrit.Runner, error) {
	server, err := cmd.Garden.DNS.Server()
	if err != nil {
		return nil, err
	}

	return ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error {
		server.NotifyStartedFunc = func() {
			close(ready)
			logger.Info("started")
		}

		serveErr := make(chan error, 1)

		go func() {
			serveErr <- server.ListenAndServe()
		}()

		for {
			select {
			case err := <-serveErr:
				return err
			case <-signals:
				server.Shutdown()
			}
		}
	}), nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/cloudfoundry-incubator/guardian/guardiancmd"
	"github.com/concourse/atc"
	"github.com/concourse/baggageclaim/baggageclaimcmd"
	"github.com/concourse/baggageclaim/fs"
	"github.com/concourse/bin/bindata"
	"github.com/jessevdk/go-flags"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

const btrfsFSType = 0x9123683e

type GardenBackend guardiancmd.GuardianCommand

func (cmd WorkerCommand) lessenRequirements(command *flags.Command) {
	command.FindOptionByLongName("garden-depot").Required = false
	command.FindOptionByLongName("garden-graph").Required = false
	command.FindOptionByLongName("garden-runc-bin").Required = false
	command.FindOptionByLongName("garden-dadoo-bin").Required = false
	command.FindOptionByLongName("garden-init-bin").Required = false
	command.FindOptionByLongName("garden-iodaemon-bin").Required = false
	command.FindOptionByLongName("garden-kawasaki-bin").Required = false
	command.FindOptionByLongName("garden-nstar-bin").Required = false
	command.FindOptionByLongName("garden-tar-bin").Required = false
}

func (cmd *WorkerCommand) gardenRunner(logger lager.Logger, args []string) (atc.Worker, ifrit.Runner, error) {
	err := cmd.checkRoot()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	err = bindata.RestoreAssets(cmd.WorkDir, "linux")
	if err != nil {
		return atc.Worker{}, nil, err
	}

	linux := filepath.Join(cmd.WorkDir, "linux")

	btrfsToolsDir := filepath.Join(linux, "btrfs")
	iptablesDir := filepath.Join(linux, "iptables")

	err = os.Setenv(
		"PATH",
		strings.Join(
			[]string{
				btrfsToolsDir,
				filepath.Join(iptablesDir, "sbin"),
				filepath.Join(iptablesDir, "bin"),
				os.Getenv("PATH"),
			},
			string(os.PathListSeparator),
		),
	)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	busyboxDir, err := cmd.extractBusybox(linux)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	depotDir := filepath.Join(linux, "depot")

	// must be readable by other users so unprivileged containers can run their
	// own `initc' process
	err = os.MkdirAll(depotDir, 0755)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	cmd.Garden.Server.BindIP = guardiancmd.IPFlag(cmd.BindIP)

	cmd.Garden.Containers.Dir = guardiancmd.DirFlag(depotDir)
	cmd.Garden.Containers.DefaultRootFSDir = guardiancmd.DirFlag(busyboxDir)

	cmd.Garden.Bin.Runc = filepath.Join(linux, "bin", "runc")
	cmd.Garden.Bin.Dadoo = guardiancmd.FileFlag(filepath.Join(linux, "bin", "dadoo"))
	cmd.Garden.Bin.Init = guardiancmd.FileFlag(filepath.Join(linux, "bin", "init"))
	cmd.Garden.Bin.IODaemon = guardiancmd.FileFlag(filepath.Join(linux, "bin", "iodaemon"))
	cmd.Garden.Bin.Kawasaki = guardiancmd.FileFlag(filepath.Join(linux, "bin", "kawasaki"))
	cmd.Garden.Bin.NSTar = guardiancmd.FileFlag(filepath.Join(linux, "bin", "nstar"))
	cmd.Garden.Bin.Tar = guardiancmd.FileFlag(filepath.Join(linux, "bin", "tar"))

	cmd.Garden.Network.AllowHostAccess = true

	worker := atc.Worker{
		Platform: "linux",
		Tags:     cmd.Tags,

		HTTPProxyURL:  cmd.HTTPProxy.String(),
		HTTPSProxyURL: cmd.HTTPSProxy.String(),
		NoProxy:       strings.Join(cmd.NoProxy, ","),
	}

	worker.ResourceTypes, err = cmd.extractResources(linux)

	worker.Name, err = cmd.workerName()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	runner := guardiancmd.GuardianCommand(cmd.Garden)
	return worker, &runner, nil
}

func (cmd *WorkerCommand) baggageclaimRunner(logger lager.Logger) (ifrit.Runner, error) {
	supportsBtrfs, err := cmd.supportsBtrfs()
	if err != nil {
		logger.Error("failed-to-check-for-btrfs-support", err)
		return nil, err
	}

	if !supportsBtrfs {
		logger.Info("using-native-volume-driver", nil)
		return cmd.naiveBaggageclaimRunner(logger)
	}

	volumesImage := filepath.Join(cmd.WorkDir, "volumes.img")
	volumesDir := filepath.Join(cmd.WorkDir, "volumes")

	err = os.MkdirAll(volumesDir, 0755)
	if err != nil {
		return nil, err
	}

	var fsStat syscall.Statfs_t
	err = syscall.Statfs(volumesDir, &fsStat)
	if err != nil {
		return nil, fmt.Errorf("failed to stat volumes filesystem: %s", err)
	}

	if fsStat.Type != btrfsFSType {
		filesystem := fs.New(logger.Session("fs"), volumesImage, volumesDir)

		err = filesystem.Create(fsStat.Blocks * uint64(fsStat.Bsize))
		if err != nil {
			return nil, fmt.Errorf("failed to set up volumes filesystem: %s", err)
		}
	}

	bc := &baggageclaimcmd.BaggageclaimCommand{
		BindIP:   baggageclaimcmd.IPFlag(cmd.Baggageclaim.BindIP.IP().String()),
		BindPort: cmd.Baggageclaim.BindPort,

		VolumesDir: baggageclaimcmd.DirFlag(volumesDir),

		Driver: "btrfs",

		ReapInterval: cmd.Baggageclaim.ReapInterval,

		Metrics: cmd.Metrics,
	}

	return bc.Runner(nil)
}

func (cmd *WorkerCommand) supportsBtrfs() (bool, error) {
	fs, err := os.Open("/proc/filesystems")
	if err != nil {
		return false, err
	}

	defer fs.Close()

	scanner := bufio.NewScanner(fs)

	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "btrfs") {
			return true, nil
		}
	}

	return false, nil
}

func (cmd *WorkerCommand) extractBusybox(linux string) (string, error) {
	archive := filepath.Join(linux, "busybox.tar.gz")

	busyboxDir := filepath.Join(linux, "busybox")
	err := os.MkdirAll(busyboxDir, 0755)
	if err != nil {
		return "", err
	}

	tarBin := filepath.Join(linux, "bin", "tar")
	tar := exec.Command(tarBin, "-zxf", archive, "-C", busyboxDir)
	tar.Stdout = os.Stdout
	tar.Stderr = os.Stderr

	err = tar.Run()
	if err != nil {
		return "", err
	}

	return busyboxDir, nil
}

func (cmd *WorkerCommand) extractResources(linux string) ([]atc.WorkerResourceType, error) {
	var resourceTypes []atc.WorkerResourceType

	binDir := filepath.Join(linux, "bin")
	resourcesDir := filepath.Join(linux, "resources")
	resourceImagesDir := filepath.Join(linux, "resource-images")

	tarBin := filepath.Join(binDir, "tar")

	infos, err := ioutil.ReadDir(resourcesDir)
	if err == nil {
		for _, info := range infos {
			archive := filepath.Join(resourcesDir, info.Name())
			resourceType := info.Name()

			imageDir := filepath.Join(resourceImagesDir, resourceType)

			err := os.RemoveAll(imageDir)
			if err != nil {
				return nil, err
			}

			err = os.MkdirAll(imageDir, 0755)
			if err != nil {
				return nil, err
			}

			tar := exec.Command(tarBin, "-zxf", archive, "-C", imageDir)
			tar.Stdout = os.Stdout
			tar.Stderr = os.Stderr

			err = tar.Run()
			if err != nil {
				return nil, err
			}

			resourceTypes = append(resourceTypes, atc.WorkerResourceType{
				Type:  resourceType,
				Image: imageDir,
			})
		}
	}

	return resourceTypes, nil
}
<commit_msg>that t really changes meaning huh<commit_after>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/cloudfoundry-incubator/guardian/guardiancmd"
	"github.com/concourse/atc"
	"github.com/concourse/baggageclaim/baggageclaimcmd"
	"github.com/concourse/baggageclaim/fs"
	"github.com/concourse/bin/bindata"
	"github.com/jessevdk/go-flags"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

const btrfsFSType = 0x9123683e

type GardenBackend guardiancmd.GuardianCommand

func (cmd WorkerCommand) lessenRequirements(command *flags.Command) {
	command.FindOptionByLongName("garden-depot").Required = false
	command.FindOptionByLongName("garden-graph").Required = false
	command.FindOptionByLongName("garden-runc-bin").Required = false
	command.FindOptionByLongName("garden-dadoo-bin").Required = false
	command.FindOptionByLongName("garden-init-bin").Required = false
	command.FindOptionByLongName("garden-iodaemon-bin").Required = false
	command.FindOptionByLongName("garden-kawasaki-bin").Required = false
	command.FindOptionByLongName("garden-nstar-bin").Required = false
	command.FindOptionByLongName("garden-tar-bin").Required = false
}

func (cmd *WorkerCommand) gardenRunner(logger lager.Logger, args []string) (atc.Worker, ifrit.Runner, error) {
	err := cmd.checkRoot()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	err = bindata.RestoreAssets(cmd.WorkDir, "linux")
	if err != nil {
		return atc.Worker{}, nil, err
	}

	linux := filepath.Join(cmd.WorkDir, "linux")

	btrfsToolsDir := filepath.Join(linux, "btrfs")
	iptablesDir := filepath.Join(linux, "iptables")

	err = os.Setenv(
		"PATH",
		strings.Join(
			[]string{
				btrfsToolsDir,
				filepath.Join(iptablesDir, "sbin"),
				filepath.Join(iptablesDir, "bin"),
				os.Getenv("PATH"),
			},
			string(os.PathListSeparator),
		),
	)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	busyboxDir, err := cmd.extractBusybox(linux)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	depotDir := filepath.Join(linux, "depot")

	// must be readable by other users so unprivileged containers can run their
	// own `initc' process
	err = os.MkdirAll(depotDir, 0755)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	cmd.Garden.Server.BindIP = guardiancmd.IPFlag(cmd.BindIP)

	cmd.Garden.Containers.Dir = guardiancmd.DirFlag(depotDir)
	cmd.Garden.Containers.DefaultRootFSDir = guardiancmd.DirFlag(busyboxDir)

	cmd.Garden.Bin.Runc = filepath.Join(linux, "bin", "runc")
	cmd.Garden.Bin.Dadoo = guardiancmd.FileFlag(filepath.Join(linux, "bin", "dadoo"))
	cmd.Garden.Bin.Init = guardiancmd.FileFlag(filepath.Join(linux, "bin", "init"))
	cmd.Garden.Bin.IODaemon = guardiancmd.FileFlag(filepath.Join(linux, "bin", "iodaemon"))
	cmd.Garden.Bin.Kawasaki = guardiancmd.FileFlag(filepath.Join(linux, "bin", "kawasaki"))
	cmd.Garden.Bin.NSTar = guardiancmd.FileFlag(filepath.Join(linux, "bin", "nstar"))
	cmd.Garden.Bin.Tar = guardiancmd.FileFlag(filepath.Join(linux, "bin", "tar"))

	cmd.Garden.Network.AllowHostAccess = true

	worker := atc.Worker{
		Platform: "linux",
		Tags:     cmd.Tags,

		HTTPProxyURL:  cmd.HTTPProxy.String(),
		HTTPSProxyURL: cmd.HTTPSProxy.String(),
		NoProxy:       strings.Join(cmd.NoProxy, ","),
	}

	worker.ResourceTypes, err = cmd.extractResources(linux)

	worker.Name, err = cmd.workerName()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	runner := guardiancmd.GuardianCommand(cmd.Garden)
	return worker, &runner, nil
}

func (cmd *WorkerCommand) baggageclaimRunner(logger lager.Logger) (ifrit.Runner, error) {
	supportsBtrfs, err := cmd.supportsBtrfs()
	if err != nil {
		logger.Error("failed-to-check-for-btrfs-support", err)
		return nil, err
	}

	if !supportsBtrfs {
		logger.Info("using-naive-volume-driver", nil)
		return cmd.naiveBaggageclaimRunner(logger)
	}

	volumesImage := filepath.Join(cmd.WorkDir, "volumes.img")
	volumesDir := filepath.Join(cmd.WorkDir, "volumes")

	err = os.MkdirAll(volumesDir, 0755)
	if err != nil {
		return nil, err
	}

	var fsStat syscall.Statfs_t
	err = syscall.Statfs(volumesDir, &fsStat)
	if err != nil {
		return nil, fmt.Errorf("failed to stat volumes filesystem: %s", err)
	}

	if fsStat.Type != btrfsFSType {
		filesystem := fs.New(logger.Session("fs"), volumesImage, volumesDir)

		err = filesystem.Create(fsStat.Blocks * uint64(fsStat.Bsize))
		if err != nil {
			return nil, fmt.Errorf("failed to set up volumes filesystem: %s", err)
		}
	}

	bc := &baggageclaimcmd.BaggageclaimCommand{
		BindIP:   baggageclaimcmd.IPFlag(cmd.Baggageclaim.BindIP.IP().String()),
		BindPort: cmd.Baggageclaim.BindPort,

		VolumesDir: baggageclaimcmd.DirFlag(volumesDir),

		Driver: "btrfs",

		ReapInterval: cmd.Baggageclaim.ReapInterval,

		Metrics: cmd.Metrics,
	}

	return bc.Runner(nil)
}

func (cmd *WorkerCommand) supportsBtrfs() (bool, error) {
	fs, err := os.Open("/proc/filesystems")
	if err != nil {
		return false, err
	}

	defer fs.Close()

	scanner := bufio.NewScanner(fs)

	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "btrfs") {
			return true, nil
		}
	}

	return false, nil
}

func (cmd *WorkerCommand) extractBusybox(linux string) (string, error) {
	archive := filepath.Join(linux, "busybox.tar.gz")

	busyboxDir := filepath.Join(linux, "busybox")
	err := os.MkdirAll(busyboxDir, 0755)
	if err != nil {
		return "", err
	}

	tarBin := filepath.Join(linux, "bin", "tar")
	tar := exec.Command(tarBin, "-zxf", archive, "-C", busyboxDir)
	tar.Stdout = os.Stdout
	tar.Stderr = os.Stderr

	err = tar.Run()
	if err != nil {
		return "", err
	}

	return busyboxDir, nil
}

func (cmd *WorkerCommand) extractResources(linux string) ([]atc.WorkerResourceType, error) {
	var resourceTypes []atc.WorkerResourceType

	binDir := filepath.Join(linux, "bin")
	resourcesDir := filepath.Join(linux, "resources")
	resourceImagesDir := filepath.Join(linux, "resource-images")

	tarBin := filepath.Join(binDir, "tar")

	infos, err := ioutil.ReadDir(resourcesDir)
	if err == nil {
		for _, info := range infos {
			archive := filepath.Join(resourcesDir, info.Name())
			resourceType := info.Name()

			imageDir := filepath.Join(resourceImagesDir, resourceType)

			err := os.RemoveAll(imageDir)
			if err != nil {
				return nil, err
			}

			err = os.MkdirAll(imageDir, 0755)
			if err != nil {
				return nil, err
			}

			tar := exec.Command(tarBin, "-zxf", archive, "-C", imageDir)
			tar.Stdout = os.Stdout
			tar.Stderr = os.Stderr

			err = tar.Run()
			if err != nil {
				return nil, err
			}

			resourceTypes = append(resourceTypes, atc.WorkerResourceType{
				Type:  resourceType,
				Image: imageDir,
			})
		}
	}

	return resourceTypes, nil
}
<|endoftext|>


<commit_before>package playedplugin

import (
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/iopred/bruxism"
	"github.com/iopred/discordgo"
)

type playedEntry struct {
	Name     string
	Duration time.Duration
}

type byDuration []*playedEntry

func (a byDuration) Len() int           { return len(a) }
func (a byDuration) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byDuration) Less(i, j int) bool { return a[i].Duration >= a[j].Duration }

type playedUser struct {
	Entries     map[string]*playedEntry
	Current     string
	LastChanged time.Time
	FirstSeen   time.Time
}

func (p *playedUser) Update(name string, now time.Time) {
	if p.Current != "" {
		pe := p.Entries[p.Current]
		if pe == nil {
			pe = &playedEntry{
				Name: p.Current,
			}
			p.Entries[p.Current] = pe
		}
		pe.Duration += now.Sub(p.LastChanged)
	}

	p.Current = name
	p.LastChanged = now
}

type playedPlugin struct {
	sync.RWMutex
	Users map[string]*playedUser
}

// Load will load plugin state from a byte array.
func (p *playedPlugin) Load(bot *bruxism.Bot, service bruxism.Service, data []byte) error {
	if service.Name() != bruxism.DiscordServiceName {
		panic("Carbonitex Plugin only supports Discord.")
	}

	if data != nil {
		if err := json.Unmarshal(data, p); err != nil {
			log.Println("Error loading data", err)
		}
	}

	go p.Run(bot, service)
	return nil
}

// Save will save plugin state to a byte array.
func (p *playedPlugin) Save() ([]byte, error) {
	return json.Marshal(p)
}

func (p *playedPlugin) Update(user string, entry string) {
	p.Lock()
	defer p.Unlock()

	t := time.Now()

	u := p.Users[user]
	if u == nil {
		u = &playedUser{
			Entries:     map[string]*playedEntry{},
			Current:     entry,
			LastChanged: t,
			FirstSeen:   t,
		}
		p.Users[user] = u
	}
	u.Update(entry, t)
}

// Run is the background go routine that executes for the life of the plugin.
func (p *playedPlugin) Run(bot *bruxism.Bot, service bruxism.Service) {
	discord := service.(*bruxism.Discord)

	discord.Session.AddHandler(func(s *discordgo.Session, r *discordgo.Ready) {
		for _, g := range r.Guilds {
			for _, pu := range g.Presences {
				e := ""
				if pu.Game != nil {
					e = pu.Game.Name
				}
				p.Update(pu.User.ID, e)
			}
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, g *discordgo.GuildCreate) {
		if g.Unavailable == nil || *g.Unavailable {
			return
		}

		for _, pu := range g.Presences {
			e := ""
			if pu.Game != nil {
				e = pu.Game.Name
			}
			p.Update(pu.User.ID, e)
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, pr *discordgo.PresencesReplace) {
		for _, pu := range *pr {
			e := ""
			if pu.Game != nil {
				e = pu.Game.Name
			}
			p.Update(pu.User.ID, e)
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, pu *discordgo.PresenceUpdate) {
		e := ""
		if pu.Game != nil {
			e = pu.Game.Name
		}
		p.Update(pu.User.ID, e)
	})
}

// Help returns a list of help strings that are printed when the user requests them.
func (p *playedPlugin) Help(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message, detailed bool) []string {
	if detailed {
		return nil
	}

	return bruxism.CommandHelp(service, "played", "[@username]", "Returns your most played games, or a users most played games if provided.")
}

var userIDRegex = regexp.MustCompile("<@!?([0-9]*)>")

func (p *playedPlugin) Message(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message) {
	defer bruxism.MessageRecover()
	if service.Name() == bruxism.DiscordServiceName && !service.IsMe(message) {
		if bruxism.MatchesCommand(service, "played", message) {
			query := strings.Join(strings.Split(message.RawMessage(), " ")[1:], " ")

			id := message.UserID()
			match := userIDRegex.FindStringSubmatch(query)
			if match != nil {
				id = match[1]
			}

			p.Lock()
			defer p.Unlock()

			u := p.Users[id]
			if u == nil {
				service.SendMessage(message.Channel(), "I haven't seen that user.")
				return
			}

			if len(u.Entries) == 0 {
				service.SendMessage(message.Channel(), "I haven't seen anything played by that user.")
				return
			}

			lc := humanize.Time(u.LastChanged)
			u.Update(u.Current, time.Now())

			pes := make(byDuration, len(u.Entries))
			i := 0
			for _, pe := range u.Entries {
				pes[i] = pe
				i++
			}

			sort.Sort(pes)

			messageText := fmt.Sprintf("*First seen %s, last update %s*\n", humanize.Time(u.FirstSeen), lc)
			for i = 0; i < len(pes) && i < 5; i++ {
				pe := pes[i]
				messageText += fmt.Sprintf("**%s**: %s\n", pe.Name, pe.Duration.String())
			}
			service.SendMessage(message.Channel(), messageText)
		}
	}
}

// Name returns the name of the plugin.
func (p *playedPlugin) Name() string {
	return "Played"
}

// New will create a played plugin.
func New() bruxism.Plugin {
	return &playedPlugin{
		Users: map[string]*playedUser{},
	}
}
<commit_msg>Format duration better.<commit_after>package playedplugin

import (
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/iopred/bruxism"
	"github.com/iopred/discordgo"
)

type playedEntry struct {
	Name     string
	Duration time.Duration
}

type byDuration []*playedEntry

func (a byDuration) Len() int           { return len(a) }
func (a byDuration) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byDuration) Less(i, j int) bool { return a[i].Duration >= a[j].Duration }

type playedUser struct {
	Entries     map[string]*playedEntry
	Current     string
	LastChanged time.Time
	FirstSeen   time.Time
}

func (p *playedUser) Update(name string, now time.Time) {
	if p.Current != "" {
		pe := p.Entries[p.Current]
		if pe == nil {
			pe = &playedEntry{
				Name: p.Current,
			}
			p.Entries[p.Current] = pe
		}
		pe.Duration += now.Sub(p.LastChanged)
	}

	p.Current = name
	p.LastChanged = now
}

type playedPlugin struct {
	sync.RWMutex
	Users map[string]*playedUser
}

// Load will load plugin state from a byte array.
func (p *playedPlugin) Load(bot *bruxism.Bot, service bruxism.Service, data []byte) error {
	if service.Name() != bruxism.DiscordServiceName {
		panic("Carbonitex Plugin only supports Discord.")
	}

	if data != nil {
		if err := json.Unmarshal(data, p); err != nil {
			log.Println("Error loading data", err)
		}
	}

	go p.Run(bot, service)
	return nil
}

// Save will save plugin state to a byte array.
func (p *playedPlugin) Save() ([]byte, error) {
	return json.Marshal(p)
}

func (p *playedPlugin) Update(user string, entry string) {
	p.Lock()
	defer p.Unlock()

	t := time.Now()

	u := p.Users[user]
	if u == nil {
		u = &playedUser{
			Entries:     map[string]*playedEntry{},
			Current:     entry,
			LastChanged: t,
			FirstSeen:   t,
		}
		p.Users[user] = u
	}
	u.Update(entry, t)
}

// Run is the background go routine that executes for the life of the plugin.
func (p *playedPlugin) Run(bot *bruxism.Bot, service bruxism.Service) {
	discord := service.(*bruxism.Discord)

	discord.Session.AddHandler(func(s *discordgo.Session, r *discordgo.Ready) {
		for _, g := range r.Guilds {
			for _, pu := range g.Presences {
				e := ""
				if pu.Game != nil {
					e = pu.Game.Name
				}
				p.Update(pu.User.ID, e)
			}
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, g *discordgo.GuildCreate) {
		if g.Unavailable == nil || *g.Unavailable {
			return
		}

		for _, pu := range g.Presences {
			e := ""
			if pu.Game != nil {
				e = pu.Game.Name
			}
			p.Update(pu.User.ID, e)
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, pr *discordgo.PresencesReplace) {
		for _, pu := range *pr {
			e := ""
			if pu.Game != nil {
				e = pu.Game.Name
			}
			p.Update(pu.User.ID, e)
		}
	})

	discord.Session.AddHandler(func(s *discordgo.Session, pu *discordgo.PresenceUpdate) {
		e := ""
		if pu.Game != nil {
			e = pu.Game.Name
		}
		p.Update(pu.User.ID, e)
	})
}

// Help returns a list of help strings that are printed when the user requests them.
func (p *playedPlugin) Help(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message, detailed bool) []string {
	if detailed {
		return nil
	}

	return bruxism.CommandHelp(service, "played", "[@username]", "Returns your most played games, or a users most played games if provided.")
}

var userIDRegex = regexp.MustCompile("<@!?([0-9]*)>")

func (p *playedPlugin) Message(bot *bruxism.Bot, service bruxism.Service, message bruxism.Message) {
	defer bruxism.MessageRecover()
	if service.Name() == bruxism.DiscordServiceName && !service.IsMe(message) {
		if bruxism.MatchesCommand(service, "played", message) {
			query := strings.Join(strings.Split(message.RawMessage(), " ")[1:], " ")

			id := message.UserID()
			match := userIDRegex.FindStringSubmatch(query)
			if match != nil {
				id = match[1]
			}

			p.Lock()
			defer p.Unlock()

			u := p.Users[id]
			if u == nil {
				service.SendMessage(message.Channel(), "I haven't seen that user.")
				return
			}

			if len(u.Entries) == 0 {
				service.SendMessage(message.Channel(), "I haven't seen anything played by that user.")
				return
			}

			lc := humanize.Time(u.LastChanged)
			u.Update(u.Current, time.Now())

			pes := make(byDuration, len(u.Entries))
			i := 0
			for _, pe := range u.Entries {
				pes[i] = pe
				i++
			}

			sort.Sort(pes)

			messageText := fmt.Sprintf("*First seen %s, last update %s*\n", humanize.Time(u.FirstSeen), lc)
			for i = 0; i < len(pes) && i < 5; i++ {
				pe := pes[i]

				du := pe.Duration

				ds := ""
				hours := int(du / time.Hour)
				if hours > 0 {
					ds += fmt.Sprintf("%dh ", hours)
					du -= time.Duration(hours) * time.Hour
				}

				minutes := int(du / time.Minute)
				ds += fmt.Sprintf("%dm ", minutes)
				du -= time.Duration(minutes) * time.Minute

				seconds := int(du / time.Second)
				ds += fmt.Sprintf("%ds", seconds)

				messageText += fmt.Sprintf("**%s**: %s\n", pe.Name, ds)
			}
			service.SendMessage(message.Channel(), messageText)
		}
	}
}

// Name returns the name of the plugin.
func (p *playedPlugin) Name() string {
	return "Played"
}

// New will create a played plugin.
func New() bruxism.Plugin {
	return &playedPlugin{
		Users: map[string]*playedUser{},
	}
}
<|endoftext|>


<commit_before>package forms

import (
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"
)

var request http.Request

// Testing basic parsing of YAML form
func TestParseYamlFile(t *testing.T) {
	b := NewBuilder("example.yml", request)

	assert.Equal(t, "Test", b.Config.Name)
}

// Test building a usable form from the yaml file
func TestBuildingForm(t *testing.T) {
	b := NewBuilder("example.yml", request)

	b.build()
	assert.Equal(t, 4, len(b.Form.Fields))
}

// Test building a form with pre-defined values
func TestBuildPreValues(t *testing.T) {
	r := http.Request{
		Method: "POST",
		Body:   ioutil.NopCloser(strings.NewReader("test=foo&test2=bar&test3=test1")),
		Header: http.Header{
			"Content-Type": {"application/x-www-form-urlencoded"},
		},
	}

	b := NewBuilder("example.yml", r)
	b.build()

	for _, field := range b.Form.Fields {
		switch field.Name {
		case "test":
			assert.Equal(t, "foo", field.Value)
		case "test2":
			assert.Equal(t, "bar", field.Value)
		case "test3":
			assert.Equal(t, "test1", field.Value)
		}
	}
}

// Build benchmark
func BenchmarkBuildFormFromConfig(t *testing.B) {
	b := NewBuilder("example.yml", request)

	b.build()
}
<commit_msg>Adding test to ensure validation items are created on build<commit_after>package forms

import (
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"
)

var request http.Request

// Testing basic parsing of YAML form
func TestParseYamlFile(t *testing.T) {
	b := NewBuilder("example.yml", request)

	assert.Equal(t, "Test", b.Config.Name)
}

// Test building a usable form from the yaml file
func TestBuildingForm(t *testing.T) {
	b := NewBuilder("example.yml", request)

	b.build()
	assert.Equal(t, 4, len(b.Form.Fields))
}

// Testing that validation items are actually created
func TestValidationItems(t *testing.T) {
	b := NewBuilder("example.yml", request)
	b.build()

	for _, field := range b.Form.Fields {
		switch field.Name {
		case "test":
			assert.Equal(t, 1, len(field.Rules))
		case "test2":
			assert.Equal(t, 1, len(field.Rules))
		case "test3":
			assert.Equal(t, 2, len(field.Rules))
		}
	}

	b.Form.validate()

	assert.Equal(t, false, b.Form.Valid)
}

// Test building a form with pre-defined values
func TestBuildPreValues(t *testing.T) {
	r := http.Request{
		Method: "POST",
		Body:   ioutil.NopCloser(strings.NewReader("test=foo&test2=bar&test3=test1")),
		Header: http.Header{
			"Content-Type": {"application/x-www-form-urlencoded"},
		},
	}

	b := NewBuilder("example.yml", r)
	b.build()

	for _, field := range b.Form.Fields {
		switch field.Name {
		case "test":
			assert.Equal(t, "foo", field.Value)
		case "test2":
			assert.Equal(t, "bar", field.Value)
		case "test3":
			assert.Equal(t, "test1", field.Value)
		}
	}
}

// Build benchmark
func BenchmarkBuildFormFromConfig(t *testing.B) {
	b := NewBuilder("example.yml", request)

	b.build()
}
<|endoftext|>


<commit_before>package jstmpl

import (
	"fmt"
	"io/ioutil"
	"testing"

	"github.com/ghodss/yaml"
	jstypes "github.com/go-jstmpl/go-jstmpl/types"
	"github.com/lestrrat/go-jshschema"
)

func ParseHschema(file string) (*hschema.HyperSchema, error) {
	b, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("fail to read the source JSON Schema file: %s", err)
	}

	var m map[string]interface{}
	if err := yaml.Unmarshal(b, &m); err != nil {
		return nil, fmt.Errorf("fail to unmarshal YAML: %s", err)
	}

	hs := hschema.New()
	if err := hs.Extract(m); err != nil {
		return nil, fmt.Errorf("fail to extract JSON Schema: %s", err)
	}

	return hs, nil
}

func TestBuilderLoopRef(t *testing.T) {
	hs, err := ParseHschema("./test/ref_loop.yml")
	if err != nil {
		t.Fatal(err)
	}
	b := NewBuilder()
	_, err = b.Build(hs)
	if err == nil {
		t.Fatalf("build should be failed: %+v", b)
	}
}

func TestBuillderNotHaveHref(t *testing.T) {
	hs, err := ParseHschema("./test/has_not_href.yml")
	if err != nil {
		t.Fatal(err)
	}
	b := NewBuilder()
	ts, err := b.Build(hs)
	if err != nil {
		t.Fatalf("fail to build: %s", err)
	}

	if ts.Title != "has not href" {
		t.Error("fail to get title")
	}
}

func TestBuilderPassBuild(t *testing.T) {
	hs, err := ParseHschema("./test/pass.yml")
	if err != nil {
		t.Fatal(err)
	}
	b := NewBuilder()
	ts, err := b.Build(hs)
	if err != nil {
		t.Fatalf("fail to build: %s", err)
	}

	if len(ts.Objects) != 1 {
		t.Fatal("fail to parse Object type schema")
	}

	if ts.Objects[0] == nil || ts.Objects[0].Title() != "test object" {
		t.Errorf("fail to get Object type schema: %+v", ts.Objects[0])
	}

	if len(ts.Arrays) != 1 {
		t.Fatal("fail to parse Array type schema")
	}

	if ts.Arrays[0] == nil || ts.Arrays[0].Title() != "test array" {
		t.Errorf("fail to get Arrays type schema: %+v", ts.Arrays[0])
	}

	if len(ts.Booleans) != 1 {
		t.Fatal("fail to parse Boolean type schema")
	}

	if ts.Booleans[0] == nil || ts.Booleans[0].Title() != "test boolean" {
		t.Errorf("fail to get Boolean type schema: %+v", ts.Booleans[0])
	}

	if len(ts.Numbers) != 1 {
		t.Fatal("fail to parse Number type schema")
	}

	if ts.Numbers[0] == nil || ts.Numbers[0].Title() != "test number" {
		t.Errorf("fail to get Numbers type schema: %+v", ts.Numbers[0])
	}

	if len(ts.Integers) != 2 {
		t.Fatal("fail to parse Integer type schema")
	}

	if ts.Integers[0] == nil || ts.Integers[0].Title() != "test integer" {
		t.Errorf("fail to get Integer type schema: %+v", ts.Integers[0])
	}

	if len(ts.Strings) != 1 {
		t.Fatal("fail to parse String type schema")
	}

	if ts.Strings[0] == nil || ts.Strings[0].Title() != "test string" {
		t.Errorf("fail to get Strings type schema: %+v", ts.Strings[0])
	}

	if len(ts.Properties) != 2 {
		t.Fatal("fail to parse Properties type schema")
	}

	if len(ts.Definitions) != 7 {
		t.Fatal("fail to parse Definitions")
	}

	for _, v := range ts.Properties {
		switch v.Key() {
		case "test_multitype_link":
			if v.Title() != "test multitype" {
				t.Errorf("fail to get Properties type not link extra schema: %+v", v)
			}
		case "test_multitype":
			if v.Title() != "test multitype" {
				t.Errorf("fail to get Properties type link extra schema: %+v", v)
			}
		default:
			t.Errorf("fail to get Properties type schema, specify one of key: %s", v)
		}
	}

	if len(ts.Links) != 1 {
		t.Fatal("fail to parse Links type schema")
	}

	for _, v := range ts.Links {
		switch obj := v.Schema.(type) {
		case *jstypes.Object:
			for _, p := range obj.Properties {
				switch p.Key() {
				case "test_multitype_link":
					if p.Title() != "test multitype" {
						t.Errorf("fail to get Links type link schema: %+v", p)
					}
				case "test_multitype":
					if p.Title() != "test multitype" {
						t.Errorf("fail to get Links type not link schema: %+v", p)
					}
				default:
					t.Errorf("failt to get Links type schema, specify one of keys: %+v", p)
				}
			}
		default:
			t.Errorf("fail to get Links type, should be object schema: %+v", v)
		}
	}
}

func TestResolvePass(t *testing.T) {
	hs, err := ParseHschema("./test/resolve_pass.yml")
	if err != nil {
		t.Fatalf("fail to parse resolve: %v", err)
	}

	if hs.Schema.Definitions["test_link"] == nil {
		t.Fatalf("fail to get schema: %+v", hs)
	}

	sc := hs.Schema.Definitions["test_link"]
	ctx := &jstypes.Context{
		Validations: map[string]bool{},
	}

	for k := range hs.Definitions {
		ctx.Key = k
	}

	scm, err := resolve(sc, hs.Schema, ctx)
	if err != nil {
		t.Fatalf("fail to resolve: %s", err)
	}

	if scm.Title() != "test" {
		t.Errorf("fail to resolve link schema: title should be test: %+v", scm)
	}
}
<commit_msg>fix parse test<commit_after>package jstmpl

import (
	"fmt"
	"io/ioutil"
	"testing"

	"github.com/ghodss/yaml"
	jstypes "github.com/go-jstmpl/go-jstmpl/types"
	"github.com/lestrrat/go-jshschema"
)

func ParseHschema(file string) (*hschema.HyperSchema, error) {
	b, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("fail to read the source JSON Schema file: %s", err)
	}

	var m map[string]interface{}
	if err := yaml.Unmarshal(b, &m); err != nil {
		return nil, fmt.Errorf("fail to unmarshal YAML: %s", err)
	}

	hs := hschema.New()
	if err := hs.Extract(m); err != nil {
		return nil, fmt.Errorf("fail to extract JSON Schema: %s", err)
	}

	return hs, nil
}

func TestBuilderLoopRef(t *testing.T) {
	hs, err := ParseHschema("./test/ref_loop.yml")
	if err != nil {
		t.Fatal(err)
	}
	b := NewBuilder()
	_, err = b.Build(hs)
	if err == nil {
		t.Fatalf("build should be failed: %+v", b)
	}
}

func TestBuillderNotHaveHref(t *testing.T) {
	hs, err := ParseHschema("./test/has_not_href.yml")
	if err != nil {
		t.Fatal(err)
	}
	b := NewBuilder()
	ts, err := b.Build(hs)
	if err != nil {
		t.Fatalf("fail to build: %s", err)
	}

	if ts.HyperSchema.Title != "has not href" {
		t.Error("fail to get title")
	}
}

func TestBuilderPassBuild(t *testing.T) {
	hs, err := ParseHschema("./test/pass.yml")
	if err != nil {
		t.Fatal(err)
	}
	b := NewBuilder()
	ts, err := b.Build(hs)
	if err != nil {
		t.Fatalf("fail to build: %s", err)
	}

	if len(ts.Objects) != 1 {
		t.Fatal("fail to parse Object type schema")
	}

	if ts.Objects[0] == nil || ts.Objects[0].Title() != "test object" {
		t.Errorf("fail to get Object type schema: %+v", ts.Objects[0])
	}

	if len(ts.Arrays) != 1 {
		t.Fatal("fail to parse Array type schema")
	}

	if ts.Arrays[0] == nil || ts.Arrays[0].Title() != "test array" {
		t.Errorf("fail to get Arrays type schema: %+v", ts.Arrays[0])
	}

	if len(ts.Booleans) != 1 {
		t.Fatal("fail to parse Boolean type schema")
	}

	if ts.Booleans[0] == nil || ts.Booleans[0].Title() != "test boolean" {
		t.Errorf("fail to get Boolean type schema: %+v", ts.Booleans[0])
	}

	if len(ts.Numbers) != 1 {
		t.Fatal("fail to parse Number type schema")
	}

	if ts.Numbers[0] == nil || ts.Numbers[0].Title() != "test number" {
		t.Errorf("fail to get Numbers type schema: %+v", ts.Numbers[0])
	}

	if len(ts.Integers) != 2 {
		t.Fatal("fail to parse Integer type schema")
	}

	if ts.Integers[0] == nil || ts.Integers[0].Title() != "test integer" {
		t.Errorf("fail to get Integer type schema: %+v", ts.Integers[0])
	}

	if len(ts.Strings) != 1 {
		t.Fatal("fail to parse String type schema")
	}

	if ts.Strings[0] == nil || ts.Strings[0].Title() != "test string" {
		t.Errorf("fail to get Strings type schema: %+v", ts.Strings[0])
	}

	if len(ts.Properties) != 2 {
		t.Fatal("fail to parse Properties type schema")
	}

	if len(ts.Definitions) != 7 {
		t.Fatal("fail to parse Definitions")
	}

	for _, v := range ts.Properties {
		switch v.Key() {
		case "test_multitype":
			i, ok := v.(*jstypes.Integer)
			if ok != true {
				t.Errorf("fail to get Properties type not link extra schema: Type Convert:%+v", v)
				continue
			}
			if i.ColumnName != "test multitype" || i.ColumnType != "int" || i.GoType != "int" {
				t.Errorf("fail to get Properties type not link extra schema: Parse: %+v", v)
			}

		case "test_multitype_link":
			i, ok := v.(*jstypes.Integer)
			if ok != true {
				t.Errorf("fail to get Properties type link extra schema: Type Convert:%+v", v)
				continue
			}
			if i.ColumnName != "test multitype" || i.ColumnType != "int" || i.GoType != "int" {
				t.Errorf("fail to get Properties type link extra schema: Parse: %+v", v)
			}

		default:
			t.Errorf("fail to get Properties type schema, specify one of key: %s", v)
		}
	}

	if len(ts.Links) != 1 {
		t.Fatal("fail to parse Links type schema")
	}

	for _, v := range ts.Links {
		switch obj := v.Schema.(type) {
		case *jstypes.Object:
			for _, p := range obj.Properties {
				switch p.Key() {
				case "test_multitype":
					i, ok := p.(*jstypes.Integer)
					if ok != true {
						t.Errorf("fail to get Links type not link extra schema: Type Convert:%+v", v)
						continue
					}
					if i.ColumnName != "test multitype" || i.ColumnType != "int" || i.GoType != "int" {
						t.Errorf("fail to get Properties type not link extra schema: Parse: %+v", v)
					}

				case "test_multitype_link":
					i, ok := p.(*jstypes.Integer)
					if ok != true {
						t.Errorf("fail to get Links type link extra schema: Type Convert:%+v", v)
						continue
					}
					if i.ColumnName != "test multitype" || i.ColumnType != "int" || i.GoType != "int" {
						t.Errorf("fail to get Properties type link extra schema: Parse: %+v", v)
					}
				default:
					t.Errorf("failt to get Links type schema, specify one of keys: %+v", p)
				}
			}
		default:
			t.Errorf("fail to get Links type, should be object schema: %+v", v)
		}
	}
}

func TestResolvePass(t *testing.T) {
	hs, err := ParseHschema("./test/resolve_pass.yml")
	if err != nil {
		t.Fatalf("fail to parse resolve: %v", err)
	}

	if hs.Schema.Definitions["test_link"] == nil {
		t.Fatalf("fail to get schema: %+v", hs)
	}

	sc := hs.Schema.Definitions["test_link"]
	ctx := &jstypes.Context{
		Validations: map[string]bool{},
	}

	for k := range hs.Definitions {
		ctx.Key = k
	}

	scm, err := resolve(sc, hs.Schema, ctx)
	if err != nil {
		t.Fatalf("fail to resolve: %s", err)
	}

	if scm.Title() != "test" {
		t.Errorf("fail to resolve link schema: title should be test: %+v", scm)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/phrase/phraseapp-go/phraseapp"
)

type PushCommand struct {
	Credentials
}

func (cmd *PushCommand) Run() error {
	if cmd.Debug {
		// suppresses content output
		cmd.Debug = false
		Debug = true
	}

	err := func() error {
		client, err := ClientFromCmdCredentials(cmd.Credentials)
		if err != nil {
			return err
		}

		sources, err := SourcesFromConfig(cmd)
		if err != nil {
			return err
		}

		for _, source := range sources {
			err := source.Push(client)
			if err != nil {
				return err
			}
		}
		return nil
	}()

	if err != nil {
		ReportError("Push Error", err.Error())
	}

	return err
}

type Sources []*Source

type Source struct {
	File          string      `yaml:"file,omitempty"`
	ProjectID     string      `yaml:"project_id,omitempty"`
	AccessToken   string      `yaml:"access_token,omitempty"`
	FileFormat    string      `yaml:"file_format,omitempty"`
	Params        *phraseapp.UploadParams `yaml:"params"`

	RemoteLocales []*phraseapp.Locale
	Extension     string
}

var separator = string(os.PathSeparator)

func (source *Source) CheckPreconditions() error {
	if err := ValidPath(source.File, source.FileFormat, ""); err != nil {
		return err
	}

	duplicatedPlaceholders := []string{}
	for _, name := range []string{"<locale_name>", "<locale_code>", "<tag>"} {
		if strings.Count(source.File, name) > 1 {
			duplicatedPlaceholders = append(duplicatedPlaceholders, name)
		}
	}

	starCount := strings.Count(source.File, "*")
	recCount := strings.Count(source.File, "**")

	if recCount == 0 && starCount > 1 || starCount-(recCount*2) > 1 {
		duplicatedPlaceholders = append(duplicatedPlaceholders, "*")
	}

	if recCount > 1 {
		duplicatedPlaceholders = append(duplicatedPlaceholders, "**")
	}

	if len(duplicatedPlaceholders) > 0 {
		dups := strings.Join(duplicatedPlaceholders, ", ")
		return fmt.Errorf(fmt.Sprintf("%s can only occur once in a file pattern!", dups))
	}

	return nil
}

func (source *Source) Push(client *phraseapp.Client) error {
	if err := source.CheckPreconditions(); err != nil {
		return err
	}

	source.Extension = filepath.Ext(source.File)

	remoteLocales, err := RemoteLocales(client, source.ProjectID)
	if err != nil {
		return err
	}
	source.RemoteLocales = remoteLocales

	localeFiles, err := source.LocaleFiles()
	if err != nil {
		return err
	}

	for _, localeFile := range localeFiles {
		fmt.Println("Uploading", localeFile.RelPath())

		if !localeFile.ExistsRemote {
			localeDetails, err := source.createLocale(client, localeFile)
			if err == nil {
				localeFile.ID = localeDetails.ID
				localeFile.RFC = localeDetails.Code
				localeFile.Name = localeDetails.Name
			} else {
				fmt.Printf("failed to create locale: %s\n", err)
				continue
			}
		}

		err = source.uploadFile(client, localeFile)
		if err != nil {
			return err
		}

		sharedMessage("push", localeFile)

		if Debug {
			fmt.Fprintln(os.Stderr, strings.Repeat("-", 10))
		}

	}

	return nil
}

func (source *Source) createLocale(client *phraseapp.Client, localeFile *LocaleFile) (*phraseapp.LocaleDetails, error) {
	if localeFile.RFC == "" {
		return nil, fmt.Errorf("no locale code specified")
	}

	localeParams := new(phraseapp.LocaleParams)

	if localeFile.Name != "" {
		localeParams.Name = &localeFile.Name
	} else if localeFile.RFC != "" {
		localeParams.Name = &localeFile.RFC
	}

	localeName := source.replacePlaceholderInParams(localeFile)
	if localeName != "" && localeName != localeFile.RFC {
		localeParams.Name = &localeName
	}

	if localeFile.RFC != "" {
		localeParams.Code = &localeFile.RFC
	}

	localeDetails, err := client.LocaleCreate(source.ProjectID, localeParams)
	if err != nil {
		return nil, err
	}
	return localeDetails, nil
}

func (source *Source) replacePlaceholderInParams(localeFile *LocaleFile) string {
	if localeFile.RFC != "" && strings.Contains(source.GetLocaleID(), "<locale_code>") {
		return strings.Replace(source.GetLocaleID(), "<locale_code>", localeFile.RFC, 1)
	}
	return ""
}

func (source *Source) uploadFile(client *phraseapp.Client, localeFile *LocaleFile) error {
	if Debug {
		fmt.Fprintln(os.Stdout, "Source file pattern:", source.File)
		fmt.Fprintln(os.Stdout, "Actual file location:", localeFile.Path)
	}

	params := new(phraseapp.UploadParams)
	*params = *source.Params

	params.File = &localeFile.Path

	if params.LocaleID == nil {
		switch {
		case localeFile.ID != "":
			params.LocaleID = &localeFile.ID
		case localeFile.RFC != "":
			params.LocaleID = &localeFile.RFC
		}
	}

	_, err := client.UploadCreate(source.ProjectID, params)
	return err
}

func (source *Source) SystemFiles() ([]string, error) {
	if strings.Contains(source.File, "**") {
		return source.recurse()
	}

	return source.glob()
}

func (source *Source) glob() ([]string, error) {
	withoutPlaceholder := placeholderRegexp.ReplaceAllString(source.File, "*")
	tokens := splitPathToTokens(withoutPlaceholder)

	fileHead := tokens[len(tokens)-1]
	if strings.HasPrefix(fileHead, ".") {
		tokens[len(tokens)-1] = "*" + fileHead
	}
	pattern := strings.Join(tokens, separator)

	files, err := filepath.Glob(pattern)
	if err != nil {
		return nil, err
	}

	if Debug {
		fmt.Fprintln(os.Stderr, "Found", len(files), "files matching the source pattern", pattern)
	}

	return files, nil
}

func (source *Source) recurse() ([]string, error) {
	files := []string{}
	err := filepath.Walk(source.root(), func(path string, f os.FileInfo, err error) error {
		if err != nil {
			errmsg := fmt.Sprintf("%s for pattern: %s", err, source.File)
			ReportError("Push Error", errmsg)
			return fmt.Errorf(errmsg)
		}
		if !f.Mode().IsDir() && strings.HasSuffix(f.Name(), source.Extension) {
			files = append(files, path)
		}
		return nil
	})

	return files, err
}

func (source *Source) root() string {
	parts := splitPathToTokens(source.File)
	rootParts := TakeWhile(parts, func(x string) bool {
		return x != "**"
	})
	root := strings.Join(rootParts, separator)
	if root == "" {
		root = "."
	}
	return root
}

// Return all locale files from disk that match the source pattern.
func (source *Source) LocaleFiles() (LocaleFiles, error) {
	filePaths, err := source.SystemFiles()
	if err != nil {
		return nil, err
	}

	tokens := splitPathToTokens(source.File)

	var localeFiles LocaleFiles
	for _, path := range filePaths {

		pathTokens := splitPathToTokens(path)
		if len(pathTokens) < len(tokens) {
			continue
		}
		localeFile := extractParamsFromPathTokens(tokens, pathTokens)

		absolutePath, err := filepath.Abs(path)
		if err != nil {
			return nil, err
		}
		localeFile.Path = absolutePath

		locale := source.getRemoteLocaleForLocaleFile(localeFile)
		if locale != nil {
			localeFile.ExistsRemote = true
			localeFile.RFC = locale.Code
			localeFile.Name = locale.Name
			localeFile.ID = locale.ID
		}

		if Debug {
			fmt.Println(fmt.Sprintf(
				"RFC:'%s', Name:'%s', Tag;'%s', Pattern:'%s'",
				localeFile.RFC, localeFile.Name, localeFile.Tag,
			))
		}

		localeFiles = append(localeFiles, localeFile)
	}

	if len(localeFiles) <= 0 {
		abs, err := filepath.Abs(source.File)
		if err != nil {
			abs = source.File
		}
		return nil, fmt.Errorf("Could not find any files on your system that matches: '%s'", abs)
	}
	return localeFiles, nil
}

func (source *Source) getRemoteLocaleForLocaleFile(localeFile *LocaleFile) *phraseapp.Locale {
	for _, remote := range source.RemoteLocales {
		if remote.Name == source.GetLocaleID() || remote.ID == source.GetLocaleID() {
			return remote
		}

		localeName := source.replacePlaceholderInParams(localeFile)
		if localeName != "" && strings.Contains(remote.Name, localeName) {
			return remote
		}

		if remote.Name == localeFile.Name {
			return remote
		}

		if remote.Name == localeFile.RFC {
			return remote
		}
	}
	return nil
}

func splitPathToTokens(s string) []string {
	tokens := []string{}
	for _, token := range strings.Split(s, separator) {
		if token == "." || token == "" {
			continue
		}
		tokens = append(tokens, token)
	}
	return tokens
}

func extractParamsFromPathTokens(srcTokens, pathTokens []string) *LocaleFile {
	localeFile := new(LocaleFile)

	for idx, token := range srcTokens {
		pathToken := pathTokens[idx]
		if token == "*" {
			continue
		}
		if token == "**" {
			break
		}
		extractParamFromPathToken(localeFile, token, pathToken)
	}

	if Contains(srcTokens, "**") {
		offset := 1
		for idx := len(srcTokens) - 1; idx >= 0; idx-- {
			token := srcTokens[idx]
			pathToken := pathTokens[len(pathTokens)-offset]
			offset += 1

			if token == "*" {
				continue
			}
			if token == "**" {
				break
			}

			extractParamFromPathToken(localeFile, token, pathToken)
		}
	}

	return localeFile
}

func extractParamFromPathToken(localeFile *LocaleFile, srcToken, pathToken string) {
	groups := placeholderRegexp.FindAllString(srcToken, -1)
	if len(groups) <= 0 {
		return
	}

	match := strings.Replace(srcToken, ".", "[.]", -1)
	if strings.HasPrefix(match, "*") {
		match = strings.Replace(match, "*", ".*", -1)
	}

	for _, group := range groups {
		replacer := fmt.Sprintf("(?P%s.+)", group)
		match = strings.Replace(match, group, replacer, 1)
	}

	if match == "" {
		return
	}

	tmpRegexp, err := regexp.Compile(match)
	if err != nil {
		return
	}

	namedMatches := tmpRegexp.SubexpNames()
	subMatches := tmpRegexp.FindStringSubmatch(pathToken)
	for i, subMatch := range subMatches {
		value := strings.Trim(subMatch, separator)
		switch namedMatches[i] {
		case "locale_code":
			localeFile.RFC = value
		case "locale_name":
			localeFile.Name = value
		case "tag":
			localeFile.Tag = value
		default:
			// ignore
		}
	}
}

// Configuration
type PushConfig struct {
	Phraseapp struct {
		AccessToken string `yaml:"access_token"`
		ProjectID   string `yaml:"project_id"`
		FileFormat  string `yaml:"file_format,omitempty"`
		Push        struct {
			Sources Sources
		}
	}
}

func SourcesFromConfig(cmd *PushCommand) (Sources, error) {
	content, err := ConfigContent()
	if err != nil {
		return nil, err
	}

	var config *PushConfig

	err = yaml.Unmarshal([]byte(content), &config)
	if err != nil {
		return nil, err
	}

	token := config.Phraseapp.AccessToken
	if cmd.Token != "" {
		token = cmd.Token
	}
	projectId := config.Phraseapp.ProjectID
	fileFormat := config.Phraseapp.FileFormat

	if &config.Phraseapp.Push == nil || config.Phraseapp.Push.Sources == nil {
		return nil, fmt.Errorf("no sources for upload specified")
	}

	sources := config.Phraseapp.Push.Sources

	validSources := []*Source{}
	for _, source := range sources {
		if source == nil {
			continue
		}
		if source.ProjectID == "" {
			source.ProjectID = projectId
		}
		if source.AccessToken == "" {
			source.AccessToken = token
		}
		if source.Params == nil {
			source.Params = new(phraseapp.UploadParams)
		}

		if source.Params.FileFormat == nil {
			switch {
			case source.FileFormat != "":
				source.Params.FileFormat = &source.FileFormat
			case fileFormat != "":
				source.Params.FileFormat = &fileFormat
			}
		}
		validSources = append(validSources, source)
	}

	if len(validSources) <= 0 {
		return nil, fmt.Errorf("no sources could be identified! Refine the sources list in your config")
	}

	return validSources, nil
}

func (source *Source) GetLocaleID() string {
	if source.Params != nil && source.Params.LocaleID != nil {
		return *source.Params.LocaleID
	}
	return ""
}
<commit_msg>push: remove undocumented special behavior<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/phrase/phraseapp-go/phraseapp"
)

type PushCommand struct {
	Credentials
}

func (cmd *PushCommand) Run() error {
	if cmd.Debug {
		// suppresses content output
		cmd.Debug = false
		Debug = true
	}

	err := func() error {
		client, err := ClientFromCmdCredentials(cmd.Credentials)
		if err != nil {
			return err
		}

		sources, err := SourcesFromConfig(cmd)
		if err != nil {
			return err
		}

		for _, source := range sources {
			err := source.Push(client)
			if err != nil {
				return err
			}
		}
		return nil
	}()

	if err != nil {
		ReportError("Push Error", err.Error())
	}

	return err
}

type Sources []*Source

type Source struct {
	File          string      `yaml:"file,omitempty"`
	ProjectID     string      `yaml:"project_id,omitempty"`
	AccessToken   string      `yaml:"access_token,omitempty"`
	FileFormat    string      `yaml:"file_format,omitempty"`
	Params        *phraseapp.UploadParams `yaml:"params"`

	RemoteLocales []*phraseapp.Locale
	Extension     string
}

var separator = string(os.PathSeparator)

func (source *Source) CheckPreconditions() error {
	if err := ValidPath(source.File, source.FileFormat, ""); err != nil {
		return err
	}

	duplicatedPlaceholders := []string{}
	for _, name := range []string{"<locale_name>", "<locale_code>", "<tag>"} {
		if strings.Count(source.File, name) > 1 {
			duplicatedPlaceholders = append(duplicatedPlaceholders, name)
		}
	}

	starCount := strings.Count(source.File, "*")
	recCount := strings.Count(source.File, "**")

	if recCount == 0 && starCount > 1 || starCount-(recCount*2) > 1 {
		duplicatedPlaceholders = append(duplicatedPlaceholders, "*")
	}

	if recCount > 1 {
		duplicatedPlaceholders = append(duplicatedPlaceholders, "**")
	}

	if len(duplicatedPlaceholders) > 0 {
		dups := strings.Join(duplicatedPlaceholders, ", ")
		return fmt.Errorf(fmt.Sprintf("%s can only occur once in a file pattern!", dups))
	}

	return nil
}

func (source *Source) Push(client *phraseapp.Client) error {
	if err := source.CheckPreconditions(); err != nil {
		return err
	}

	source.Extension = filepath.Ext(source.File)

	remoteLocales, err := RemoteLocales(client, source.ProjectID)
	if err != nil {
		return err
	}
	source.RemoteLocales = remoteLocales

	localeFiles, err := source.LocaleFiles()
	if err != nil {
		return err
	}

	for _, localeFile := range localeFiles {
		fmt.Println("Uploading", localeFile.RelPath())

		if !localeFile.ExistsRemote {
			localeDetails, err := source.createLocale(client, localeFile)
			if err == nil {
				localeFile.ID = localeDetails.ID
				localeFile.RFC = localeDetails.Code
				localeFile.Name = localeDetails.Name
			} else {
				fmt.Printf("failed to create locale: %s\n", err)
				continue
			}
		}

		err = source.uploadFile(client, localeFile)
		if err != nil {
			return err
		}

		sharedMessage("push", localeFile)

		if Debug {
			fmt.Fprintln(os.Stderr, strings.Repeat("-", 10))
		}

	}

	return nil
}

func (source *Source) createLocale(client *phraseapp.Client, localeFile *LocaleFile) (*phraseapp.LocaleDetails, error) {
	if localeFile.RFC == "" {
		return nil, fmt.Errorf("no locale code specified")
	}

	localeParams := new(phraseapp.LocaleParams)

	if localeFile.Name != "" {
		localeParams.Name = &localeFile.Name
	} else if localeFile.RFC != "" {
		localeParams.Name = &localeFile.RFC
	}

	localeName := source.replacePlaceholderInParams(localeFile)
	if localeName != "" && localeName != localeFile.RFC {
		localeParams.Name = &localeName
	}

	if localeFile.RFC != "" {
		localeParams.Code = &localeFile.RFC
	}

	localeDetails, err := client.LocaleCreate(source.ProjectID, localeParams)
	if err != nil {
		return nil, err
	}
	return localeDetails, nil
}

func (source *Source) replacePlaceholderInParams(localeFile *LocaleFile) string {
	if localeFile.RFC != "" && strings.Contains(source.GetLocaleID(), "<locale_code>") {
		return strings.Replace(source.GetLocaleID(), "<locale_code>", localeFile.RFC, 1)
	}
	return ""
}

func (source *Source) uploadFile(client *phraseapp.Client, localeFile *LocaleFile) error {
	if Debug {
		fmt.Fprintln(os.Stdout, "Source file pattern:", source.File)
		fmt.Fprintln(os.Stdout, "Actual file location:", localeFile.Path)
	}

	params := new(phraseapp.UploadParams)
	*params = *source.Params

	params.File = &localeFile.Path

	if params.LocaleID == nil {
		switch {
		case localeFile.ID != "":
			params.LocaleID = &localeFile.ID
		case localeFile.RFC != "":
			params.LocaleID = &localeFile.RFC
		}
	}

	_, err := client.UploadCreate(source.ProjectID, params)
	return err
}

func (source *Source) SystemFiles() ([]string, error) {
	if strings.Contains(source.File, "**") {
		return source.recurse()
	}

	return source.glob()
}

func (source *Source) glob() ([]string, error) {
	pattern := placeholderRegexp.ReplaceAllString(source.File, "*")
	files, err := filepath.Glob(pattern)
	if err != nil {
		return nil, err
	}

	if Debug {
		fmt.Fprintln(os.Stderr, "Found", len(files), "files matching the source pattern", pattern)
	}

	return files, nil
}

func (source *Source) recurse() ([]string, error) {
	files := []string{}
	err := filepath.Walk(source.root(), func(path string, f os.FileInfo, err error) error {
		if err != nil {
			errmsg := fmt.Sprintf("%s for pattern: %s", err, source.File)
			ReportError("Push Error", errmsg)
			return fmt.Errorf(errmsg)
		}
		if !f.Mode().IsDir() && strings.HasSuffix(f.Name(), source.Extension) {
			files = append(files, path)
		}
		return nil
	})

	return files, err
}

func (source *Source) root() string {
	parts := splitPathToTokens(source.File)
	rootParts := TakeWhile(parts, func(x string) bool {
		return x != "**"
	})
	root := strings.Join(rootParts, separator)
	if root == "" {
		root = "."
	}
	return root
}

// Return all locale files from disk that match the source pattern.
func (source *Source) LocaleFiles() (LocaleFiles, error) {
	filePaths, err := source.SystemFiles()
	if err != nil {
		return nil, err
	}

	tokens := splitPathToTokens(source.File)

	var localeFiles LocaleFiles
	for _, path := range filePaths {

		pathTokens := splitPathToTokens(path)
		if len(pathTokens) < len(tokens) {
			continue
		}
		localeFile := extractParamsFromPathTokens(tokens, pathTokens)

		absolutePath, err := filepath.Abs(path)
		if err != nil {
			return nil, err
		}
		localeFile.Path = absolutePath

		locale := source.getRemoteLocaleForLocaleFile(localeFile)
		if locale != nil {
			localeFile.ExistsRemote = true
			localeFile.RFC = locale.Code
			localeFile.Name = locale.Name
			localeFile.ID = locale.ID
		}

		if Debug {
			fmt.Println(fmt.Sprintf(
				"RFC:'%s', Name:'%s', Tag;'%s', Pattern:'%s'",
				localeFile.RFC, localeFile.Name, localeFile.Tag,
			))
		}

		localeFiles = append(localeFiles, localeFile)
	}

	if len(localeFiles) <= 0 {
		abs, err := filepath.Abs(source.File)
		if err != nil {
			abs = source.File
		}
		return nil, fmt.Errorf("Could not find any files on your system that matches: '%s'", abs)
	}
	return localeFiles, nil
}

func (source *Source) getRemoteLocaleForLocaleFile(localeFile *LocaleFile) *phraseapp.Locale {
	for _, remote := range source.RemoteLocales {
		if remote.Name == source.GetLocaleID() || remote.ID == source.GetLocaleID() {
			return remote
		}

		localeName := source.replacePlaceholderInParams(localeFile)
		if localeName != "" && strings.Contains(remote.Name, localeName) {
			return remote
		}

		if remote.Name == localeFile.Name {
			return remote
		}

		if remote.Name == localeFile.RFC {
			return remote
		}
	}
	return nil
}

func splitPathToTokens(s string) []string {
	tokens := []string{}
	for _, token := range strings.Split(s, separator) {
		if token == "." || token == "" {
			continue
		}
		tokens = append(tokens, token)
	}
	return tokens
}

func extractParamsFromPathTokens(srcTokens, pathTokens []string) *LocaleFile {
	localeFile := new(LocaleFile)

	for idx, token := range srcTokens {
		pathToken := pathTokens[idx]
		if token == "*" {
			continue
		}
		if token == "**" {
			break
		}
		extractParamFromPathToken(localeFile, token, pathToken)
	}

	if Contains(srcTokens, "**") {
		offset := 1
		for idx := len(srcTokens) - 1; idx >= 0; idx-- {
			token := srcTokens[idx]
			pathToken := pathTokens[len(pathTokens)-offset]
			offset += 1

			if token == "*" {
				continue
			}
			if token == "**" {
				break
			}

			extractParamFromPathToken(localeFile, token, pathToken)
		}
	}

	return localeFile
}

func extractParamFromPathToken(localeFile *LocaleFile, srcToken, pathToken string) {
	groups := placeholderRegexp.FindAllString(srcToken, -1)
	if len(groups) <= 0 {
		return
	}

	match := strings.Replace(srcToken, ".", "[.]", -1)
	if strings.HasPrefix(match, "*") {
		match = strings.Replace(match, "*", ".*", -1)
	}

	for _, group := range groups {
		replacer := fmt.Sprintf("(?P%s.+)", group)
		match = strings.Replace(match, group, replacer, 1)
	}

	if match == "" {
		return
	}

	tmpRegexp, err := regexp.Compile(match)
	if err != nil {
		return
	}

	namedMatches := tmpRegexp.SubexpNames()
	subMatches := tmpRegexp.FindStringSubmatch(pathToken)
	for i, subMatch := range subMatches {
		value := strings.Trim(subMatch, separator)
		switch namedMatches[i] {
		case "locale_code":
			localeFile.RFC = value
		case "locale_name":
			localeFile.Name = value
		case "tag":
			localeFile.Tag = value
		default:
			// ignore
		}
	}
}

// Configuration
type PushConfig struct {
	Phraseapp struct {
		AccessToken string `yaml:"access_token"`
		ProjectID   string `yaml:"project_id"`
		FileFormat  string `yaml:"file_format,omitempty"`
		Push        struct {
			Sources Sources
		}
	}
}

func SourcesFromConfig(cmd *PushCommand) (Sources, error) {
	content, err := ConfigContent()
	if err != nil {
		return nil, err
	}

	var config *PushConfig

	err = yaml.Unmarshal([]byte(content), &config)
	if err != nil {
		return nil, err
	}

	token := config.Phraseapp.AccessToken
	if cmd.Token != "" {
		token = cmd.Token
	}
	projectId := config.Phraseapp.ProjectID
	fileFormat := config.Phraseapp.FileFormat

	if &config.Phraseapp.Push == nil || config.Phraseapp.Push.Sources == nil {
		return nil, fmt.Errorf("no sources for upload specified")
	}

	sources := config.Phraseapp.Push.Sources

	validSources := []*Source{}
	for _, source := range sources {
		if source == nil {
			continue
		}
		if source.ProjectID == "" {
			source.ProjectID = projectId
		}
		if source.AccessToken == "" {
			source.AccessToken = token
		}
		if source.Params == nil {
			source.Params = new(phraseapp.UploadParams)
		}

		if source.Params.FileFormat == nil {
			switch {
			case source.FileFormat != "":
				source.Params.FileFormat = &source.FileFormat
			case fileFormat != "":
				source.Params.FileFormat = &fileFormat
			}
		}
		validSources = append(validSources, source)
	}

	if len(validSources) <= 0 {
		return nil, fmt.Errorf("no sources could be identified! Refine the sources list in your config")
	}

	return validSources, nil
}

func (source *Source) GetLocaleID() string {
	if source.Params != nil && source.Params.LocaleID != nil {
		return *source.Params.LocaleID
	}
	return ""
}
<|endoftext|>