<commit_before>package git

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/docker/docker/pkg/locker"
	"github.com/moby/buildkit/cache"
	"github.com/moby/buildkit/cache/metadata"
	"github.com/moby/buildkit/client"
	"github.com/moby/buildkit/identity"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/snapshot"
	"github.com/moby/buildkit/source"
	"github.com/moby/buildkit/util/progress/logs"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	bolt "go.etcd.io/bbolt"
)

var validHex = regexp.MustCompile(`^[a-f0-9]{40}$`)

type Opt struct {
	CacheAccessor cache.Accessor
	MetadataStore *metadata.Store
}

type gitSource struct {
	md     *metadata.Store
	cache  cache.Accessor
	locker *locker.Locker
}

// Supported returns nil if the system supports Git source
func Supported() error {
	if err := exec.Command("git", "version").Run(); err != nil {
		return errors.Wrap(err, "failed to find git binary")
	}
	return nil
}

func NewSource(opt Opt) (source.Source, error) {
	gs := &gitSource{
		md:     opt.MetadataStore,
		cache:  opt.CacheAccessor,
		locker: locker.New(),
	}
	return gs, nil
}

func (gs *gitSource) ID() string {
	return source.GitScheme
}

// needs to be called with repo lock
func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target string, release func(), retErr error) {
	remoteKey := "git-remote::" + remote

	sis, err := gs.md.Search(remoteKey)
	if err != nil {
		return "", nil, errors.Wrapf(err, "failed to search metadata for %s", remote)
	}

	var remoteRef cache.MutableRef
	for _, si := range sis {
		remoteRef, err = gs.cache.GetMutable(ctx, si.ID())
		if err != nil {
			if cache.IsLocked(err) {
				// should never really happen as no other function should access this metadata, but lets be graceful
				logrus.Warnf("mutable ref for %s  %s was locked: %v", remote, si.ID(), err)
				continue
			}
			return "", nil, errors.Wrapf(err, "failed to get mutable ref for %s", remote)
		}
		break
	}

	initializeRepo := false
	if remoteRef == nil {
		remoteRef, err = gs.cache.New(ctx, nil, cache.CachePolicyRetain, cache.WithDescription(fmt.Sprintf("shared git repo for %s", remote)))
		if err != nil {
			return "", nil, errors.Wrapf(err, "failed to create new mutable for %s", remote)
		}
		initializeRepo = true
	}

	releaseRemoteRef := func() {
		remoteRef.Release(context.TODO())
	}

	defer func() {
		if retErr != nil && remoteRef != nil {
			releaseRemoteRef()
		}
	}()

	mount, err := remoteRef.Mount(ctx, false)
	if err != nil {
		return "", nil, err
	}

	lm := snapshot.LocalMounter(mount)
	dir, err := lm.Mount()
	if err != nil {
		return "", nil, err
	}

	defer func() {
		if retErr != nil {
			lm.Unmount()
		}
	}()

	if initializeRepo {
		if _, err := gitWithinDir(ctx, dir, "", "init", "--bare"); err != nil {
			return "", nil, errors.Wrapf(err, "failed to init repo at %s", dir)
		}

		if _, err := gitWithinDir(ctx, dir, "", "remote", "add", "origin", remote); err != nil {
			return "", nil, errors.Wrapf(err, "failed add origin repo at %s", dir)
		}

		// same new remote metadata
		si, _ := gs.md.Get(remoteRef.ID())
		v, err := metadata.NewValue(remoteKey)
		v.Index = remoteKey
		if err != nil {
			return "", nil, err
		}

		if err := si.Update(func(b *bolt.Bucket) error {
			return si.SetValue(b, "git-remote", v)
		}); err != nil {
			return "", nil, err
		}
	}
	return dir, func() {
		lm.Unmount()
		releaseRemoteRef()
	}, nil
}

type gitSourceHandler struct {
	*gitSource
	src      source.GitIdentifier
	cacheKey string
}

func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
	key := sha
	if gs.src.KeepGitDir {
		key += ".git"
	}
	return key
}

func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *session.Manager) (source.SourceInstance, error) {
	gitIdentifier, ok := id.(*source.GitIdentifier)
	if !ok {
		return nil, errors.Errorf("invalid git identifier %v", id)
	}

	return &gitSourceHandler{
		src:       *gitIdentifier,
		gitSource: gs,
	}, nil
}

func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bool, error) {
	remote := gs.src.Remote
	ref := gs.src.Ref
	if ref == "" {
		ref = "master"
	}
	gs.locker.Lock(remote)
	defer gs.locker.Unlock(remote)

	if isCommitSHA(ref) {
		ref = gs.shaToCacheKey(ref)
		gs.cacheKey = ref
		return ref, true, nil
	}

	gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote)
	if err != nil {
		return "", false, err
	}
	defer unmountGitDir()

	// TODO: should we assume that remote tag is immutable? add a timer?

	buf, err := gitWithinDir(ctx, gitDir, "", "ls-remote", "origin", ref)
	if err != nil {
		return "", false, errors.Wrapf(err, "failed to fetch remote %s", remote)
	}
	out := buf.String()
	idx := strings.Index(out, "\t")
	if idx == -1 {
		return "", false, errors.Errorf("failed to find commit SHA from output: %s", string(out))
	}

	sha := string(out[:idx])
	if !isCommitSHA(sha) {
		return "", false, errors.Errorf("invalid commit sha %q", sha)
	}
	sha = gs.shaToCacheKey(sha)
	gs.cacheKey = sha
	return sha, true, nil
}

func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRef, retErr error) {
	ref := gs.src.Ref
	if ref == "" {
		ref = "master"
	}

	cacheKey := gs.cacheKey
	if cacheKey == "" {
		var err error
		cacheKey, _, err = gs.CacheKey(ctx, 0)
		if err != nil {
			return nil, err
		}
	}

	snapshotKey := "git-snapshot::" + cacheKey + ":" + gs.src.Subdir
	gs.locker.Lock(snapshotKey)
	defer gs.locker.Unlock(snapshotKey)

	sis, err := gs.md.Search(snapshotKey)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to search metadata for %s", snapshotKey)
	}
	if len(sis) > 0 {
		return gs.cache.Get(ctx, sis[0].ID())
	}

	gs.locker.Lock(gs.src.Remote)
	defer gs.locker.Unlock(gs.src.Remote)
	gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote)
	if err != nil {
		return nil, err
	}
	defer unmountGitDir()

	doFetch := true
	if isCommitSHA(ref) {
		// skip fetch if commit already exists
		if _, err := gitWithinDir(ctx, gitDir, "", "cat-file", "-e", ref+"^{commit}"); err == nil {
			doFetch = false
		}
	}

	if doFetch {
		// make sure no old lock files have leaked
		os.RemoveAll(filepath.Join(gitDir, "shallow.lock"))

		args := []string{"fetch"}
		if !isCommitSHA(ref) { // TODO: find a branch from ls-remote?
			args = append(args, "--depth=1", "--no-tags")
		} else {
			if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil {
				args = append(args, "--unshallow")
			}
		}
		args = append(args, "origin")
		if !isCommitSHA(ref) {
			args = append(args, "--force", ref+":tags/"+ref)
			// local refs are needed so they would be advertised on next fetches
			// TODO: is there a better way to do this?
		}
		if _, err := gitWithinDir(ctx, gitDir, "", args...); err != nil {
			return nil, errors.Wrapf(err, "failed to fetch remote %s", gs.src.Remote)
		}
	}

	checkoutRef, err := gs.cache.New(ctx, nil, cache.WithRecordType(client.UsageRecordTypeGitCheckout), cache.WithDescription(fmt.Sprintf("git snapshot for %s#%s", gs.src.Remote, ref)))
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create new mutable for %s", gs.src.Remote)
	}

	defer func() {
		if retErr != nil && checkoutRef != nil {
			checkoutRef.Release(context.TODO())
		}
	}()

	mount, err := checkoutRef.Mount(ctx, false)
	if err != nil {
		return nil, err
	}
	lm := snapshot.LocalMounter(mount)
	checkoutDir, err := lm.Mount()
	if err != nil {
		return nil, err
	}
	defer func() {
		if retErr != nil && lm != nil {
			lm.Unmount()
		}
	}()

	if gs.src.KeepGitDir {
		checkoutDirGit := filepath.Join(checkoutDir, ".git")
		if err := os.MkdirAll(checkoutDir, 0711); err != nil {
			return nil, err
		}
		_, err = gitWithinDir(ctx, checkoutDirGit, "", "init")
		if err != nil {
			return nil, err
		}
		_, err = gitWithinDir(ctx, checkoutDirGit, "", "remote", "add", "origin", gitDir)
		if err != nil {
			return nil, err
		}
		pullref := ref
		if isCommitSHA(ref) {
			pullref = "refs/buildkit/" + identity.NewID()
			_, err = gitWithinDir(ctx, gitDir, "", "update-ref", pullref, ref)
			if err != nil {
				return nil, err
			}
		} else {
			pullref += ":" + pullref
		}
		_, err = gitWithinDir(ctx, checkoutDirGit, "", "fetch", "-u", "--depth=1", "origin", pullref)
		if err != nil {
			return nil, err
		}
		_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, "checkout", "FETCH_HEAD")
		if err != nil {
			return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
		}
		gitDir = checkoutDirGit
	} else {
		_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".")
		if err != nil {
			return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
		}
	}

	_, err = gitWithinDir(ctx, gitDir, checkoutDir, "submodule", "update", "--init", "--recursive", "--depth=1")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to update submodules for %s", gs.src.Remote)
	}

	if idmap := mount.IdentityMapping(); idmap != nil {
		u := idmap.RootPair()
		err := filepath.Walk(gitDir, func(p string, f os.FileInfo, err error) error {
			return os.Lchown(p, u.UID, u.GID)
		})
		if err != nil {
			return nil, errors.Wrap(err, "failed to remap git checkout")
		}
	}

	lm.Unmount()
	lm = nil

	snap, err := checkoutRef.Commit(ctx)
	if err != nil {
		return nil, err
	}
	checkoutRef = nil

	defer func() {
		if retErr != nil {
			snap.Release(context.TODO())
		}
	}()

	si, _ := gs.md.Get(snap.ID())
	v, err := metadata.NewValue(snapshotKey)
	v.Index = snapshotKey
	if err != nil {
		return nil, err
	}
	if err := si.Update(func(b *bolt.Bucket) error {
		return si.SetValue(b, "git-snapshot", v)
	}); err != nil {
		return nil, err
	}

	return snap, nil
}

func isCommitSHA(str string) bool {
	return validHex.MatchString(str)
}

func gitWithinDir(ctx context.Context, gitDir, workDir string, args ...string) (*bytes.Buffer, error) {
	a := []string{"--git-dir", gitDir}
	if workDir != "" {
		a = append(a, "--work-tree", workDir)
	}
	return git(ctx, workDir, append(a, args...)...)
}

func git(ctx context.Context, dir string, args ...string) (*bytes.Buffer, error) {
	for {
		stdout, stderr := logs.NewLogStreams(ctx, false)
		defer stdout.Close()
		defer stderr.Close()
		cmd := exec.Command("git", args...)
		cmd.Dir = dir // some commands like submodule require this
		buf := bytes.NewBuffer(nil)
		errbuf := bytes.NewBuffer(nil)
		cmd.Stdout = io.MultiWriter(stdout, buf)
		cmd.Stderr = io.MultiWriter(stderr, errbuf)
		// remote git commands spawn helper processes that inherit FDs and don't
		// handle parent death signal so exec.CommandContext can't be used
		err := runProcessGroup(ctx, cmd)
		if err != nil {
			if strings.Contains(errbuf.String(), "--depth") || strings.Contains(errbuf.String(), "shallow") {
				if newArgs := argsNoDepth(args); len(args) > len(newArgs) {
					args = newArgs
					continue
				}
			}
		}
		return buf, err
	}
}

func argsNoDepth(args []string) []string {
	out := make([]string, 0, len(args))
	for _, a := range args {
		if a != "--depth=1" {
			out = append(out, a)
		}
	}
	return out
}
<commit_msg>Add comment about the use of --force<commit_after>package git

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/docker/docker/pkg/locker"
	"github.com/moby/buildkit/cache"
	"github.com/moby/buildkit/cache/metadata"
	"github.com/moby/buildkit/client"
	"github.com/moby/buildkit/identity"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/snapshot"
	"github.com/moby/buildkit/source"
	"github.com/moby/buildkit/util/progress/logs"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	bolt "go.etcd.io/bbolt"
)

var validHex = regexp.MustCompile(`^[a-f0-9]{40}$`)

type Opt struct {
	CacheAccessor cache.Accessor
	MetadataStore *metadata.Store
}

type gitSource struct {
	md     *metadata.Store
	cache  cache.Accessor
	locker *locker.Locker
}

// Supported returns nil if the system supports Git source
func Supported() error {
	if err := exec.Command("git", "version").Run(); err != nil {
		return errors.Wrap(err, "failed to find git binary")
	}
	return nil
}

func NewSource(opt Opt) (source.Source, error) {
	gs := &gitSource{
		md:     opt.MetadataStore,
		cache:  opt.CacheAccessor,
		locker: locker.New(),
	}
	return gs, nil
}

func (gs *gitSource) ID() string {
	return source.GitScheme
}

// needs to be called with repo lock
func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target string, release func(), retErr error) {
	remoteKey := "git-remote::" + remote

	sis, err := gs.md.Search(remoteKey)
	if err != nil {
		return "", nil, errors.Wrapf(err, "failed to search metadata for %s", remote)
	}

	var remoteRef cache.MutableRef
	for _, si := range sis {
		remoteRef, err = gs.cache.GetMutable(ctx, si.ID())
		if err != nil {
			if cache.IsLocked(err) {
				// should never really happen as no other function should access this metadata, but lets be graceful
				logrus.Warnf("mutable ref for %s  %s was locked: %v", remote, si.ID(), err)
				continue
			}
			return "", nil, errors.Wrapf(err, "failed to get mutable ref for %s", remote)
		}
		break
	}

	initializeRepo := false
	if remoteRef == nil {
		remoteRef, err = gs.cache.New(ctx, nil, cache.CachePolicyRetain, cache.WithDescription(fmt.Sprintf("shared git repo for %s", remote)))
		if err != nil {
			return "", nil, errors.Wrapf(err, "failed to create new mutable for %s", remote)
		}
		initializeRepo = true
	}

	releaseRemoteRef := func() {
		remoteRef.Release(context.TODO())
	}

	defer func() {
		if retErr != nil && remoteRef != nil {
			releaseRemoteRef()
		}
	}()

	mount, err := remoteRef.Mount(ctx, false)
	if err != nil {
		return "", nil, err
	}

	lm := snapshot.LocalMounter(mount)
	dir, err := lm.Mount()
	if err != nil {
		return "", nil, err
	}

	defer func() {
		if retErr != nil {
			lm.Unmount()
		}
	}()

	if initializeRepo {
		if _, err := gitWithinDir(ctx, dir, "", "init", "--bare"); err != nil {
			return "", nil, errors.Wrapf(err, "failed to init repo at %s", dir)
		}

		if _, err := gitWithinDir(ctx, dir, "", "remote", "add", "origin", remote); err != nil {
			return "", nil, errors.Wrapf(err, "failed add origin repo at %s", dir)
		}

		// same new remote metadata
		si, _ := gs.md.Get(remoteRef.ID())
		v, err := metadata.NewValue(remoteKey)
		v.Index = remoteKey
		if err != nil {
			return "", nil, err
		}

		if err := si.Update(func(b *bolt.Bucket) error {
			return si.SetValue(b, "git-remote", v)
		}); err != nil {
			return "", nil, err
		}
	}
	return dir, func() {
		lm.Unmount()
		releaseRemoteRef()
	}, nil
}

type gitSourceHandler struct {
	*gitSource
	src      source.GitIdentifier
	cacheKey string
}

func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
	key := sha
	if gs.src.KeepGitDir {
		key += ".git"
	}
	return key
}

func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *session.Manager) (source.SourceInstance, error) {
	gitIdentifier, ok := id.(*source.GitIdentifier)
	if !ok {
		return nil, errors.Errorf("invalid git identifier %v", id)
	}

	return &gitSourceHandler{
		src:       *gitIdentifier,
		gitSource: gs,
	}, nil
}

func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bool, error) {
	remote := gs.src.Remote
	ref := gs.src.Ref
	if ref == "" {
		ref = "master"
	}
	gs.locker.Lock(remote)
	defer gs.locker.Unlock(remote)

	if isCommitSHA(ref) {
		ref = gs.shaToCacheKey(ref)
		gs.cacheKey = ref
		return ref, true, nil
	}

	gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote)
	if err != nil {
		return "", false, err
	}
	defer unmountGitDir()

	// TODO: should we assume that remote tag is immutable? add a timer?

	buf, err := gitWithinDir(ctx, gitDir, "", "ls-remote", "origin", ref)
	if err != nil {
		return "", false, errors.Wrapf(err, "failed to fetch remote %s", remote)
	}
	out := buf.String()
	idx := strings.Index(out, "\t")
	if idx == -1 {
		return "", false, errors.Errorf("failed to find commit SHA from output: %s", string(out))
	}

	sha := string(out[:idx])
	if !isCommitSHA(sha) {
		return "", false, errors.Errorf("invalid commit sha %q", sha)
	}
	sha = gs.shaToCacheKey(sha)
	gs.cacheKey = sha
	return sha, true, nil
}

func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRef, retErr error) {
	ref := gs.src.Ref
	if ref == "" {
		ref = "master"
	}

	cacheKey := gs.cacheKey
	if cacheKey == "" {
		var err error
		cacheKey, _, err = gs.CacheKey(ctx, 0)
		if err != nil {
			return nil, err
		}
	}

	snapshotKey := "git-snapshot::" + cacheKey + ":" + gs.src.Subdir
	gs.locker.Lock(snapshotKey)
	defer gs.locker.Unlock(snapshotKey)

	sis, err := gs.md.Search(snapshotKey)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to search metadata for %s", snapshotKey)
	}
	if len(sis) > 0 {
		return gs.cache.Get(ctx, sis[0].ID())
	}

	gs.locker.Lock(gs.src.Remote)
	defer gs.locker.Unlock(gs.src.Remote)
	gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote)
	if err != nil {
		return nil, err
	}
	defer unmountGitDir()

	doFetch := true
	if isCommitSHA(ref) {
		// skip fetch if commit already exists
		if _, err := gitWithinDir(ctx, gitDir, "", "cat-file", "-e", ref+"^{commit}"); err == nil {
			doFetch = false
		}
	}

	if doFetch {
		// make sure no old lock files have leaked
		os.RemoveAll(filepath.Join(gitDir, "shallow.lock"))

		args := []string{"fetch"}
		if !isCommitSHA(ref) { // TODO: find a branch from ls-remote?
			args = append(args, "--depth=1", "--no-tags")
		} else {
			if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil {
				args = append(args, "--unshallow")
			}
		}
		args = append(args, "origin")
		if !isCommitSHA(ref) {
			args = append(args, "--force", ref+":tags/"+ref)
			// local refs are needed so they would be advertised on next fetches. Force is used
			// in case the ref is a branch and it now points to a different commit sha
			// TODO: is there a better way to do this?
		}
		if _, err := gitWithinDir(ctx, gitDir, "", args...); err != nil {
			return nil, errors.Wrapf(err, "failed to fetch remote %s", gs.src.Remote)
		}
	}

	checkoutRef, err := gs.cache.New(ctx, nil, cache.WithRecordType(client.UsageRecordTypeGitCheckout), cache.WithDescription(fmt.Sprintf("git snapshot for %s#%s", gs.src.Remote, ref)))
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create new mutable for %s", gs.src.Remote)
	}

	defer func() {
		if retErr != nil && checkoutRef != nil {
			checkoutRef.Release(context.TODO())
		}
	}()

	mount, err := checkoutRef.Mount(ctx, false)
	if err != nil {
		return nil, err
	}
	lm := snapshot.LocalMounter(mount)
	checkoutDir, err := lm.Mount()
	if err != nil {
		return nil, err
	}
	defer func() {
		if retErr != nil && lm != nil {
			lm.Unmount()
		}
	}()

	if gs.src.KeepGitDir {
		checkoutDirGit := filepath.Join(checkoutDir, ".git")
		if err := os.MkdirAll(checkoutDir, 0711); err != nil {
			return nil, err
		}
		_, err = gitWithinDir(ctx, checkoutDirGit, "", "init")
		if err != nil {
			return nil, err
		}
		_, err = gitWithinDir(ctx, checkoutDirGit, "", "remote", "add", "origin", gitDir)
		if err != nil {
			return nil, err
		}
		pullref := ref
		if isCommitSHA(ref) {
			pullref = "refs/buildkit/" + identity.NewID()
			_, err = gitWithinDir(ctx, gitDir, "", "update-ref", pullref, ref)
			if err != nil {
				return nil, err
			}
		} else {
			pullref += ":" + pullref
		}
		_, err = gitWithinDir(ctx, checkoutDirGit, "", "fetch", "-u", "--depth=1", "origin", pullref)
		if err != nil {
			return nil, err
		}
		_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, "checkout", "FETCH_HEAD")
		if err != nil {
			return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
		}
		gitDir = checkoutDirGit
	} else {
		_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".")
		if err != nil {
			return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
		}
	}

	_, err = gitWithinDir(ctx, gitDir, checkoutDir, "submodule", "update", "--init", "--recursive", "--depth=1")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to update submodules for %s", gs.src.Remote)
	}

	if idmap := mount.IdentityMapping(); idmap != nil {
		u := idmap.RootPair()
		err := filepath.Walk(gitDir, func(p string, f os.FileInfo, err error) error {
			return os.Lchown(p, u.UID, u.GID)
		})
		if err != nil {
			return nil, errors.Wrap(err, "failed to remap git checkout")
		}
	}

	lm.Unmount()
	lm = nil

	snap, err := checkoutRef.Commit(ctx)
	if err != nil {
		return nil, err
	}
	checkoutRef = nil

	defer func() {
		if retErr != nil {
			snap.Release(context.TODO())
		}
	}()

	si, _ := gs.md.Get(snap.ID())
	v, err := metadata.NewValue(snapshotKey)
	v.Index = snapshotKey
	if err != nil {
		return nil, err
	}
	if err := si.Update(func(b *bolt.Bucket) error {
		return si.SetValue(b, "git-snapshot", v)
	}); err != nil {
		return nil, err
	}

	return snap, nil
}

func isCommitSHA(str string) bool {
	return validHex.MatchString(str)
}

func gitWithinDir(ctx context.Context, gitDir, workDir string, args ...string) (*bytes.Buffer, error) {
	a := []string{"--git-dir", gitDir}
	if workDir != "" {
		a = append(a, "--work-tree", workDir)
	}
	return git(ctx, workDir, append(a, args...)...)
}

func git(ctx context.Context, dir string, args ...string) (*bytes.Buffer, error) {
	for {
		stdout, stderr := logs.NewLogStreams(ctx, false)
		defer stdout.Close()
		defer stderr.Close()
		cmd := exec.Command("git", args...)
		cmd.Dir = dir // some commands like submodule require this
		buf := bytes.NewBuffer(nil)
		errbuf := bytes.NewBuffer(nil)
		cmd.Stdout = io.MultiWriter(stdout, buf)
		cmd.Stderr = io.MultiWriter(stderr, errbuf)
		// remote git commands spawn helper processes that inherit FDs and don't
		// handle parent death signal so exec.CommandContext can't be used
		err := runProcessGroup(ctx, cmd)
		if err != nil {
			if strings.Contains(errbuf.String(), "--depth") || strings.Contains(errbuf.String(), "shallow") {
				if newArgs := argsNoDepth(args); len(args) > len(newArgs) {
					args = newArgs
					continue
				}
			}
		}
		return buf, err
	}
}

func argsNoDepth(args []string) []string {
	out := make([]string, 0, len(args))
	for _, a := range args {
		if a != "--depth=1" {
			out = append(out, a)
		}
	}
	return out
}
<|endoftext|>


<commit_before>package force

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/nimajalali/go-force/forcejson"
)

const (
	version      = "1.0.0"
	userAgent    = "go-force/" + version
	contentType  = "application/json"
	responseType = "application/json"
)

func get(path string, payload url.Values, out interface{}) error {
	return request("GET", path, payload, nil, out)
}

func post(path string, payload url.Values, body, out interface{}) error {
	return request("POST", path, payload, body, out)
}

func patch(path string, payload url.Values, body, out interface{}) error {
	return request("PATCH", path, payload, body, out)
}

func delete(path string, payload url.Values) error {
	return request("DELETE", path, payload, nil, nil)
}

func request(method, path string, params url.Values, payload, out interface{}) error {
	if err := oauth.Validate(); err != nil {
		return fmt.Errorf("Error creating %v request: %v", method, err)
	}

	// Build Uri
	var uri bytes.Buffer
	uri.WriteString(oauth.InstanceUrl)
	uri.WriteString(path)
	if params != nil && len(params) != 0 {
		uri.WriteString("?")
		uri.WriteString(params.Encode())
	}

	// Build body
	var body io.Reader
	if payload != nil {

		jsonBytes, err := forcejson.Marshal(payload)
		if err != nil {
			return fmt.Errorf("Error marshaling encoded payload: %v", err)
		}

		body = bytes.NewReader(jsonBytes)
	}

	// Build Request
	req, err := http.NewRequest(method, uri.String(), body)
	if err != nil {
		return fmt.Errorf("Error creating %v request: %v", method, err)
	}

	// Add Headers
	req.Header.Set("User-Agent", userAgent)
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("Accept", responseType)
	req.Header.Set("Authorization", fmt.Sprintf("%v %v", "Bearer", oauth.AccessToken))

	// Send
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return fmt.Errorf("Error sending %v request: %v", method, err)
	}
	defer resp.Body.Close()

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("Error reading response bytes: %v", err)
	}

	// Attempt to parse response into out
	if out != nil {
		if err := forcejson.Unmarshal(respBytes, out); err != nil {

			// Attempt to parse response as a force.com api error before returning unmarshal err
			apiErrors := ApiErrors{}
			if marshalErr := forcejson.Unmarshal(respBytes, &apiErrors); marshalErr == nil {
				if apiErrors.Validate() {
					// Check if error is oauth token expired
					if oauth.Expired(apiErrors) {
						// Reauthenticate then attempt query again
						oauthErr := oauth.Authenticate()
						if oauthErr != nil {
							return oauthErr
						}

						return request(method, path, params, payload, out)
					}

					return apiErrors
				}
			}

			// Not a force.com api error. Just an unmarshalling error.
			return fmt.Errorf("Unable to unmarshal response to object: %v", err)
		}
	}

	return nil
}
<commit_msg>Check for errors even when response is not expected, update and delete errors where being missed<commit_after>package force

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/nimajalali/go-force/forcejson"
)

const (
	version      = "1.0.0"
	userAgent    = "go-force/" + version
	contentType  = "application/json"
	responseType = "application/json"
)

func get(path string, payload url.Values, out interface{}) error {
	return request("GET", path, payload, nil, out)
}

func post(path string, payload url.Values, body, out interface{}) error {
	return request("POST", path, payload, body, out)
}

func patch(path string, payload url.Values, body, out interface{}) error {
	return request("PATCH", path, payload, body, out)
}

func delete(path string, payload url.Values) error {
	return request("DELETE", path, payload, nil, nil)
}

func request(method, path string, params url.Values, payload, out interface{}) error {
	if err := oauth.Validate(); err != nil {
		return fmt.Errorf("Error creating %v request: %v", method, err)
	}

	// Build Uri
	var uri bytes.Buffer
	uri.WriteString(oauth.InstanceUrl)
	uri.WriteString(path)
	if params != nil && len(params) != 0 {
		uri.WriteString("?")
		uri.WriteString(params.Encode())
	}

	// Build body
	var body io.Reader
	if payload != nil {

		jsonBytes, err := forcejson.Marshal(payload)
		if err != nil {
			return fmt.Errorf("Error marshaling encoded payload: %v", err)
		}

		body = bytes.NewReader(jsonBytes)
	}

	// Build Request
	req, err := http.NewRequest(method, uri.String(), body)
	if err != nil {
		return fmt.Errorf("Error creating %v request: %v", method, err)
	}

	// Add Headers
	req.Header.Set("User-Agent", userAgent)
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("Accept", responseType)
	req.Header.Set("Authorization", fmt.Sprintf("%v %v", "Bearer", oauth.AccessToken))

	// Send
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return fmt.Errorf("Error sending %v request: %v", method, err)
	}
	defer resp.Body.Close()

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("Error reading response bytes: %v", err)
	}

	// Attempt to parse response into out
	var objectUnmarshalErr error
	if out != nil {
		objectUnmarshalErr = forcejson.Unmarshal(respBytes, out)
		if objectUnmarshalErr == nil {
			return nil
		}
	}

	// Attempt to parse response as a force.com api error before returning object unmarshal err
	apiErrors := ApiErrors{}
	if marshalErr := forcejson.Unmarshal(respBytes, &apiErrors); marshalErr == nil {
		if apiErrors.Validate() {
			// Check if error is oauth token expired
			if oauth.Expired(apiErrors) {
				// Reauthenticate then attempt query again
				oauthErr := oauth.Authenticate()
				if oauthErr != nil {
					return oauthErr
				}

				return request(method, path, params, payload, out)
			}

			return apiErrors
		}
	}

	if objectUnmarshalErr != nil {
		// Not a force.com api error. Just an unmarshalling error.
		return fmt.Errorf("Unable to unmarshal response to object: %v", objectUnmarshalErr)
	}

	// Sometimes no response is expected. For example delete and update. We still have to make sure an error wasn't returned.
	return nil
}
<|endoftext|>


<commit_before>package libkb

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	"time"
)

type PDPKA string

type PDPKALoginPackage struct {
	pdpka5 PDPKA
	pdpka4 PDPKA
}

type loginIdentifier interface {
	value() string
}

type loginIdentifierEmail string

func (l loginIdentifierEmail) value() string { return string(l) }

type loginIdentifierUsername string

func (l loginIdentifierUsername) value() string { return string(l) }

type loginIdentifierUID keybase1.UID

func (l loginIdentifierUID) value() string { return keybase1.UID(l).String() }

func (p PDPKA) String() string { return string(p) }

type authPayload struct {
	Body struct {
		Auth struct {
			Nonce   string `json:"nonce"`
			Session string `json:"session"`
		} `json:"auth"`
		Key struct {
			Email    string `json:"email,omitempty"`
			Host     string `json:"host"`
			Kid      string `json:"kid"`
			UID      string `json:"uid,omitempty"`
			Username string `json:"username,omitempty"`
		} `json:"key"`
		Type    string `json:"type"`
		Version int    `json:"version"`
	} `json:"body"`
	Ctime    int    `json:"ctime"`
	ExpireIn int    `json:"expire_in"`
	Tag      string `json:"tag"`
}

func seedToPDPKAKey(seed []byte) (ret NaclSigningKeyPair, err error) {
	if len(seed) != NaclSigningKeySecretSize {
		return ret, fmt.Errorf("wrong size secret in computePDPKA (%d != %d)", len(seed), NaclSigningKeySecretSize)
	}
	var secret [NaclSigningKeySecretSize]byte
	copy(secret[:], seed)
	return MakeNaclSigningKeyPairFromSecret(secret)
}

func seedToPDPKAKID(seed []byte) (ret keybase1.KID, err error) {
	var signingKey NaclSigningKeyPair
	signingKey, err = seedToPDPKAKey(seed)
	if err != nil {
		return ret, err
	}
	return signingKey.GetKID(), nil
}

func computePDPKA(li loginIdentifier, seed []byte, loginSession []byte) (ret PDPKA, err error) {

	var nonce [16]byte
	if _, err = rand.Read(nonce[:]); err != nil {
		return ret, err
	}

	var ap authPayload
	ap.Body.Auth.Nonce = hex.EncodeToString(nonce[:])
	ap.Body.Auth.Session = base64.StdEncoding.EncodeToString(loginSession)

	var signingKey NaclSigningKeyPair
	signingKey, err = seedToPDPKAKey(seed)
	if err != nil {
		return ret, err
	}

	ap.Body.Key.Kid = signingKey.GetKID().String()
	ap.Body.Key.Host = CanonicalHost
	ap.Body.Type = "auth"
	ap.Body.Version = 1
	ap.Tag = "signature"
	ap.Ctime = int(time.Now().Unix())
	ap.ExpireIn = 60 * 60 * 24 // good for one day, to deal with clock skew

	switch li.(type) {
	case loginIdentifierEmail:
		ap.Body.Key.Email = li.value()
	case loginIdentifierUsername:
		ap.Body.Key.Username = li.value()
	case loginIdentifierUID:
		ap.Body.Key.UID = li.value()
	}

	var jsonRaw []byte
	if jsonRaw, err = json.Marshal(ap); err != nil {
		return ret, err
	}

	var sig string
	if sig, _, err = signingKey.SignToString(jsonRaw); err != nil {
		return ret, err
	}

	ret = PDPKA(sig)
	return ret, nil
}

func computeLoginPackageFromUID(u keybase1.UID, ps *PassphraseStream, loginSession []byte) (ret PDPKALoginPackage, err error) {
	return computeLoginPackage(loginIdentifierUID(u), ps, loginSession)
}

func computeLoginPackageFromEmailOrUsername(eou string, ps *PassphraseStream, loginSession []byte) (ret PDPKALoginPackage, err error) {
	var li loginIdentifier
	if CheckUsername.F(eou) {
		li = loginIdentifierUsername(eou)
	} else if CheckEmail.F(eou) {
		li = loginIdentifierEmail(eou)
	} else {
		return ret, fmt.Errorf("expected an email or username; got neither (%s)", eou)
	}
	return computeLoginPackage(li, ps, loginSession)
}

func computeLoginPackage(li loginIdentifier, ps *PassphraseStream, loginSession []byte) (ret PDPKALoginPackage, err error) {
	ret.pdpka5, err = computePDPKA(li, ps.EdDSASeed(), loginSession)
	if err != nil {
		return ret, err
	}
	ret.pdpka4, err = computePDPKA(li, ps.PWHash(), loginSession)
	if err != nil {
		return ret, err
	}
	return ret, nil
}

func (lp PDPKALoginPackage) PopulateArgs(h *HTTPArgs) {
	h.Add("pdpka4", S{string(lp.pdpka4)})
	h.Add("pdpka5", S{string(lp.pdpka5)})
}

func (lp PDPKALoginPackage) PDPKA4() PDPKA { return lp.pdpka4 }
func (lp PDPKALoginPackage) PDPKA5() PDPKA { return lp.pdpka5 }
<commit_msg>add comments so that lint is clean<commit_after>package libkb

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	"time"
)

// PDPKA is a "Passphrase-Derived Public Key Authentication". In this case, it's a
// armored, packed, signature that's been output by our signing interface.
type PDPKA string

// PDPKALoginPackage contains all relevant PDPKA versions in use at this
// time. For now, versions 4 and 5.
type PDPKALoginPackage struct {
	pdpka5 PDPKA
	pdpka4 PDPKA
}

type loginIdentifier interface {
	value() string
}

type loginIdentifierEmail string

func (l loginIdentifierEmail) value() string { return string(l) }

type loginIdentifierUsername string

func (l loginIdentifierUsername) value() string { return string(l) }

type loginIdentifierUID keybase1.UID

func (l loginIdentifierUID) value() string { return keybase1.UID(l).String() }

func (p PDPKA) String() string { return string(p) }

type authPayload struct {
	Body struct {
		Auth struct {
			Nonce   string `json:"nonce"`
			Session string `json:"session"`
		} `json:"auth"`
		Key struct {
			Email    string `json:"email,omitempty"`
			Host     string `json:"host"`
			Kid      string `json:"kid"`
			UID      string `json:"uid,omitempty"`
			Username string `json:"username,omitempty"`
		} `json:"key"`
		Type    string `json:"type"`
		Version int    `json:"version"`
	} `json:"body"`
	Ctime    int    `json:"ctime"`
	ExpireIn int    `json:"expire_in"`
	Tag      string `json:"tag"`
}

func seedToPDPKAKey(seed []byte) (ret NaclSigningKeyPair, err error) {
	if len(seed) != NaclSigningKeySecretSize {
		return ret, fmt.Errorf("wrong size secret in computePDPKA (%d != %d)", len(seed), NaclSigningKeySecretSize)
	}
	var secret [NaclSigningKeySecretSize]byte
	copy(secret[:], seed)
	return MakeNaclSigningKeyPairFromSecret(secret)
}

func seedToPDPKAKID(seed []byte) (ret keybase1.KID, err error) {
	var signingKey NaclSigningKeyPair
	signingKey, err = seedToPDPKAKey(seed)
	if err != nil {
		return ret, err
	}
	return signingKey.GetKID(), nil
}

func computePDPKA(li loginIdentifier, seed []byte, loginSession []byte) (ret PDPKA, err error) {

	var nonce [16]byte
	if _, err = rand.Read(nonce[:]); err != nil {
		return ret, err
	}

	var ap authPayload
	ap.Body.Auth.Nonce = hex.EncodeToString(nonce[:])
	ap.Body.Auth.Session = base64.StdEncoding.EncodeToString(loginSession)

	var signingKey NaclSigningKeyPair
	signingKey, err = seedToPDPKAKey(seed)
	if err != nil {
		return ret, err
	}

	ap.Body.Key.Kid = signingKey.GetKID().String()
	ap.Body.Key.Host = CanonicalHost
	ap.Body.Type = "auth"
	ap.Body.Version = 1
	ap.Tag = "signature"
	ap.Ctime = int(time.Now().Unix())
	ap.ExpireIn = 60 * 60 * 24 // good for one day, to deal with clock skew

	switch li.(type) {
	case loginIdentifierEmail:
		ap.Body.Key.Email = li.value()
	case loginIdentifierUsername:
		ap.Body.Key.Username = li.value()
	case loginIdentifierUID:
		ap.Body.Key.UID = li.value()
	}

	var jsonRaw []byte
	if jsonRaw, err = json.Marshal(ap); err != nil {
		return ret, err
	}

	var sig string
	if sig, _, err = signingKey.SignToString(jsonRaw); err != nil {
		return ret, err
	}

	ret = PDPKA(sig)
	return ret, nil
}

func computeLoginPackageFromUID(u keybase1.UID, ps *PassphraseStream, loginSession []byte) (ret PDPKALoginPackage, err error) {
	return computeLoginPackage(loginIdentifierUID(u), ps, loginSession)
}

func computeLoginPackageFromEmailOrUsername(eou string, ps *PassphraseStream, loginSession []byte) (ret PDPKALoginPackage, err error) {
	var li loginIdentifier
	if CheckUsername.F(eou) {
		li = loginIdentifierUsername(eou)
	} else if CheckEmail.F(eou) {
		li = loginIdentifierEmail(eou)
	} else {
		return ret, fmt.Errorf("expected an email or username; got neither (%s)", eou)
	}
	return computeLoginPackage(li, ps, loginSession)
}

func computeLoginPackage(li loginIdentifier, ps *PassphraseStream, loginSession []byte) (ret PDPKALoginPackage, err error) {
	ret.pdpka5, err = computePDPKA(li, ps.EdDSASeed(), loginSession)
	if err != nil {
		return ret, err
	}
	ret.pdpka4, err = computePDPKA(li, ps.PWHash(), loginSession)
	if err != nil {
		return ret, err
	}
	return ret, nil
}

// PopulateArgs populates the given HTTP args with parameters in this PDPKA package.
// Right now that includes v4 and v5 of the PDPKA login system.
func (lp PDPKALoginPackage) PopulateArgs(h *HTTPArgs) {
	h.Add("pdpka4", S{string(lp.pdpka4)})
	h.Add("pdpka5", S{string(lp.pdpka5)})
}

// PDPKA4 gets the v4 of the PDPKA token for this login package
func (lp PDPKALoginPackage) PDPKA4() PDPKA { return lp.pdpka4 }

// PDPKA5 gets the v4 of the PDPKA token for this login package
func (lp PDPKALoginPackage) PDPKA5() PDPKA { return lp.pdpka5 }
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/weed-fs/go/glog"
	"github.com/chrislusf/weed-fs/go/util"
	"github.com/chrislusf/weed-fs/go/weed/weed_server"
	"github.com/gorilla/mux"
)

func init() {
	cmdMaster.Run = runMaster // break init cycle
}

var cmdMaster = &Command{
	UsageLine: "master -port=9333",
	Short:     "start a master server",
	Long: `start a master server to provide volume=>location mapping service
  and sequence number of file ids

  `,
}

var (
	mport                   = cmdMaster.Flag.Int("port", 9333, "http listen port")
	masterIp                = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
	masterBindIp            = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	mPublicUrl              = cmdMaster.Flag.String("publicUrl", "", "peer accessible <ip>|<server_name>:port")
	metaFolder              = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
	masterPeers             = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list")
	volumeSizeLimitMB       = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	mpulse                  = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	confFile                = cmdMaster.Flag.String("conf", "/etc/weedfs/weedfs.conf", "Deprecating! xml configuration file")
	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
	mTimeout                = cmdMaster.Flag.Int("idleTimeout", 10, "connection idle seconds")
	mMaxCpu                 = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	garbageThreshold        = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
	masterWhiteListOption   = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	masterSecureKey         = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")

	masterWhiteList []string
)

func runMaster(cmd *Command, args []string) bool {
	if *mMaxCpu < 1 {
		*mMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*mMaxCpu)
	if err := util.TestFolderWritable(*metaFolder); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
	}
	if *masterWhiteListOption != "" {
		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
	}

	r := mux.NewRouter()
	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
		*volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold,
		masterWhiteList, *masterSecureKey,
	)

	listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)

	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)

	listener, e := util.NewListener(listeningAddress, time.Duration(*mTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Master startup error: %v", e)
	}

	go func() {
		time.Sleep(100 * time.Millisecond)
		myMasterAddress := *masterIp + ":" + strconv.Itoa(*mport)
		if *mPublicUrl != "" {
			myMasterAddress = *mPublicUrl
		}
		var peers []string
		if *masterPeers != "" {
			peers = strings.Split(*masterPeers, ",")
		}
		raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
		ms.SetRaftServer(raftServer)
	}()

	if e := http.Serve(listener, r); e != nil {
		glog.Fatalf("Fail to serve: %v", e)
	}
	return true
}
<commit_msg>Adjust publicUrl to volume server only.<commit_after>package main

import (
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/weed-fs/go/glog"
	"github.com/chrislusf/weed-fs/go/util"
	"github.com/chrislusf/weed-fs/go/weed/weed_server"
	"github.com/gorilla/mux"
)

func init() {
	cmdMaster.Run = runMaster // break init cycle
}

var cmdMaster = &Command{
	UsageLine: "master -port=9333",
	Short:     "start a master server",
	Long: `start a master server to provide volume=>location mapping service
  and sequence number of file ids

  `,
}

var (
	mport                   = cmdMaster.Flag.Int("port", 9333, "http listen port")
	masterIp                = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
	masterBindIp            = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	metaFolder              = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
	masterPeers             = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list")
	volumeSizeLimitMB       = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	mpulse                  = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	confFile                = cmdMaster.Flag.String("conf", "/etc/weedfs/weedfs.conf", "Deprecating! xml configuration file")
	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
	mTimeout                = cmdMaster.Flag.Int("idleTimeout", 10, "connection idle seconds")
	mMaxCpu                 = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	garbageThreshold        = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
	masterWhiteListOption   = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	masterSecureKey         = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")

	masterWhiteList []string
)

func runMaster(cmd *Command, args []string) bool {
	if *mMaxCpu < 1 {
		*mMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*mMaxCpu)
	if err := util.TestFolderWritable(*metaFolder); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
	}
	if *masterWhiteListOption != "" {
		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
	}

	r := mux.NewRouter()
	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
		*volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold,
		masterWhiteList, *masterSecureKey,
	)

	listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)

	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)

	listener, e := util.NewListener(listeningAddress, time.Duration(*mTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Master startup error: %v", e)
	}

	go func() {
		time.Sleep(100 * time.Millisecond)
		myMasterAddress := *masterIp + ":" + strconv.Itoa(*mport)
		var peers []string
		if *masterPeers != "" {
			peers = strings.Split(*masterPeers, ",")
		}
		raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
		ms.SetRaftServer(raftServer)
	}()

	if e := http.Serve(listener, r); e != nil {
		glog.Fatalf("Fail to serve: %v", e)
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
)

type StackTrace struct {
	Stack     []*StackItem
	RealStack string
}

func NewStackTrace(in string) StackTrace {
	return parseStackTrace(in)
}

func (s *StackTrace) List() (items []string) {
	for _, stackItem := range s.Stack {
		items = append(items, stackItem.Line())
	}
	return items
}

func (s *StackTrace) ErrorLocation() string {
	lastPanicIndex := -1
	for index, item := range s.Stack {
		if item.isPanic() {
			lastPanicIndex = index
		}
	}

	if lastPanicIndex == -1 {
		return ""
	}

	var errorLocation *StackItem
	for _, item := range s.Stack[lastPanicIndex : len(s.Stack)-1] {
		if item.isGoLibFile() {
			errorLocation = item
			break
		}

		if item.isClientFile() {
			errorLocation = item
			break
		}
	}

	if errorLocation != nil {
		return errorLocation.ItemContext()
	}

	return "no error location found"
}

type StackItem struct {
	Method       string
	Name         string
	AbsolutePath string
	LineNo       string
}

func (s *StackItem) Line() string {
	return fmt.Sprintf("%s:%s - %s", s.AbsolutePath, s.LineNo, s.Method)
}

func (s *StackItem) ItemContext() string {
	return fmt.Sprintf("%s:%s - %s", s.Name, s.LineNo, s.Method)
}

func (s *StackItem) isVendored() bool {
	return strings.Contains(s.AbsolutePath, "/vendor/") || strings.Contains(s.AbsolutePath, "/Godeps/")
}

func (s *StackItem) isPanic() bool {
	return strings.Contains(s.AbsolutePath, "panic.go")
}

func (s *StackItem) isGoLibFile() bool {
	return strings.Contains(s.AbsolutePath, "/github.com/phrase/phraseapp-go/")
}

func (s *StackItem) isClientFile() bool {
	return strings.Contains(s.AbsolutePath, "/github.com/phrase/phraseapp-client/") && !s.isVendored()
}
<commit_msg>unified default message<commit_after>package main

import (
	"fmt"
	"strings"
)

type StackTrace struct {
	Stack     []*StackItem
	RealStack string
}

func NewStackTrace(in string) StackTrace {
	return parseStackTrace(in)
}

func (s *StackTrace) List() (items []string) {
	for _, stackItem := range s.Stack {
		items = append(items, stackItem.Line())
	}
	return items
}

func (s *StackTrace) ErrorLocation() string {
	defaultMessage := "no error location found"

	lastPanicIndex := -1
	for index, item := range s.Stack {
		if item.isPanic() {
			lastPanicIndex = index
		}
	}

	if lastPanicIndex == -1 {
		return defaultMessage
	}

	var errorLocation *StackItem
	for _, item := range s.Stack[lastPanicIndex : len(s.Stack)-1] {
		if item.isGoLibFile() {
			errorLocation = item
			break
		}

		if item.isClientFile() {
			errorLocation = item
			break
		}
	}

	if errorLocation != nil {
		return errorLocation.ItemContext()
	}

	return defaultMessage
}

type StackItem struct {
	Method       string
	Name         string
	AbsolutePath string
	LineNo       string
}

func (s *StackItem) Line() string {
	return fmt.Sprintf("%s:%s - %s", s.AbsolutePath, s.LineNo, s.Method)
}

func (s *StackItem) ItemContext() string {
	return fmt.Sprintf("%s:%s - %s", s.Name, s.LineNo, s.Method)
}

func (s *StackItem) isVendored() bool {
	return strings.Contains(s.AbsolutePath, "/vendor/") || strings.Contains(s.AbsolutePath, "/Godeps/")
}

func (s *StackItem) isPanic() bool {
	return strings.Contains(s.AbsolutePath, "panic.go")
}

func (s *StackItem) isGoLibFile() bool {
	return strings.Contains(s.AbsolutePath, "/github.com/phrase/phraseapp-go/")
}

func (s *StackItem) isClientFile() bool {
	return strings.Contains(s.AbsolutePath, "/github.com/phrase/phraseapp-client/") && !s.isVendored()
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	"github.com/maruel/subcommands"
	"golang.org/x/sync/errgroup"

	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/data/text"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/rts/filegraph/git"
	"go.chromium.org/luci/rts/presubmit/eval"
)

func cmdSelect() *subcommands.Command {
	return &subcommands.Command{
		UsageLine: `select -changed-files <path> -model-dir <path> -skip-test-files <path>`,
		ShortDesc: "compute the set of test files to skip",
		LongDesc: text.Doc(`
			Compute the set of test files to skip.

			Flags -changed-files, -model-dir and -skip-test-files are required.
		`),
		CommandRun: func() subcommands.CommandRun {
			r := &selectRun{}
			r.Flags.StringVar(&r.changedFilesPath, "changed-files", "", text.Doc(`
				Path to the file with changed files.
				Each line of the file must be a filename, with "//" prefix.
			`))
			r.Flags.StringVar(&r.modelDir, "model-dir", "", text.Doc(`
				Path to the directory with the model files.
				Normally it is coming from CIPD package "chromium/rts/model"
				and precomputed by "rts-chromium create-model" command.
			`))
			r.Flags.StringVar(&r.skipTestFilesPath, "skip-test-files", "", text.Doc(`
				Path to the file where to write the test file names that should be skipped.
				Each line of the file will be a filename, with "//" prefix.
			`))
			r.Flags.Float64Var(&r.targetChangeRecall, "target-change-recall", 0.99, text.Doc(`
				The target fraction of bad changes to be caught by the selection strategy.
				It must be a value in (0.0, 1.0) range.
			`))
			return r
		},
	}
}

type selectRun struct {
	baseCommandRun

	// Direct input.

	changedFilesPath   string
	modelDir           string
	skipTestFilesPath  string
	targetChangeRecall float64

	// Indirect input.

	testFiles    stringset.Set
	changedFiles stringset.Set
	strategy     git.SelectionStrategy
	evalResult   *eval.Result
}

func (r *selectRun) validateFlags() error {
	switch {
	case r.changedFilesPath == "":
		return errors.New("-changed-files is required")
	case r.modelDir == "":
		return errors.New("-model-dir is required")
	case r.skipTestFilesPath == "":
		return errors.New("-skip-test-files is required")
	case !(r.targetChangeRecall > 0 && r.targetChangeRecall < 1):
		return errors.New("-target-change-recall must be in (0.0, 1.0) range")
	default:
		return nil
	}
}

func (r *selectRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(a, r, env)
	if len(args) != 0 {
		return r.done(errors.New("unexpected positional arguments"))
	}

	if err := r.validateFlags(); err != nil {
		return r.done(err)
	}

	if err := r.loadInput(ctx); err != nil {
		return r.done(err)
	}

	if t := r.chooseThreshold(); t == nil {
		return r.done(errors.Reason("no threshold for target change recall %.4f", r.targetChangeRecall).Err())
	} else {
		r.strategy.Threshold = t.Value
		logging.Infof(ctx, "chosen threhsold: %#v", r.strategy.Threshold)
	}

	f, err := os.Create(r.skipTestFilesPath)
	if err != nil {
		return r.done(errors.Annotate(err, "failed to create the skip file at %q", r.skipTestFilesPath).Err())
	}
	defer f.Close()
	return r.done(r.selectTests(func(skippedFile string) error {
		_, err := fmt.Fprintln(f, skippedFile)
		return err
	}))
}

// chooseThreshold returns the affectedness threshold based on
// r.targetChangeRecall and r.evalResult.
func (r *selectRun) chooseThreshold() *eval.Threshold {
	var ret *eval.Threshold
	for _, t := range r.evalResult.Thresholds {
		if t.ChangeRecall < r.targetChangeRecall {
			continue
		}
		if ret == nil || ret.ChangeRecall > t.ChangeRecall {
			ret = t
		}
	}
	return ret
}

// loadInput loads all the input of the subcommand.
func (r *selectRun) loadInput(ctx context.Context) error {
	eg, ctx := errgroup.WithContext(ctx)
	defer eg.Wait()

	gitGraphDir := filepath.Join(r.modelDir, "git-file-graph")
	eg.Go(func() error {
		err := r.loadGraph(filepath.Join(gitGraphDir, "graph.fg"))
		return errors.Annotate(err, "failed to load file graph").Err()
	})
	eg.Go(func() error {
		err := r.loadEvalResult(filepath.Join(gitGraphDir, "eval-results.json"))
		return errors.Annotate(err, "failed to load eval results").Err()
	})

	eg.Go(func() (err error) {
		r.testFiles, err = loadStringSet(filepath.Join(r.modelDir, "test-file-set.txt"))
		return errors.Annotate(err, "failed to load test files set").Err()
	})

	eg.Go(func() (err error) {
		r.changedFiles, err = loadStringSet(r.changedFilesPath)
		return errors.Annotate(err, "failed to load changed files set").Err()
	})

	return eg.Wait()
}

// loadEvalResult loads r.evalResult, including threhsolds.
func (r *selectRun) loadEvalResult(fileName string) error {
	f, err := os.Open(fileName)
	if err != nil {
		return err
	}
	defer f.Close()

	// TODO(nodir): consider turning eval.Result into a protobuf message for
	// encoding stability.
	r.evalResult = &eval.Result{}
	return json.NewDecoder(f).Decode(r.evalResult)
}

// loadGraph loads r.strategy.Graph from the model.
func (r *selectRun) loadGraph(fileName string) error {
	f, err := os.Open(fileName)
	if err != nil {
		return err
	}
	defer f.Close()

	// Note: it might be dangerous to sync with the current checkout.
	// There might have been such change in the repo that the chosen threshold,
	// the model or both are no longer good. Thus, do not sync.
	r.strategy.Graph = &git.Graph{}
	return r.strategy.Graph.Read(bufio.NewReader(f))
}

// loadStringSet loads a set of newline-separated strings from a text file.
func loadStringSet(fileName string) (stringset.Set, error) {
	f, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	set := stringset.New(0)
	scan := bufio.NewScanner(f)
	for scan.Scan() {
		set.Add(scan.Text())
	}
	return set, scan.Err()
}
<commit_msg>[rts] Fix typos<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	"github.com/maruel/subcommands"
	"golang.org/x/sync/errgroup"

	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/data/text"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/rts/filegraph/git"
	"go.chromium.org/luci/rts/presubmit/eval"
)

func cmdSelect() *subcommands.Command {
	return &subcommands.Command{
		UsageLine: `select -changed-files <path> -model-dir <path> -skip-test-files <path>`,
		ShortDesc: "compute the set of test files to skip",
		LongDesc: text.Doc(`
			Compute the set of test files to skip.

			Flags -changed-files, -model-dir and -skip-test-files are required.
		`),
		CommandRun: func() subcommands.CommandRun {
			r := &selectRun{}
			r.Flags.StringVar(&r.changedFilesPath, "changed-files", "", text.Doc(`
				Path to the file with changed files.
				Each line of the file must be a filename, with "//" prefix.
			`))
			r.Flags.StringVar(&r.modelDir, "model-dir", "", text.Doc(`
				Path to the directory with the model files.
				Normally it is coming from CIPD package "chromium/rts/model"
				and precomputed by "rts-chromium create-model" command.
			`))
			r.Flags.StringVar(&r.skipTestFilesPath, "skip-test-files", "", text.Doc(`
				Path to the file where to write the test file names that should be skipped.
				Each line of the file will be a filename, with "//" prefix.
			`))
			r.Flags.Float64Var(&r.targetChangeRecall, "target-change-recall", 0.99, text.Doc(`
				The target fraction of bad changes to be caught by the selection strategy.
				It must be a value in (0.0, 1.0) range.
			`))
			return r
		},
	}
}

type selectRun struct {
	baseCommandRun

	// Direct input.

	changedFilesPath   string
	modelDir           string
	skipTestFilesPath  string
	targetChangeRecall float64

	// Indirect input.

	testFiles    stringset.Set
	changedFiles stringset.Set
	strategy     git.SelectionStrategy
	evalResult   *eval.Result
}

func (r *selectRun) validateFlags() error {
	switch {
	case r.changedFilesPath == "":
		return errors.New("-changed-files is required")
	case r.modelDir == "":
		return errors.New("-model-dir is required")
	case r.skipTestFilesPath == "":
		return errors.New("-skip-test-files is required")
	case !(r.targetChangeRecall > 0 && r.targetChangeRecall < 1):
		return errors.New("-target-change-recall must be in (0.0, 1.0) range")
	default:
		return nil
	}
}

func (r *selectRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(a, r, env)
	if len(args) != 0 {
		return r.done(errors.New("unexpected positional arguments"))
	}

	if err := r.validateFlags(); err != nil {
		return r.done(err)
	}

	if err := r.loadInput(ctx); err != nil {
		return r.done(err)
	}

	if t := r.chooseThreshold(); t == nil {
		return r.done(errors.Reason("no threshold for target change recall %.4f", r.targetChangeRecall).Err())
	} else {
		r.strategy.Threshold = t.Value
		logging.Infof(ctx, "chosen threshold: %#v", r.strategy.Threshold)
	}

	f, err := os.Create(r.skipTestFilesPath)
	if err != nil {
		return r.done(errors.Annotate(err, "failed to create the skip file at %q", r.skipTestFilesPath).Err())
	}
	defer f.Close()
	return r.done(r.selectTests(func(skippedFile string) error {
		_, err := fmt.Fprintln(f, skippedFile)
		return err
	}))
}

// chooseThreshold returns the affectedness threshold based on
// r.targetChangeRecall and r.evalResult.
func (r *selectRun) chooseThreshold() *eval.Threshold {
	var ret *eval.Threshold
	for _, t := range r.evalResult.Thresholds {
		if t.ChangeRecall < r.targetChangeRecall {
			continue
		}
		if ret == nil || ret.ChangeRecall > t.ChangeRecall {
			ret = t
		}
	}
	return ret
}

// loadInput loads all the input of the subcommand.
func (r *selectRun) loadInput(ctx context.Context) error {
	eg, ctx := errgroup.WithContext(ctx)
	defer eg.Wait()

	gitGraphDir := filepath.Join(r.modelDir, "git-file-graph")
	eg.Go(func() error {
		err := r.loadGraph(filepath.Join(gitGraphDir, "graph.fg"))
		return errors.Annotate(err, "failed to load file graph").Err()
	})
	eg.Go(func() error {
		err := r.loadEvalResult(filepath.Join(gitGraphDir, "eval-results.json"))
		return errors.Annotate(err, "failed to load eval results").Err()
	})

	eg.Go(func() (err error) {
		r.testFiles, err = loadStringSet(filepath.Join(r.modelDir, "test-file-set.txt"))
		return errors.Annotate(err, "failed to load test files set").Err()
	})

	eg.Go(func() (err error) {
		r.changedFiles, err = loadStringSet(r.changedFilesPath)
		return errors.Annotate(err, "failed to load changed files set").Err()
	})

	return eg.Wait()
}

// loadEvalResult loads r.evalResult, including thresholds.
func (r *selectRun) loadEvalResult(fileName string) error {
	f, err := os.Open(fileName)
	if err != nil {
		return err
	}
	defer f.Close()

	// TODO(nodir): consider turning eval.Result into a protobuf message for
	// encoding stability.
	r.evalResult = &eval.Result{}
	return json.NewDecoder(f).Decode(r.evalResult)
}

// loadGraph loads r.strategy.Graph from the model.
func (r *selectRun) loadGraph(fileName string) error {
	f, err := os.Open(fileName)
	if err != nil {
		return err
	}
	defer f.Close()

	// Note: it might be dangerous to sync with the current checkout.
	// There might have been such change in the repo that the chosen threshold,
	// the model or both are no longer good. Thus, do not sync.
	r.strategy.Graph = &git.Graph{}
	return r.strategy.Graph.Read(bufio.NewReader(f))
}

// loadStringSet loads a set of newline-separated strings from a text file.
func loadStringSet(fileName string) (stringset.Set, error) {
	f, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	set := stringset.New(0)
	scan := bufio.NewScanner(f)
	for scan.Scan() {
		set.Add(scan.Text())
	}
	return set, scan.Err()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/jessevdk/go-flags"
)

const unknownReturnCode = 3

const portPlaceHolder = "{}"

var opts struct {
	Host string `short:"H" long:"hostname" description:"host name" required:"true"`
	Port string `short:"p" long:"port" description:"port number" required:"true"`

	Timeout time.Duration `short:"t" long:"timeout" description:"connection times out" default:"10s"`

	Verbose bool `short:"V" long:"verbose" description:"verbose" default:"false"`

	ProxyProto bool `short:"P" long:"proxyproto" description:"use ProxyProto" default:"true"`
}

// When there is no argument for command line, mikoi is in proxy server mode.
var serverMode bool

func main() {
	args, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(unknownReturnCode)
	}

	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "cmd args:", args)
	}

	if len(args) > 1 && !hasPortPlaceHolder(args) {
		fmt.Fprintln(os.Stderr, "command that mikoi will fork must have at least one port place holder")
		os.Exit(unknownReturnCode)
	}

	if len(args) == 0 {
		if opts.Verbose {
			fmt.Fprintln(os.Stderr, "mikoi is now running as proxy server mode")
		}
		serverMode = true
	}

	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		fmt.Fprintln(os.Stderr, "unable to listen to localhost")
		os.Exit(unknownReturnCode)
	}

	laddr, ok := ln.Addr().(*net.TCPAddr)
	if !ok {
		fmt.Fprintln(os.Stderr, "unable to get a ephemeral port")
		os.Exit(unknownReturnCode)
	}

	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi is now listening to", laddr.Port)
	}

	serverErrCh := make(chan error)
	// launching proxy server
	go server(ln, serverErrCh)

	cmdErrCh := make(chan error)
	if !serverMode {
		cmdArgs := replacePortPlaceHolder(laddr.Port, args)
		if opts.Verbose {
			fmt.Fprintln(os.Stderr, "cmd:", cmdArgs)
		}

		go cmdExecuter(cmdArgs, cmdErrCh)

		if opts.Verbose {
			fmt.Fprintln(os.Stderr, "waiting for command to be finished")
		}
	}

	for {
		select {
		case err := <-serverErrCh:
			fmt.Fprintln(os.Stderr, err)
			if serverMode {
				return
			}
		case err := <-cmdErrCh:
			if opts.Verbose {
				fmt.Fprintf(os.Stderr, "command was finished: %#v\n", err)
			}
			if err == nil {
				return
			}
			if eerr, ok := err.(*exec.ExitError); ok {
				if ws, wsok := eerr.Sys().(syscall.WaitStatus); wsok {
					es := ws.ExitStatus()
					os.Exit(es)
				}
			}
			os.Exit(unknownReturnCode)
		}
	}
}

func serve(conn net.Conn, errCh chan<- error) {
	defer conn.Close()

	// opening a connection to server
	pconn, err := net.Dial("tcp", opts.Host+":"+opts.Port)
	if err != nil {
		errCh <- err
		return
	}
	defer pconn.Close()
	pconn.SetDeadline(time.Now().Add(opts.Timeout))

	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi connects a connection to server")
	}

	// Upgrade pconn to use ProxyProtocol
	if opts.ProxyProto {
		pconn = &ProxyConn{Conn: pconn}
	}

	// proxying a connection from plugin to server
	doneCh := make(chan struct{})
	go func() {
		io.Copy(pconn, conn)
		doneCh <- struct{}{}
	}()
	go func() {
		io.Copy(conn, pconn)
		doneCh <- struct{}{}
	}()

	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi is now proxying traffic")
	}

	<-doneCh

	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi is closing connections")
	}
}

func server(ln net.Listener, errCh chan<- error) {
	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi is launching server")
	}

	defer ln.Close()
	for {
		conn, err := ln.Accept()
		if err != nil {
			errCh <- err
			continue
		}
		conn.SetDeadline(time.Now().Add(opts.Timeout))

		if opts.Verbose {
			fmt.Fprintln(os.Stderr, "mikoi accepts a connection from plugin")
		}

		go serve(conn, errCh)
	}
}

func cmdExecuter(cmdArgs []string, errCh chan<- error) {
	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi is executing command")
	}
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	errCh <- cmd.Run()
}

func hasPortPlaceHolder(args []string) bool {
	for _, a := range args {
		if strings.Contains(a, portPlaceHolder) {
			return true
		}
	}
	return false
}

func replacePortPlaceHolder(port int, args []string) []string {
	p := strconv.Itoa(port)
	ret := make([]string, 0, len(args))
	for _, a := range args {
		ret = append(ret, strings.Replace(a, portPlaceHolder, p, -1))
	}
	return ret
}
<commit_msg>Set timeout for Dial<commit_after>package main

import (
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/jessevdk/go-flags"
)

const unknownReturnCode = 3

const portPlaceHolder = "{}"

var opts struct {
	Host string `short:"H" long:"hostname" description:"host name" required:"true"`
	Port string `short:"p" long:"port" description:"port number" required:"true"`

	Timeout time.Duration `short:"t" long:"timeout" description:"connection times out" default:"10s"`

	Verbose bool `short:"V" long:"verbose" description:"verbose" default:"false"`

	ProxyProto bool `short:"P" long:"proxyproto" description:"use ProxyProto" default:"true"`
}

// When there is no argument for command line, mikoi is in proxy server mode.
var serverMode bool

func main() {
	args, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(unknownReturnCode)
	}

	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "cmd args:", args)
	}

	if len(args) > 1 && !hasPortPlaceHolder(args) {
		fmt.Fprintln(os.Stderr, "command that mikoi will fork must have at least one port place holder")
		os.Exit(unknownReturnCode)
	}

	if len(args) == 0 {
		if opts.Verbose {
			fmt.Fprintln(os.Stderr, "mikoi is now running as proxy server mode")
		}
		serverMode = true
	}

	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		fmt.Fprintln(os.Stderr, "unable to listen to localhost")
		os.Exit(unknownReturnCode)
	}

	laddr, ok := ln.Addr().(*net.TCPAddr)
	if !ok {
		fmt.Fprintln(os.Stderr, "unable to get a ephemeral port")
		os.Exit(unknownReturnCode)
	}

	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi is now listening to", laddr.Port)
	}

	serverErrCh := make(chan error)
	// launching proxy server
	go server(ln, serverErrCh)

	cmdErrCh := make(chan error)
	if !serverMode {
		cmdArgs := replacePortPlaceHolder(laddr.Port, args)
		if opts.Verbose {
			fmt.Fprintln(os.Stderr, "cmd:", cmdArgs)
		}

		go cmdExecuter(cmdArgs, cmdErrCh)

		if opts.Verbose {
			fmt.Fprintln(os.Stderr, "waiting for command to be finished")
		}
	}

	for {
		select {
		case err := <-serverErrCh:
			fmt.Fprintln(os.Stderr, err)
			if serverMode {
				return
			}
		case err := <-cmdErrCh:
			if opts.Verbose {
				fmt.Fprintf(os.Stderr, "command was finished: %#v\n", err)
			}
			if err == nil {
				return
			}
			if eerr, ok := err.(*exec.ExitError); ok {
				if ws, wsok := eerr.Sys().(syscall.WaitStatus); wsok {
					es := ws.ExitStatus()
					os.Exit(es)
				}
			}
			os.Exit(unknownReturnCode)
		}
	}
}

func serve(conn net.Conn, errCh chan<- error) {
	defer conn.Close()
	conn.SetDeadline(time.Now().Add(opts.Timeout))

	// opening a connection to server
	pconn, err := net.DialTimeout("tcp", opts.Host+":"+opts.Port, opts.Timeout)
	if err != nil {
		errCh <- err
		return
	}
	defer pconn.Close()
	pconn.SetDeadline(time.Now().Add(opts.Timeout))

	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi connects a connection to server")
	}

	// Upgrade pconn to use ProxyProtocol
	if opts.ProxyProto {
		pconn = &ProxyConn{Conn: pconn}
	}

	// proxying a connection from plugin to server
	doneCh := make(chan struct{})
	go func() {
		io.Copy(pconn, conn)
		doneCh <- struct{}{}
	}()
	go func() {
		io.Copy(conn, pconn)
		doneCh <- struct{}{}
	}()

	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi is now proxying traffic")
	}

	<-doneCh

	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi is closing connections")
	}
}

func server(ln net.Listener, errCh chan<- error) {
	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi is launching server")
	}

	defer ln.Close()
	for {
		conn, err := ln.Accept()
		if err != nil {
			errCh <- err
			continue
		}

		if opts.Verbose {
			fmt.Fprintln(os.Stderr, "mikoi accepts a connection from plugin")
		}

		go serve(conn, errCh)
	}
}

func cmdExecuter(cmdArgs []string, errCh chan<- error) {
	if opts.Verbose {
		fmt.Fprintln(os.Stderr, "mikoi is executing command")
	}
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	errCh <- cmd.Run()
}

func hasPortPlaceHolder(args []string) bool {
	for _, a := range args {
		if strings.Contains(a, portPlaceHolder) {
			return true
		}
	}
	return false
}

func replacePortPlaceHolder(port int, args []string) []string {
	p := strconv.Itoa(port)
	ret := make([]string, 0, len(args))
	for _, a := range args {
		ret = append(ret, strings.Replace(a, portPlaceHolder, p, -1))
	}
	return ret
}
<|endoftext|>


<commit_before>package main

import (
        "flag"
        "fmt"
        "log"
        "net"
        "time"
        "errors"

        "github.com/lowstz/slackhookgo"
)

type alertMessage struct {
        url   string
        text  string
        color string
}

func (am *alertMessage) sentSlack() {
        err := slackhookgo.Send(
                am.url,
                slackhookgo.NewSlackMessage(
                        "username",
                        "backup",
                ).AddAttachment(
                        slackhookgo.MessageAttachment{
                                Color: am.color,
                                Text:  am.text,
                                Title: "<!channel>",
                        },
                ),
        )
        checkIfError(err)
}

func checkIfError(err error) {
        if err == nil {
                return
        }
        log.Printf("error: %s", err)
}

func main() {
        var (
                c       string
                t       string
                lastSt  error = errors.New("zero step")
                protocol = flag.String("protocol", "tcp", "protocol tcp/udp")
                host     = flag.String("host", "ya.ru", "destination host")
                port     = flag.String("port", "80", "destination port")
                interval = flag.Uint("interval", 5, "interval check seconds")
                url      = flag.String("url", "", "hook url")
        )

        flag.Parse()
        address := fmt.Sprintf("%s:%s", *host, *port)

        for {
                conn, err := net.DialTimeout(*protocol, address, 3*time.Second)
                checkIfError(err)
                if err != lastSt {
                        if err == nil {         // normal
                                c = "good"
                                t = " reachable"
                                conn.Close()
                        } else {                // not normal
                                c = "danger"
                                t = " unreachable"
                        }
                        lastSt = err            // key of success
                        am := alertMessage{
                                color:  c,
                                text:   "Destination host " + *host + ":" + *port + t,
                                url:    *url,
                        }
                        am.sentSlack()
                        fmt.Printf("the message is send\n")
                } else {
                fmt.Printf("to do - nothing\n") // to do - nothin
                }
                time.Sleep(time.Duration(*interval) * time.Second)
        }
}
<commit_msg>Delete check.go<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"fmt"
	"go/ast"
	"go/token"
	"go/types"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/tools/go/loader"
)

func toDiscard(vu *varUsage) bool {
	if vu.discard {
		return true
	}
	for to := range vu.assigned {
		if toDiscard(to) {
			return true
		}
	}
	return false
}

func (v *visitor) interfaceMatching(vr *types.Var, vu *varUsage) (string, string) {
	if toDiscard(vu) {
		return "", ""
	}
	allFuncs := typeFuncMap(vr.Type())
	if allFuncs == nil {
		return "", ""
	}
	called := make(map[string]string, len(vu.calls))
	for fname := range vu.calls {
		called[fname] = allFuncs[fname]
	}
	s := funcMapString(called)
	name := v.ifaceOf(s)
	if name == "" {
		return "", ""
	}
	return name, s
}

func orderedPkgs(prog *loader.Program) ([]*types.Package, error) {
	// InitialPackages() is not in the order that we passed to it
	// via Import() calls.
	// For now, make it deterministic by sorting import paths
	// alphabetically.
	unordered := prog.InitialPackages()
	paths := make([]string, 0, len(unordered))
	byPath := make(map[string]*types.Package, len(unordered))
	for _, info := range unordered {
		if info.Errors != nil {
			return nil, info.Errors[0]
		}
		path := info.Pkg.Path()
		paths = append(paths, path)
		byPath[path] = info.Pkg
	}
	sort.Sort(ByAlph(paths))
	pkgs := make([]*types.Package, 0, len(unordered))
	for _, path := range paths {
		pkgs = append(pkgs, byPath[path])
	}
	return pkgs, nil
}

// relPathErr converts errors by go/types and go/loader that use
// absolute paths into errors with relative paths
func relPathErr(err error, wd string) error {
	errStr := fmt.Sprintf("%v", err)
	if !strings.HasPrefix(errStr, "/") {
		return err
	}
	if strings.HasPrefix(errStr, wd) {
		return fmt.Errorf(errStr[len(wd)+1:])
	}
	return err
}

func CheckArgs(args []string, w io.Writer, verbose bool) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	paths, err := recurse(args)
	if err != nil {
		return err
	}
	c := newCache()
	if _, err := c.FromArgs(paths, false); err != nil {
		return err
	}
	prog, err := c.Load()
	if err != nil {
		return err
	}
	pkgs, err := orderedPkgs(prog)
	if err != nil {
		return relPathErr(err, wd)
	}
	c.typesGet(pkgs)
	for _, pkg := range pkgs {
		info := prog.AllPackages[pkg]
		if verbose {
			fmt.Fprintln(w, info.Pkg.Path())
		}
		v := &visitor{
			cache:       c,
			PackageInfo: info,
			wd:          wd,
			w:           w,
			fset:        prog.Fset,
			vars:        make(map[*types.Var]*varUsage),
		}
		for _, f := range info.Files {
			ast.Walk(v, f)
		}
	}
	return nil
}

type varUsage struct {
	calls   map[string]struct{}
	discard bool

	assigned map[*varUsage]struct{}
}

type visitor struct {
	*cache
	*loader.PackageInfo

	wd    string
	w     io.Writer
	fset  *token.FileSet
	signs []*types.Signature
	warns [][]string
	level int

	vars map[*types.Var]*varUsage
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *visitor) varUsage(id *ast.Ident) *varUsage {
	vr, ok := v.ObjectOf(id).(*types.Var)
	if !ok {
		return nil
	}
	if vu, e := v.vars[vr]; e {
		return vu
	}
	vu := &varUsage{
		calls:    make(map[string]struct{}),
		assigned: make(map[*varUsage]struct{}),
	}
	v.vars[vr] = vu
	return vu
}

func (v *visitor) addUsed(id *ast.Ident, as types.Type) {
	if as == nil {
		return
	}
	vu := v.varUsage(id)
	if vu == nil {
		// not a variable
		return
	}
	iface, ok := as.Underlying().(*types.Interface)
	if !ok {
		vu.discard = true
		return
	}
	for i := 0; i < iface.NumMethods(); i++ {
		m := iface.Method(i)
		vu.calls[m.Name()] = struct{}{}
	}
}

func (v *visitor) addAssign(to, from *ast.Ident) {
	pto := v.varUsage(to)
	pfrom := v.varUsage(from)
	if pto == nil || pfrom == nil {
		// either isn't a variable
		return
	}
	pfrom.assigned[pto] = struct{}{}
}

func (v *visitor) discard(e ast.Expr) {
	id, ok := e.(*ast.Ident)
	if !ok {
		return
	}
	vu := v.varUsage(id)
	if vu == nil {
		// not a variable
		return
	}
	vu.discard = true
}

func (v *visitor) comparedWith(e ast.Expr, with ast.Expr) {
	if _, ok := with.(*ast.BasicLit); ok {
		v.discard(e)
	}
}

func (v *visitor) implementsIface(sign *types.Signature) bool {
	s := signString(sign)
	return v.funcOf(s) != ""
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	var sign *types.Signature
	switch x := node.(type) {
	case *ast.FuncLit:
		sign = v.Types[x].Type.(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
	case *ast.FuncDecl:
		sign = v.Defs[x.Name].Type().(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
	case *ast.SelectorExpr:
		if _, ok := v.TypeOf(x.Sel).(*types.Signature); !ok {
			v.discard(x.X)
		}
	case *ast.UnaryExpr:
		v.discard(x.X)
	case *ast.IndexExpr:
		v.discard(x.X)
	case *ast.IncDecStmt:
		v.discard(x.X)
	case *ast.BinaryExpr:
		v.onBinary(x)
	case *ast.AssignStmt:
		v.onAssign(x)
	case *ast.KeyValueExpr:
		v.onKeyValue(x)
	case *ast.CompositeLit:
		v.onComposite(x)
	case *ast.CallExpr:
		v.onCall(x)
	case nil:
		if top := v.signs[len(v.signs)-1]; top != nil {
			v.funcEnded(top)
		}
		v.signs = v.signs[:len(v.signs)-1]
	}
	if node != nil {
		v.signs = append(v.signs, sign)
		if sign != nil {
			v.level++
		}
	}
	return v
}

func (v *visitor) onBinary(be *ast.BinaryExpr) {
	switch be.Op {
	case token.EQL, token.NEQ:
	default:
		v.discard(be.X)
		v.discard(be.Y)
		return
	}
	v.comparedWith(be.X, be.Y)
	v.comparedWith(be.Y, be.X)
}

func (v *visitor) onAssign(as *ast.AssignStmt) {
	for i, e := range as.Rhs {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		left := as.Lhs[i]
		v.addUsed(id, v.Types[left].Type)
		if lid, ok := left.(*ast.Ident); ok {
			v.addAssign(lid, id)
		}
	}
}

func (v *visitor) onKeyValue(kv *ast.KeyValueExpr) {
	if id, ok := kv.Key.(*ast.Ident); ok {
		v.addUsed(id, v.TypeOf(kv.Value))
	}
	if id, ok := kv.Value.(*ast.Ident); ok {
		v.addUsed(id, v.TypeOf(kv.Key))
	}
}

func (v *visitor) onComposite(cl *ast.CompositeLit) {
	for _, e := range cl.Elts {
		if kv, ok := e.(*ast.KeyValueExpr); ok {
			v.onKeyValue(kv)
		}
	}
}

func (v *visitor) onCall(ce *ast.CallExpr) {
	if sign, ok := v.TypeOf(ce.Fun).(*types.Signature); ok {
		v.onMethodCall(ce, sign)
		return
	}
	if len(ce.Args) == 1 {
		v.discard(ce.Args[0])
	}
}

func (v *visitor) onMethodCall(ce *ast.CallExpr, sign *types.Signature) {
	for i, e := range ce.Args {
		if id, ok := e.(*ast.Ident); ok {
			v.addUsed(id, paramType(sign, i))
		}
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	vu := v.varUsage(left)
	if vu == nil {
		// not a variable
		return
	}
	vu.calls[sel.Sel.Name] = struct{}{}
}

func (v *visitor) funcEnded(sign *types.Signature) {
	v.level--
	v.warns = append(v.warns, v.funcWarns(sign))
	if v.level > 0 {
		return
	}
	for i := len(v.warns) - 1; i >= 0; i-- {
		warns := v.warns[i]
		for _, warn := range warns {
			fmt.Fprintln(v.w, warn)
		}
	}
	v.warns = nil
	v.vars = make(map[*types.Var]*varUsage)
}

func (v *visitor) funcWarns(sign *types.Signature) []string {
	var warns []string
	params := sign.Params()
	for i := 0; i < params.Len(); i++ {
		vr := params.At(i)
		vu := v.vars[vr]
		if vu == nil {
			continue
		}
		if warn := v.paramWarn(vr, vu); warn != "" {
			warns = append(warns, warn)
		}
	}
	return warns
}

func (v *visitor) paramWarn(vr *types.Var, vu *varUsage) string {
	ifname, iftype := v.interfaceMatching(vr, vu)
	if ifname == "" {
		return ""
	}
	t := vr.Type()
	if _, haveIface := t.Underlying().(*types.Interface); haveIface {
		if ifname == t.String() {
			return ""
		}
		have := funcMapString(typeFuncMap(t))
		if have == iftype {
			return ""
		}
	}
	pos := v.fset.Position(vr.Pos())
	fname := pos.Filename
	// go/loader seems to like absolute paths
	if rel, err := filepath.Rel(v.wd, fname); err == nil {
		fname = rel
	}
	pname := v.Pkg.Path()
	if strings.HasPrefix(ifname, pname+".") {
		ifname = ifname[len(pname)+1:]
	}
	return fmt.Sprintf("%s:%d:%d: %s can be %s",
		fname, pos.Line, pos.Column, vr.Name(), ifname)
}
<commit_msg>Add CheckArgs doc<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"fmt"
	"go/ast"
	"go/token"
	"go/types"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/tools/go/loader"
)

func toDiscard(vu *varUsage) bool {
	if vu.discard {
		return true
	}
	for to := range vu.assigned {
		if toDiscard(to) {
			return true
		}
	}
	return false
}

func (v *visitor) interfaceMatching(vr *types.Var, vu *varUsage) (string, string) {
	if toDiscard(vu) {
		return "", ""
	}
	allFuncs := typeFuncMap(vr.Type())
	if allFuncs == nil {
		return "", ""
	}
	called := make(map[string]string, len(vu.calls))
	for fname := range vu.calls {
		called[fname] = allFuncs[fname]
	}
	s := funcMapString(called)
	name := v.ifaceOf(s)
	if name == "" {
		return "", ""
	}
	return name, s
}

func orderedPkgs(prog *loader.Program) ([]*types.Package, error) {
	// InitialPackages() is not in the order that we passed to it
	// via Import() calls.
	// For now, make it deterministic by sorting import paths
	// alphabetically.
	unordered := prog.InitialPackages()
	paths := make([]string, 0, len(unordered))
	byPath := make(map[string]*types.Package, len(unordered))
	for _, info := range unordered {
		if info.Errors != nil {
			return nil, info.Errors[0]
		}
		path := info.Pkg.Path()
		paths = append(paths, path)
		byPath[path] = info.Pkg
	}
	sort.Sort(ByAlph(paths))
	pkgs := make([]*types.Package, 0, len(unordered))
	for _, path := range paths {
		pkgs = append(pkgs, byPath[path])
	}
	return pkgs, nil
}

// relPathErr converts errors by go/types and go/loader that use
// absolute paths into errors with relative paths
func relPathErr(err error, wd string) error {
	errStr := fmt.Sprintf("%v", err)
	if !strings.HasPrefix(errStr, "/") {
		return err
	}
	if strings.HasPrefix(errStr, wd) {
		return fmt.Errorf(errStr[len(wd)+1:])
	}
	return err
}

// CheckArgs checks the packages specified by their import paths in
// args, and writes the results in w. Can give verbose output if
// specified, printing each package as it is checked.
func CheckArgs(args []string, w io.Writer, verbose bool) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	paths, err := recurse(args)
	if err != nil {
		return err
	}
	c := newCache()
	if _, err := c.FromArgs(paths, false); err != nil {
		return err
	}
	prog, err := c.Load()
	if err != nil {
		return err
	}
	pkgs, err := orderedPkgs(prog)
	if err != nil {
		return relPathErr(err, wd)
	}
	c.typesGet(pkgs)
	for _, pkg := range pkgs {
		info := prog.AllPackages[pkg]
		if verbose {
			fmt.Fprintln(w, info.Pkg.Path())
		}
		v := &visitor{
			cache:       c,
			PackageInfo: info,
			wd:          wd,
			w:           w,
			fset:        prog.Fset,
			vars:        make(map[*types.Var]*varUsage),
		}
		for _, f := range info.Files {
			ast.Walk(v, f)
		}
	}
	return nil
}

type varUsage struct {
	calls   map[string]struct{}
	discard bool

	assigned map[*varUsage]struct{}
}

type visitor struct {
	*cache
	*loader.PackageInfo

	wd    string
	w     io.Writer
	fset  *token.FileSet
	signs []*types.Signature
	warns [][]string
	level int

	vars map[*types.Var]*varUsage
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *visitor) varUsage(id *ast.Ident) *varUsage {
	vr, ok := v.ObjectOf(id).(*types.Var)
	if !ok {
		return nil
	}
	if vu, e := v.vars[vr]; e {
		return vu
	}
	vu := &varUsage{
		calls:    make(map[string]struct{}),
		assigned: make(map[*varUsage]struct{}),
	}
	v.vars[vr] = vu
	return vu
}

func (v *visitor) addUsed(id *ast.Ident, as types.Type) {
	if as == nil {
		return
	}
	vu := v.varUsage(id)
	if vu == nil {
		// not a variable
		return
	}
	iface, ok := as.Underlying().(*types.Interface)
	if !ok {
		vu.discard = true
		return
	}
	for i := 0; i < iface.NumMethods(); i++ {
		m := iface.Method(i)
		vu.calls[m.Name()] = struct{}{}
	}
}

func (v *visitor) addAssign(to, from *ast.Ident) {
	pto := v.varUsage(to)
	pfrom := v.varUsage(from)
	if pto == nil || pfrom == nil {
		// either isn't a variable
		return
	}
	pfrom.assigned[pto] = struct{}{}
}

func (v *visitor) discard(e ast.Expr) {
	id, ok := e.(*ast.Ident)
	if !ok {
		return
	}
	vu := v.varUsage(id)
	if vu == nil {
		// not a variable
		return
	}
	vu.discard = true
}

func (v *visitor) comparedWith(e ast.Expr, with ast.Expr) {
	if _, ok := with.(*ast.BasicLit); ok {
		v.discard(e)
	}
}

func (v *visitor) implementsIface(sign *types.Signature) bool {
	s := signString(sign)
	return v.funcOf(s) != ""
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	var sign *types.Signature
	switch x := node.(type) {
	case *ast.FuncLit:
		sign = v.Types[x].Type.(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
	case *ast.FuncDecl:
		sign = v.Defs[x.Name].Type().(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
	case *ast.SelectorExpr:
		if _, ok := v.TypeOf(x.Sel).(*types.Signature); !ok {
			v.discard(x.X)
		}
	case *ast.UnaryExpr:
		v.discard(x.X)
	case *ast.IndexExpr:
		v.discard(x.X)
	case *ast.IncDecStmt:
		v.discard(x.X)
	case *ast.BinaryExpr:
		v.onBinary(x)
	case *ast.AssignStmt:
		v.onAssign(x)
	case *ast.KeyValueExpr:
		v.onKeyValue(x)
	case *ast.CompositeLit:
		v.onComposite(x)
	case *ast.CallExpr:
		v.onCall(x)
	case nil:
		if top := v.signs[len(v.signs)-1]; top != nil {
			v.funcEnded(top)
		}
		v.signs = v.signs[:len(v.signs)-1]
	}
	if node != nil {
		v.signs = append(v.signs, sign)
		if sign != nil {
			v.level++
		}
	}
	return v
}

func (v *visitor) onBinary(be *ast.BinaryExpr) {
	switch be.Op {
	case token.EQL, token.NEQ:
	default:
		v.discard(be.X)
		v.discard(be.Y)
		return
	}
	v.comparedWith(be.X, be.Y)
	v.comparedWith(be.Y, be.X)
}

func (v *visitor) onAssign(as *ast.AssignStmt) {
	for i, e := range as.Rhs {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		left := as.Lhs[i]
		v.addUsed(id, v.Types[left].Type)
		if lid, ok := left.(*ast.Ident); ok {
			v.addAssign(lid, id)
		}
	}
}

func (v *visitor) onKeyValue(kv *ast.KeyValueExpr) {
	if id, ok := kv.Key.(*ast.Ident); ok {
		v.addUsed(id, v.TypeOf(kv.Value))
	}
	if id, ok := kv.Value.(*ast.Ident); ok {
		v.addUsed(id, v.TypeOf(kv.Key))
	}
}

func (v *visitor) onComposite(cl *ast.CompositeLit) {
	for _, e := range cl.Elts {
		if kv, ok := e.(*ast.KeyValueExpr); ok {
			v.onKeyValue(kv)
		}
	}
}

func (v *visitor) onCall(ce *ast.CallExpr) {
	if sign, ok := v.TypeOf(ce.Fun).(*types.Signature); ok {
		v.onMethodCall(ce, sign)
		return
	}
	if len(ce.Args) == 1 {
		v.discard(ce.Args[0])
	}
}

func (v *visitor) onMethodCall(ce *ast.CallExpr, sign *types.Signature) {
	for i, e := range ce.Args {
		if id, ok := e.(*ast.Ident); ok {
			v.addUsed(id, paramType(sign, i))
		}
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	vu := v.varUsage(left)
	if vu == nil {
		// not a variable
		return
	}
	vu.calls[sel.Sel.Name] = struct{}{}
}

func (v *visitor) funcEnded(sign *types.Signature) {
	v.level--
	v.warns = append(v.warns, v.funcWarns(sign))
	if v.level > 0 {
		return
	}
	for i := len(v.warns) - 1; i >= 0; i-- {
		warns := v.warns[i]
		for _, warn := range warns {
			fmt.Fprintln(v.w, warn)
		}
	}
	v.warns = nil
	v.vars = make(map[*types.Var]*varUsage)
}

func (v *visitor) funcWarns(sign *types.Signature) []string {
	var warns []string
	params := sign.Params()
	for i := 0; i < params.Len(); i++ {
		vr := params.At(i)
		vu := v.vars[vr]
		if vu == nil {
			continue
		}
		if warn := v.paramWarn(vr, vu); warn != "" {
			warns = append(warns, warn)
		}
	}
	return warns
}

func (v *visitor) paramWarn(vr *types.Var, vu *varUsage) string {
	ifname, iftype := v.interfaceMatching(vr, vu)
	if ifname == "" {
		return ""
	}
	t := vr.Type()
	if _, haveIface := t.Underlying().(*types.Interface); haveIface {
		if ifname == t.String() {
			return ""
		}
		have := funcMapString(typeFuncMap(t))
		if have == iftype {
			return ""
		}
	}
	pos := v.fset.Position(vr.Pos())
	fname := pos.Filename
	// go/loader seems to like absolute paths
	if rel, err := filepath.Rel(v.wd, fname); err == nil {
		fname = rel
	}
	pname := v.Pkg.Path()
	if strings.HasPrefix(ifname, pname+".") {
		ifname = ifname[len(pname)+1:]
	}
	return fmt.Sprintf("%s:%d:%d: %s can be %s",
		fname, pos.Line, pos.Column, vr.Name(), ifname)
}
<|endoftext|>