<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lib

import (
	"context"
	"crypto"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"time"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/digest"
	repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
	"github.com/maruel/subcommands"
	"golang.org/x/sync/errgroup"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/data/caching/cache"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/isolated"
	isol "go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/system/filesystem"
	"go.chromium.org/luci/common/system/signals"
)

// CmdDownload returns an object for the `download` subcommand.
func CmdDownload(defaultAuthOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "download <options>...",
		ShortDesc: "download directory tree from a CAS server.",
		LongDesc: `Downloads directory tree from the CAS server.

Tree is referenced by their digest "<digest hash>/<size bytes>"`,
		CommandRun: func() subcommands.CommandRun {
			c := downloadRun{}
			c.Init(defaultAuthOpts)
			c.cachePolicies.AddFlags(&c.Flags)
			c.Flags.StringVar(&c.cacheDir, "cache-dir", "", "Cache directory to store downloaded files.")
			c.Flags.StringVar(&c.digest, "digest", "", `Digest of root directory proto "<digest hash>/<size bytes>".`)
			c.Flags.StringVar(&c.dir, "dir", "", "Directory to download tree.")
			c.Flags.StringVar(&c.dumpStatsJSON, "dump-stats-json", "", "Dump download stats to json file.")
			return &c
		},
	}
}

type downloadRun struct {
	commonFlags
	digest        string
	dir           string
	dumpStatsJSON string

	cacheDir      string
	cachePolicies cache.Policies
}

func (r *downloadRun) parse(a subcommands.Application, args []string) error {
	if err := r.commonFlags.Parse(); err != nil {
		return err
	}
	if len(args) != 0 {
		return errors.Reason("position arguments not expected").Err()
	}

	if r.cacheDir == "" && !r.cachePolicies.IsDefault() {
		return errors.New("cache-dir is necessary when cache-max-size, cache-max-items or cache-min-free-space are specified")
	}

	return nil
}

// doDownload downloads directory tree from the CAS server.
func (r *downloadRun) doDownload(ctx context.Context) error {
	ctx, cancel := context.WithCancel(ctx)
	signals.HandleInterrupt(cancel)

	d, err := digest.NewFromString(r.digest)
	if err != nil {
		return errors.Annotate(err, "failed to parse digest: %s", r.digest).Err()
	}

	c, err := newCasClient(ctx, r.casFlags.Instance, r.parsedAuthOpts, true)
	if err != nil {
		return err
	}

	rootDir := &repb.Directory{}
	if err := c.ReadProto(ctx, d, rootDir); err != nil {
		return errors.Annotate(err, "failed to read root directory proto").Err()
	}

	start := time.Now()
	dirs, err := c.GetDirectoryTree(ctx, d.ToProto())
	if err != nil {
		return errors.Annotate(err, "failed to call GetDirectoryTree").Err()
	}
	logger := logging.Get(ctx)
	logger.Infof("finished GetDirectoryTree api call: %d, took %s", len(dirs), time.Since(start))

	start = time.Now()
	t := &repb.Tree{
		Root:     rootDir,
		Children: dirs,
	}

	outputs, err := c.FlattenTree(t, r.dir)
	if err != nil {
		return errors.Annotate(err, "failed to call FlattenTree").Err()
	}

	to := make(map[digest.Digest]*client.TreeOutput)

	var diskcache *cache.Cache
	if r.cacheDir != "" {
		diskcache, err = cache.New(r.cachePolicies, r.cacheDir, crypto.SHA256)
		if err != nil {
			return errors.Annotate(err, "failed to create initialize cache").Err()
		}
		defer diskcache.Close()
	}

	// Files have the same digest are downloaded only once, so we need to
	// copy duplicates files later.
	var dups []*client.TreeOutput

	{
		// Create directories in this scope.
		dirs := make([]string, 0, len(outputs))
		for path, output := range outputs {
			if output.IsEmptyDirectory {
				dirs = append(dirs, path)
			} else {
				dirs = append(dirs, filepath.Dir(path))
			}
		}
		sort.Strings(dirs)

		for i, dir := range dirs {
			if i > 0 && dirs[i-1] == dir {
				continue
			}
			if err := os.MkdirAll(dir, 0o700); err != nil {
				return errors.Annotate(err, "failed to create directory").Err()
			}
		}
	}

	for path, output := range outputs {
		if output.IsEmptyDirectory {
			continue
		}

		if output.SymlinkTarget != "" {
			if err := os.Symlink(output.SymlinkTarget, path); err != nil {
				return errors.Annotate(err, "failed to create symlink").Err()
			}
			continue
		}

		if diskcache != nil && diskcache.Touch(isolated.HexDigest(output.Digest.Hash)) {
			mode := 0o600
			if output.IsExecutable {
				mode = 0o700
			}

			if err := diskcache.Hardlink(isolated.HexDigest(output.Digest.Hash), path, os.FileMode(mode)); err != nil {
				return err
			}
			continue
		}

		if _, ok := to[output.Digest]; ok {
			dups = append(dups, output)
		} else {
			to[output.Digest] = output
		}
	}
	logger.Infof("finished copy from cache (if any), dups: %d, to: %d, took %s", len(dirs), len(to), time.Since(start))

	start = time.Now()
	if err := c.DownloadFiles(ctx, "", to); err != nil {
		return errors.Annotate(err, "failed to download files").Err()
	}
	logger.Infof("finished DownloadFiles api call, took %s", time.Since(start))

	if diskcache != nil {
		start = time.Now()
		outputs := make([]*client.TreeOutput, 0, len(to))
		for _, output := range to {
			outputs = append(outputs, output)
		}

		// This is to utilize locality of disk access.
		sort.Slice(outputs, func(i, j int) bool {
			return outputs[i].Path < outputs[j].Path
		})

		for _, output := range outputs {
			if err := diskcache.AddFileWithoutValidation(
				isolated.HexDigest(output.Digest.Hash), output.Path); err != nil {
				return errors.Annotate(err, "failed to add cache; path=%s digest=%s", output.Path, output.Digest).Err()
			}
		}
		logger.Infof("finished cache addition, took %s", time.Since(start))
	}

	start = time.Now()
	eg, _ := errgroup.WithContext(ctx)

	// limit the number of concurrent I/O threads.
	ch := make(chan struct{}, runtime.NumCPU())

	for _, dup := range dups {
		src := to[dup.Digest]
		dst := dup
		ch <- struct{}{}
		eg.Go(func() (err error) {
			defer func() { <-ch }()
			mode := 0o600
			if dst.IsExecutable {
				mode = 0o700
			}

			if err := filesystem.Copy(dst.Path, src.Path, os.FileMode(mode)); err != nil {
				return errors.Annotate(err, "failed to copy file from '%s' to '%s'", src.Path, dst.Path).Err()
			}

			return nil
		})
	}
	err = eg.Wait()

	logger.Infof("finished files copy of %d, took %s", len(dups), time.Since(start))

	if dsj := r.dumpStatsJSON; dsj != "" {
		cold := make([]int64, 0, len(to))
		for d := range to {
			cold = append(cold, d.Size)
		}
		hot := make([]int64, 0, len(outputs)-len(to))
		for _, output := range outputs {
			d := output.Digest
			if _, ok := to[d]; !ok {
				hot = append(hot, d.Size)
			}
		}

		if err := isol.WriteStats(dsj, hot, cold); err != nil {
			return errors.Annotate(err, "failed to write stats json").Err()
		}
	}

	return err
}

func (r *downloadRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(a, r, env)
	logging.Get(ctx).Infof("start command")
	if err := r.parse(a, args); err != nil {
		errors.Log(ctx, err)
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	defer r.profiler.Stop()

	if err := r.doDownload(ctx); err != nil {
		errors.Log(ctx, err)
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}

	return 0
}
<commit_msg>[cas] extract directory creation to separated function<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lib

import (
	"context"
	"crypto"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"time"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/digest"
	repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
	"github.com/maruel/subcommands"
	"golang.org/x/sync/errgroup"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/data/caching/cache"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/isolated"
	isol "go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/system/filesystem"
	"go.chromium.org/luci/common/system/signals"
)

// CmdDownload returns an object for the `download` subcommand.
func CmdDownload(defaultAuthOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "download <options>...",
		ShortDesc: "download directory tree from a CAS server.",
		LongDesc: `Downloads directory tree from the CAS server.

Tree is referenced by their digest "<digest hash>/<size bytes>"`,
		CommandRun: func() subcommands.CommandRun {
			c := downloadRun{}
			c.Init(defaultAuthOpts)
			c.cachePolicies.AddFlags(&c.Flags)
			c.Flags.StringVar(&c.cacheDir, "cache-dir", "", "Cache directory to store downloaded files.")
			c.Flags.StringVar(&c.digest, "digest", "", `Digest of root directory proto "<digest hash>/<size bytes>".`)
			c.Flags.StringVar(&c.dir, "dir", "", "Directory to download tree.")
			c.Flags.StringVar(&c.dumpStatsJSON, "dump-stats-json", "", "Dump download stats to json file.")
			return &c
		},
	}
}

type downloadRun struct {
	commonFlags
	digest        string
	dir           string
	dumpStatsJSON string

	cacheDir      string
	cachePolicies cache.Policies
}

func (r *downloadRun) parse(a subcommands.Application, args []string) error {
	if err := r.commonFlags.Parse(); err != nil {
		return err
	}
	if len(args) != 0 {
		return errors.Reason("position arguments not expected").Err()
	}

	if r.cacheDir == "" && !r.cachePolicies.IsDefault() {
		return errors.New("cache-dir is necessary when cache-max-size, cache-max-items or cache-min-free-space are specified")
	}

	return nil
}

func createDirectories(outputs map[string]*client.TreeOutput) error {
	dirs := make([]string, 0, len(outputs))
	for path, output := range outputs {
		if output.IsEmptyDirectory {
			dirs = append(dirs, path)
		} else {
			dirs = append(dirs, filepath.Dir(path))
		}
	}
	sort.Strings(dirs)

	for i, dir := range dirs {
		if i > 0 && dirs[i-1] == dir {
			continue
		}
		if err := os.MkdirAll(dir, 0o700); err != nil {
			return errors.Annotate(err, "failed to create directory").Err()
		}
	}
	return nil
}

// doDownload downloads directory tree from the CAS server.
func (r *downloadRun) doDownload(ctx context.Context) error {
	ctx, cancel := context.WithCancel(ctx)
	signals.HandleInterrupt(cancel)

	d, err := digest.NewFromString(r.digest)
	if err != nil {
		return errors.Annotate(err, "failed to parse digest: %s", r.digest).Err()
	}

	c, err := newCasClient(ctx, r.casFlags.Instance, r.parsedAuthOpts, true)
	if err != nil {
		return err
	}

	rootDir := &repb.Directory{}
	if err := c.ReadProto(ctx, d, rootDir); err != nil {
		return errors.Annotate(err, "failed to read root directory proto").Err()
	}

	start := time.Now()
	dirs, err := c.GetDirectoryTree(ctx, d.ToProto())
	if err != nil {
		return errors.Annotate(err, "failed to call GetDirectoryTree").Err()
	}
	logger := logging.Get(ctx)
	logger.Infof("finished GetDirectoryTree api call: %d, took %s", len(dirs), time.Since(start))

	start = time.Now()
	t := &repb.Tree{
		Root:     rootDir,
		Children: dirs,
	}

	outputs, err := c.FlattenTree(t, r.dir)
	if err != nil {
		return errors.Annotate(err, "failed to call FlattenTree").Err()
	}

	to := make(map[digest.Digest]*client.TreeOutput)

	var diskcache *cache.Cache
	if r.cacheDir != "" {
		diskcache, err = cache.New(r.cachePolicies, r.cacheDir, crypto.SHA256)
		if err != nil {
			return errors.Annotate(err, "failed to create initialize cache").Err()
		}
		defer diskcache.Close()
	}

	if err := createDirectories(outputs); err != nil {
		return err
	}

	// Files have the same digest are downloaded only once, so we need to
	// copy duplicates files later.
	var dups []*client.TreeOutput

	for path, output := range outputs {
		if output.IsEmptyDirectory {
			continue
		}

		if output.SymlinkTarget != "" {
			if err := os.Symlink(output.SymlinkTarget, path); err != nil {
				return errors.Annotate(err, "failed to create symlink").Err()
			}
			continue
		}

		if diskcache != nil && diskcache.Touch(isolated.HexDigest(output.Digest.Hash)) {
			mode := 0o600
			if output.IsExecutable {
				mode = 0o700
			}

			if err := diskcache.Hardlink(isolated.HexDigest(output.Digest.Hash), path, os.FileMode(mode)); err != nil {
				return err
			}
			continue
		}

		if _, ok := to[output.Digest]; ok {
			dups = append(dups, output)
		} else {
			to[output.Digest] = output
		}
	}
	logger.Infof("finished copy from cache (if any), dups: %d, to: %d, took %s", len(dirs), len(to), time.Since(start))

	start = time.Now()
	if err := c.DownloadFiles(ctx, "", to); err != nil {
		return errors.Annotate(err, "failed to download files").Err()
	}
	logger.Infof("finished DownloadFiles api call, took %s", time.Since(start))

	if diskcache != nil {
		start = time.Now()
		outputs := make([]*client.TreeOutput, 0, len(to))
		for _, output := range to {
			outputs = append(outputs, output)
		}

		// This is to utilize locality of disk access.
		sort.Slice(outputs, func(i, j int) bool {
			return outputs[i].Path < outputs[j].Path
		})

		for _, output := range outputs {
			if err := diskcache.AddFileWithoutValidation(
				isolated.HexDigest(output.Digest.Hash), output.Path); err != nil {
				return errors.Annotate(err, "failed to add cache; path=%s digest=%s", output.Path, output.Digest).Err()
			}
		}
		logger.Infof("finished cache addition, took %s", time.Since(start))
	}

	start = time.Now()
	eg, _ := errgroup.WithContext(ctx)

	// limit the number of concurrent I/O threads.
	ch := make(chan struct{}, runtime.NumCPU())

	for _, dup := range dups {
		src := to[dup.Digest]
		dst := dup
		ch <- struct{}{}
		eg.Go(func() (err error) {
			defer func() { <-ch }()
			mode := 0o600
			if dst.IsExecutable {
				mode = 0o700
			}

			if err := filesystem.Copy(dst.Path, src.Path, os.FileMode(mode)); err != nil {
				return errors.Annotate(err, "failed to copy file from '%s' to '%s'", src.Path, dst.Path).Err()
			}

			return nil
		})
	}
	err = eg.Wait()

	logger.Infof("finished files copy of %d, took %s", len(dups), time.Since(start))

	if dsj := r.dumpStatsJSON; dsj != "" {
		cold := make([]int64, 0, len(to))
		for d := range to {
			cold = append(cold, d.Size)
		}
		hot := make([]int64, 0, len(outputs)-len(to))
		for _, output := range outputs {
			d := output.Digest
			if _, ok := to[d]; !ok {
				hot = append(hot, d.Size)
			}
		}

		if err := isol.WriteStats(dsj, hot, cold); err != nil {
			return errors.Annotate(err, "failed to write stats json").Err()
		}
	}

	return err
}

func (r *downloadRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(a, r, env)
	logging.Get(ctx).Infof("start command")
	if err := r.parse(a, args); err != nil {
		errors.Log(ctx, err)
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	defer r.profiler.Stop()

	if err := r.doDownload(ctx); err != nil {
		errors.Log(ctx, err)
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}

	return 0
}
<|endoftext|>


<commit_before>// Author  Raido Pahtma
// License MIT

package main

import "fmt"
import "os"
import "os/signal"
import "log"
import "time"
import "regexp"
import "strconv"

import "github.com/jessevdk/go-flags"
import "github.com/proactivity-lab/go-sf-connection"

const ApplicationVersionMajor = 0
const ApplicationVersionMinor = 1
const ApplicationVersionPatch = 0

var ApplicationBuildDate string
var ApplicationBuildDistro string

func main() {

	var opts struct {
		Positional struct {
			ConnectionString string `description:"Connectionstring sf@HOST:PORT"`
		} `positional-args:"yes"`
		Reconnect   uint   `long:"reconnect" default:"30" description:"Reconnect period, seconds"`
		Debug       bool   `short:"D" long:"debug" default:"false" description:"Debug mode, print raw packets"`
		ShowVersion func() `short:"V" long:"version" description:"Show application version"`
	}

	opts.ShowVersion = func() {
		if ApplicationBuildDate == "" {
			ApplicationBuildDate = "YYYY-mm-dd_HH:MM:SS"
		}
		if ApplicationBuildDistro == "" {
			ApplicationBuildDistro = "unknown"
		}
		fmt.Printf("sflistener %d.%d.%d (%s %s)\n", ApplicationVersionMajor, ApplicationVersionMinor, ApplicationVersionPatch, ApplicationBuildDate, ApplicationBuildDistro)
		os.Exit(0)
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Printf("Argument parser error: %s", err)
		os.Exit(1)
	}

	host := "localhost"
	port := 9002

	if opts.Positional.ConnectionString != "" {
		re := regexp.MustCompile("sf@([a-zA-Z0-9.-]+)(:([0-9]+))?")
		match := re.FindStringSubmatch(opts.Positional.ConnectionString) // [sf@localhost:9002 localhost :9002 9002]
		if len(match) == 4 {
			host = match[1]
			if len(match[3]) > 0 {
				p, err := strconv.ParseUint(match[3], 10, 16)
				if err == nil {
					port = int(p)
				} else {
					fmt.Printf("ERROR: %s cannot be used as a TCP port number!\n", match[2])
					os.Exit(1)
				}
			}
		} else {
			fmt.Printf("ERROR: %s cannot be used as a connectionstring!\n", opts.Positional.ConnectionString)
			os.Exit(1)
		}
	}

	signals := make(chan os.Signal)
	signal.Notify(signals, os.Interrupt, os.Kill)

	dsp := sfconnection.NewPacketDispatcher(0, new(sfconnection.Message))
	receive := make(chan sfconnection.Packet)
	dsp.RegisterSnooper(receive)

	sfc := sfconnection.NewSfConnection()
	sfc.AddDispatcher(dsp)

	logger := log.New(os.Stdout, "INFO:  ", log.Ldate|log.Ltime)
	if opts.Debug {
		sfc.SetDebugLogger(log.New(os.Stdout, "DEBUG: ", log.Ldate|log.Ltime))
		sfc.SetInfoLogger(logger)
	}
	sfc.SetWarningLogger(log.New(os.Stdout, "WARN:  ", log.Ldate|log.Ltime))
	sfc.SetErrorLogger(log.New(os.Stdout, "ERROR: ", log.Ldate|log.Ltime))

	sfc.Autoconnect(host, port, time.Duration(opts.Reconnect)*time.Second)

	for interrupted := false; interrupted == false; {
		select {
		case msg := <-receive:
			logger.Printf("%s\n", msg)
		case sig := <-signals:
			signal.Stop(signals)
			logger.Printf("signal %s\n", sig)
			sfc.Disconnect()
			interrupted = true
		}
	}
}
<commit_msg>Reformatted logging.<commit_after>// Author  Raido Pahtma
// License MIT

package main

import "fmt"
import "os"
import "os/signal"
import "log"
import "time"
import "regexp"
import "strconv"

import "github.com/jessevdk/go-flags"
import "github.com/proactivity-lab/go-sf-connection"

const ApplicationVersionMajor = 0
const ApplicationVersionMinor = 1
const ApplicationVersionPatch = 0

var ApplicationBuildDate string
var ApplicationBuildDistro string

func main() {

	var opts struct {
		Positional struct {
			ConnectionString string `description:"Connectionstring sf@HOST:PORT"`
		} `positional-args:"yes"`
		Reconnect   uint   `long:"reconnect" default:"30" description:"Reconnect period, seconds"`
		Debug       bool   `short:"D" long:"debug" default:"false" description:"Debug mode, print raw packets"`
		ShowVersion func() `short:"V" long:"version" description:"Show application version"`
	}

	opts.ShowVersion = func() {
		if ApplicationBuildDate == "" {
			ApplicationBuildDate = "YYYY-mm-dd_HH:MM:SS"
		}
		if ApplicationBuildDistro == "" {
			ApplicationBuildDistro = "unknown"
		}
		fmt.Printf("sflistener %d.%d.%d (%s %s)\n", ApplicationVersionMajor, ApplicationVersionMinor, ApplicationVersionPatch, ApplicationBuildDate, ApplicationBuildDistro)
		os.Exit(0)
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Printf("Argument parser error: %s", err)
		os.Exit(1)
	}

	host := "localhost"
	port := 9002

	if opts.Positional.ConnectionString != "" {
		re := regexp.MustCompile("sf@([a-zA-Z0-9.-]+)(:([0-9]+))?")
		match := re.FindStringSubmatch(opts.Positional.ConnectionString) // [sf@localhost:9002 localhost :9002 9002]
		if len(match) == 4 {
			host = match[1]
			if len(match[3]) > 0 {
				p, err := strconv.ParseUint(match[3], 10, 16)
				if err == nil {
					port = int(p)
				} else {
					fmt.Printf("ERROR: %s cannot be used as a TCP port number!\n", match[2])
					os.Exit(1)
				}
			}
		} else {
			fmt.Printf("ERROR: %s cannot be used as a connectionstring!\n", opts.Positional.ConnectionString)
			os.Exit(1)
		}
	}

	signals := make(chan os.Signal)
	signal.Notify(signals, os.Interrupt, os.Kill)

	dsp := sfconnection.NewPacketDispatcher(0, new(sfconnection.Message))
	receive := make(chan sfconnection.Packet)
	dsp.RegisterSnooper(receive)

	sfc := sfconnection.NewSfConnection()
	sfc.AddDispatcher(dsp)

	logformat := log.Ldate | log.Ltime | log.Lmicroseconds
	var logger *log.Logger
	if opts.Debug {
		logger = log.New(os.Stdout, "INFO:  ", logformat)
		sfc.SetDebugLogger(log.New(os.Stdout, "DEBUG: ", logformat))
		sfc.SetInfoLogger(logger)
	} else {
		logger = log.New(os.Stdout, "", logformat)
	}
	sfc.SetWarningLogger(log.New(os.Stdout, "WARN:  ", logformat))
	sfc.SetErrorLogger(log.New(os.Stdout, "ERROR: ", logformat))

	sfc.Autoconnect(host, port, time.Duration(opts.Reconnect)*time.Second)

	for interrupted := false; interrupted == false; {
		select {
		case msg := <-receive:
			logger.Printf("%s\n", msg)
		case sig := <-signals:
			signal.Stop(signals)
			logger.Printf("signal %s\n", sig)
			sfc.Disconnect()
			interrupted = true
		}
	}
}
<|endoftext|>


<commit_before>// +build v0.1.1

//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

/*
"hooks": {
    "prestart": [{
      "path": "/bin/ls",
      "args": null,
      "env": null
    }],
    "poststop": [{
      "path": "/bin/pwd",
      "args": null,
      "env": null
    }]
  },
*/

package linuxhooks

import (
	"github.com/huawei-openlab/oct/tools/specsValidator/manager"
	"github.com/opencontainers/specs"
)

func TestSuiteLinuxHooksPrestart() string {
	var pre []specs.Hook = []specs.Hook{
		{
			Path: "/bin/bash",
			Args: []string{"-c", "touch /testHooksPrestart.txt"},
			Env:  []string{""},
		},
	}
	var testResult manager.TestResult
	linuxspec, linuxruntimespec := setHooks(pre, true)
	info := " :Prestart Path=" + pre[0].Path
	result, err := testHooks(&linuxspec, &linuxruntimespec, "testHooksPrestart.txt", info)
	testResult.Set("TestSuiteLinuxHooksPrestart", pre, err, result)
	return testResult.Marshal()
}
<commit_msg>modify the hook test case output<commit_after>// +build v0.1.1

//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package linuxhooks

import (
	"github.com/huawei-openlab/oct/tools/specsValidator/manager"
	"github.com/opencontainers/specs"
)

func TestSuiteLinuxHooksPrestart() string {
	var pre []specs.Hook = []specs.Hook{
		{
			Path: "/bin/bash",
			Args: []string{"-c", "touch /testHooksPrestart.txt"},
			Env:  []string{""},
		},
	}
	var testResult manager.TestResult
	linuxspec, linuxruntimespec := setHooks(pre, true)
	info := "may be not support"
	result, err := testHooks(&linuxspec, &linuxruntimespec, "testHooksPrestart.txt", info)
	testResult.Set("TestSuiteLinuxHooksPrestart", pre, err, result)
	return testResult.Marshal()
}
<|endoftext|>


<commit_before>package hooks

import (
	"errors"
	"fmt"
	"reflect"
	"strings"

	"github.com/gernest/ngorm/builder"
	"github.com/gernest/ngorm/engine"
	"github.com/gernest/ngorm/errmsg"
	"github.com/gernest/ngorm/model"
	"github.com/gernest/ngorm/scope"
	"github.com/gernest/ngorm/search"
	"github.com/gernest/ngorm/util"
)

//Query executes SQL querries.
func Query(b *Book, e *engine.Engine) error {
	var isSlice, isPtr bool
	var resultType reflect.Type
	results := reflect.ValueOf(e.Scope.Value)
	if results.Kind() == reflect.Ptr {
		results = results.Elem()
	}
	if orderBy, ok := e.Scope.Get(model.OrderByPK); ok {
		pf, err := scope.PrimaryField(e, e.Scope.Value)
		if err != nil {
		} else {
			search.Order(e, fmt.Sprintf("%v.%v %v",
				scope.QuotedTableName(e, e.Scope.Value), scope.Quote(e, pf.DBName), orderBy))
		}

	}
	if value, ok := e.Scope.Get(model.QueryDestination); ok {
		results = reflect.Indirect(reflect.ValueOf(value))
	}
	if kind := results.Kind(); kind == reflect.Slice {
		isSlice = true
		resultType = results.Type().Elem()
		results.Set(reflect.MakeSlice(results.Type(), 0, 0))

		if resultType.Kind() == reflect.Ptr {
			isPtr = true
			resultType = resultType.Elem()
		}
	} else if kind != reflect.Struct {
		return errors.New("unsupported destination, should be slice or struct")
	}
	err := builder.PrepareQuery(e, e.Scope.Value)
	if err != nil {
		return err
	}
	e.RowsAffected = 0
	if str, ok := e.Scope.Get(model.QueryOption); ok {
		e.Scope.SQL += util.AddExtraSpaceIfExist(fmt.Sprint(str))
	}

	rows, err := e.SQLDB.Query(e.Scope.SQL, e.Scope.SQLVars...)
	if err != nil {
		return err
	}
	defer func() {
		_ = rows.Close()
	}()

	columns, _ := rows.Columns()
	for rows.Next() {
		e.RowsAffected++
		elem := results
		if isSlice {
			elem = reflect.New(resultType).Elem()
		}
		fields, err := scope.Fields(e, elem.Addr().Interface())
		if err != nil {
			return err
		}
		scope.Scan(rows, columns, fields)
		if isSlice {
			if isPtr {
				results.Set(reflect.Append(results, elem.Addr()))
			} else {
				results.Set(reflect.Append(results, elem))
			}
		}
	}
	return nil
}

//AfterQuery executes any call back after the  Qeery hook has been executed. Any
//callback registered with qeky model.HookQueryAfterFind will be executed.
func AfterQuery(b *Book, e *engine.Engine) error {
	af, ok := b.Query.Get(model.HookAfterFindQuery)
	if ok {
		return af.Exec(b, e)
	}
	return nil
}

//BeforeCreate a callback executed before crating anew record.
func BeforeCreate(b *Book, e *engine.Engine) error {
	bs, ok := b.Create.Get(model.HookBeforeSave)
	if ok {
		err := bs.Exec(b, e)
		if err != nil {
			return err
		}
	}
	bc, ok := b.Create.Get(model.HookBeforeCreate)
	if ok {
		err := bc.Exec(b, e)
		if err != nil {
			return err
		}
	}
	return nil
}

//Create the hook executed to create a new record.
func Create(b *Book, e *engine.Engine) error {
	var (
		columns, placeholders []string

		// The blank columns with default values
		cv []string
	)
	fds, err := scope.Fields(e, e.Scope.Value)
	if err != nil {
		return err
	}

	for _, field := range fds {
		if scope.ChangeableField(e, field) {
			if field.IsNormal {
				if field.IsBlank && field.HasDefaultValue {
					cv = append(cv, scope.Quote(e, field.DBName))
					e.Scope.Set("gorm:blank_columns_with_default_value", cv)
				} else if !field.IsPrimaryKey || !field.IsBlank {
					columns = append(columns, scope.Quote(e, field.DBName))
					placeholders = append(placeholders, scope.AddToVars(e, field.Field.Interface()))
				}
			} else if field.Relationship != nil && field.Relationship.Kind == "belongs_to" {
				for _, foreignKey := range field.Relationship.ForeignDBNames {
					foreignField, err := scope.FieldByName(e, e.Scope.Value, foreignKey)
					if err != nil {
						return err
					}
					if !scope.ChangeableField(e, foreignField) {
						columns = append(columns, scope.Quote(e, foreignField.DBName))
						placeholders = append(placeholders, scope.AddToVars(e, foreignField.Field.Interface()))
					}
				}
			}
		}
	}

	var (
		returningColumn = "*"
		tableName       = scope.QuotedTableName(e, e.Scope.Value)

		extraOption string
	)

	primaryField, err := scope.PrimaryField(e, e.Scope.Value)
	if err != nil {
		return err
	}
	if str, ok := e.Scope.Get("gorm:insert_option"); ok {
		extraOption = fmt.Sprint(str)
	}

	if primaryField != nil {
		returningColumn = scope.Quote(e, primaryField.DBName)
	}

	lastInsertIDReturningSuffix :=
		e.Dialect.LastInsertIDReturningSuffix(tableName, returningColumn)

	if len(columns) == 0 {
		sql := fmt.Sprintf(
			"INSERT INTO %v DEFAULT VALUES%v%v",
			tableName,
			util.AddExtraSpaceIfExist(extraOption),
			util.AddExtraSpaceIfExist(lastInsertIDReturningSuffix),
		)
		e.Scope.SQL = strings.Replace(sql, "$$", "?", -1)
	} else {
		sql := fmt.Sprintf(
			"INSERT INTO %v (%v) VALUES (%v)%v%v",
			scope.QuotedTableName(e, e.Scope.Value),
			strings.Join(columns, ","),
			strings.Join(placeholders, ","),
			util.AddExtraSpaceIfExist(extraOption),
			util.AddExtraSpaceIfExist(lastInsertIDReturningSuffix),
		)
		e.Scope.SQL = strings.Replace(sql, "$$", "?", -1)
	}

	// execute create sql
	if lastInsertIDReturningSuffix == "" || primaryField == nil {
		result, err := e.SQLDB.Exec(e.Scope.SQL, e.Scope.SQLVars...)
		if err != nil {
			return err
		}
		// set rows affected count
		e.RowsAffected, _ = result.RowsAffected()

		// set primary value to primary field
		if primaryField != nil && primaryField.IsBlank {
			primaryValue, err := result.LastInsertId()
			if err != nil {
				return err
			}
			_ = primaryField.Set(primaryValue)
		}
	} else {
		if primaryField.Field.CanAddr() {
			err := e.SQLDB.QueryRow(
				e.Scope.SQL,
				e.Scope.SQLVars...,
			).Scan(primaryField.Field.Addr().Interface())
			if err != nil {
				return err
			}
			primaryField.IsBlank = false
			e.RowsAffected = 1
		} else {
			return errmsg.ErrUnaddressable
		}
	}
	return nil
}

//AfterCreate hook executed after a new record has been created.
func AfterCreate(b *Book, e *engine.Engine) error {
	ac, ok := b.Create.Get(model.HookAfterCreate)
	if ok {
		err := ac.Exec(b, e)
		if err != nil {
			return err
		}
	}
	as, ok := b.Create.Get(model.HookAfterSave)
	if ok {
		err := as.Exec(b, e)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>[hooks] Don't execute SQL code in Create hook<commit_after>package hooks

import (
	"errors"
	"fmt"
	"reflect"
	"strings"

	"github.com/gernest/ngorm/builder"
	"github.com/gernest/ngorm/engine"
	"github.com/gernest/ngorm/model"
	"github.com/gernest/ngorm/scope"
	"github.com/gernest/ngorm/search"
	"github.com/gernest/ngorm/util"
)

//Query executes SQL querries.
func Query(b *Book, e *engine.Engine) error {
	var isSlice, isPtr bool
	var resultType reflect.Type
	results := reflect.ValueOf(e.Scope.Value)
	if results.Kind() == reflect.Ptr {
		results = results.Elem()
	}
	if orderBy, ok := e.Scope.Get(model.OrderByPK); ok {
		pf, err := scope.PrimaryField(e, e.Scope.Value)
		if err != nil {
		} else {
			search.Order(e, fmt.Sprintf("%v.%v %v",
				scope.QuotedTableName(e, e.Scope.Value), scope.Quote(e, pf.DBName), orderBy))
		}

	}
	if value, ok := e.Scope.Get(model.QueryDestination); ok {
		results = reflect.Indirect(reflect.ValueOf(value))
	}
	if kind := results.Kind(); kind == reflect.Slice {
		isSlice = true
		resultType = results.Type().Elem()
		results.Set(reflect.MakeSlice(results.Type(), 0, 0))

		if resultType.Kind() == reflect.Ptr {
			isPtr = true
			resultType = resultType.Elem()
		}
	} else if kind != reflect.Struct {
		return errors.New("unsupported destination, should be slice or struct")
	}
	err := builder.PrepareQuery(e, e.Scope.Value)
	if err != nil {
		return err
	}
	e.RowsAffected = 0
	if str, ok := e.Scope.Get(model.QueryOption); ok {
		e.Scope.SQL += util.AddExtraSpaceIfExist(fmt.Sprint(str))
	}

	rows, err := e.SQLDB.Query(e.Scope.SQL, e.Scope.SQLVars...)
	if err != nil {
		return err
	}
	defer func() {
		_ = rows.Close()
	}()

	columns, _ := rows.Columns()
	for rows.Next() {
		e.RowsAffected++
		elem := results
		if isSlice {
			elem = reflect.New(resultType).Elem()
		}
		fields, err := scope.Fields(e, elem.Addr().Interface())
		if err != nil {
			return err
		}
		scope.Scan(rows, columns, fields)
		if isSlice {
			if isPtr {
				results.Set(reflect.Append(results, elem.Addr()))
			} else {
				results.Set(reflect.Append(results, elem))
			}
		}
	}
	return nil
}

//AfterQuery executes any call back after the  Qeery hook has been executed. Any
//callback registered with qeky model.HookQueryAfterFind will be executed.
func AfterQuery(b *Book, e *engine.Engine) error {
	af, ok := b.Query.Get(model.HookAfterFindQuery)
	if ok {
		return af.Exec(b, e)
	}
	return nil
}

//BeforeCreate a callback executed before crating anew record.
func BeforeCreate(b *Book, e *engine.Engine) error {
	bs, ok := b.Create.Get(model.HookBeforeSave)
	if ok {
		err := bs.Exec(b, e)
		if err != nil {
			return err
		}
	}
	bc, ok := b.Create.Get(model.HookBeforeCreate)
	if ok {
		err := bc.Exec(b, e)
		if err != nil {
			return err
		}
	}
	return nil
}

//Create the hook executed to create a new record.
func Create(b *Book, e *engine.Engine) error {
	var (
		columns, placeholders []string

		// The blank columns with default values
		cv []string
	)
	fds, err := scope.Fields(e, e.Scope.Value)
	if err != nil {
		return err
	}

	for _, field := range fds {
		if scope.ChangeableField(e, field) {
			if field.IsNormal {
				if field.IsBlank && field.HasDefaultValue {
					cv = append(cv, scope.Quote(e, field.DBName))
					e.Scope.Set("gorm:blank_columns_with_default_value", cv)
				} else if !field.IsPrimaryKey || !field.IsBlank {
					columns = append(columns, scope.Quote(e, field.DBName))
					placeholders = append(placeholders, scope.AddToVars(e, field.Field.Interface()))
				}
			} else if field.Relationship != nil && field.Relationship.Kind == "belongs_to" {
				for _, foreignKey := range field.Relationship.ForeignDBNames {
					foreignField, err := scope.FieldByName(e, e.Scope.Value, foreignKey)
					if err != nil {
						return err
					}
					if !scope.ChangeableField(e, foreignField) {
						columns = append(columns, scope.Quote(e, foreignField.DBName))
						placeholders = append(placeholders, scope.AddToVars(e, foreignField.Field.Interface()))
					}
				}
			}
		}
	}

	var (
		returningColumn = "*"
		tableName       = scope.QuotedTableName(e, e.Scope.Value)

		extraOption string
	)

	primaryField, err := scope.PrimaryField(e, e.Scope.Value)
	if err != nil {
		return err
	}
	if str, ok := e.Scope.Get("gorm:insert_option"); ok {
		extraOption = fmt.Sprint(str)
	}

	if primaryField != nil {
		returningColumn = scope.Quote(e, primaryField.DBName)
	}

	lastInsertIDReturningSuffix :=
		e.Dialect.LastInsertIDReturningSuffix(tableName, returningColumn)

	if len(columns) == 0 {
		sql := fmt.Sprintf(
			"INSERT INTO %v DEFAULT VALUES%v%v",
			tableName,
			util.AddExtraSpaceIfExist(extraOption),
			util.AddExtraSpaceIfExist(lastInsertIDReturningSuffix),
		)
		e.Scope.SQL = strings.Replace(sql, "$$", "?", -1)
	} else {
		sql := fmt.Sprintf(
			"INSERT INTO %v (%v) VALUES (%v)%v%v",
			scope.QuotedTableName(e, e.Scope.Value),
			strings.Join(columns, ","),
			strings.Join(placeholders, ","),
			util.AddExtraSpaceIfExist(extraOption),
			util.AddExtraSpaceIfExist(lastInsertIDReturningSuffix),
		)
		e.Scope.SQL = strings.Replace(sql, "$$", "?", -1)
	}

	// execute create sql
	//if lastInsertIDReturningSuffix == "" || primaryField == nil {
	//result, err := e.SQLDB.Exec(e.Scope.SQL, e.Scope.SQLVars...)
	//if err != nil {
	//return err
	//}
	//// set rows affected count
	//e.RowsAffected, _ = result.RowsAffected()

	//// set primary value to primary field
	//if primaryField != nil && primaryField.IsBlank {
	//primaryValue, err := result.LastInsertId()
	//if err != nil {
	//return err
	//}
	//_ = primaryField.Set(primaryValue)
	//}
	//} else {
	//if primaryField.Field.CanAddr() {
	//err := e.SQLDB.QueryRow(
	//e.Scope.SQL,
	//e.Scope.SQLVars...,
	//).Scan(primaryField.Field.Addr().Interface())
	//if err != nil {
	//return err
	//}
	//primaryField.IsBlank = false
	//e.RowsAffected = 1
	//} else {
	//return errmsg.ErrUnaddressable
	//}
	//}
	return nil
}

//AfterCreate hook executed after a new record has been created.
func AfterCreate(b *Book, e *engine.Engine) error {
	ac, ok := b.Create.Get(model.HookAfterCreate)
	if ok {
		err := ac.Exec(b, e)
		if err != nil {
			return err
		}
	}
	as, ok := b.Create.Get(model.HookAfterSave)
	if ok {
		err := as.Exec(b, e)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package memfs

import (
	"errors"
	"fmt"
	"os"
	filepath "path"
	"sort"
	"sync"
	"time"

	"github.com/jbuchbinder/vfs"
)

var (
	// ErrReadOnly is returned if the file is read-only and write operations are disabled.
	ErrReadOnly = errors.New("File is read-only")
	// ErrWriteOnly is returned if the file is write-only and read operations are disabled.
	ErrWriteOnly = errors.New("File is write-only")
	// ErrIsDirectory is returned if the file under operation is not a regular file but a directory.
	ErrIsDirectory = errors.New("Is directory")
)

// PathSeparator used to separate path segments
const PathSeparator = "/"

// MemFS is a in-memory filesystem
type MemFS struct {
	root *fileInfo
	wd   *fileInfo
	lock *sync.RWMutex
}

// Create a new MemFS filesystem which entirely resides in memory
func Create() *MemFS {
	root := &fileInfo{
		name: "/",
		dir:  true,
	}
	return &MemFS{
		root: root,
		wd:   root,
		lock: &sync.RWMutex{},
	}
}

type fileInfo struct {
	name    string
	dir     bool
	mode    os.FileMode
	parent  *fileInfo
	size    int64
	modTime time.Time
	fs      vfs.Filesystem
	childs  map[string]*fileInfo
	buf     *[]byte
	mutex   *sync.RWMutex
}

func (fi fileInfo) Sys() interface{} {
	return fi.fs
}

func (fi fileInfo) Size() int64 {
	if fi.dir {
		return 0
	}
	fi.mutex.RLock()
	l := len(*(fi.buf))
	fi.mutex.RUnlock()
	return int64(l)
}

func (fi fileInfo) IsDir() bool {
	return fi.dir
}

// ModTime returns the modification time.
// Modification time is updated on:
// 	- Creation
// 	- Rename
// 	- Open
func (fi fileInfo) ModTime() time.Time {
	return fi.modTime
}

func (fi fileInfo) Mode() os.FileMode {
	return fi.mode
}

func (fi fileInfo) Name() string {
	return fi.name
}

func (fi fileInfo) AbsPath() string {
	if fi.parent != nil {
		return filepath.Join(fi.parent.AbsPath(), fi.name)
	}
	return "/"
}

// PathSeparator returns the path separator
func (fs *MemFS) PathSeparator() uint8 {
	return '/'
}

// Mkdir creates a new directory with given permissions
func (fs *MemFS) Mkdir(name string, perm os.FileMode) error {
	fs.lock.Lock()
	defer fs.lock.Unlock()
	name = filepath.Clean(name)
	base := filepath.Base(name)
	parent, fi, err := fs.fileInfo(name)
	if err != nil {
		return &os.PathError{"mkdir", name, err}
	}
	if fi != nil {
		return &os.PathError{"mkdir", name, fmt.Errorf("Directory %q already exists", name)}
	}

	fi = &fileInfo{
		name:    base,
		dir:     true,
		mode:    perm,
		parent:  parent,
		modTime: time.Now(),
		fs:      fs,
	}
	parent.childs[base] = fi
	return nil
}

// byName implements sort.Interface
type byName []os.FileInfo

// Len returns the length of the slice
func (f byName) Len() int { return len(f) }

// Less sorts slice by Name
func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() }

// Swap two elements by index
func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] }

// ReadDir reads the directory named by path and returns a list of sorted directory entries.
func (fs *MemFS) ReadDir(path string) ([]os.FileInfo, error) {
	fs.lock.RLock()
	defer fs.lock.RUnlock()

	path = filepath.Clean(path)
	_, fi, err := fs.fileInfo(path)
	if err != nil {
		return nil, &os.PathError{"readdir", path, err}
	}
	if fi == nil || !fi.dir {
		return nil, &os.PathError{"readdir", path, vfs.ErrNotDirectory}
	}

	fis := make([]os.FileInfo, 0, len(fi.childs))
	for _, e := range fi.childs {
		fis = append(fis, e)
	}
	sort.Sort(byName(fis))
	return fis, nil
}

func (fs *MemFS) fileInfo(path string) (parent *fileInfo, node *fileInfo, err error) {
	path = filepath.Clean(path)
	segments := vfs.SplitPath(path, PathSeparator)

	// Shortcut for working directory and root
	if len(segments) == 1 {
		if segments[0] == "" {
			return nil, fs.root, nil
		} else if segments[0] == "." {
			return fs.wd.parent, fs.wd, nil
		}
	}

	// Determine root to traverse
	parent = fs.root
	if segments[0] == "." {
		parent = fs.wd
	}
	segments = segments[1:]

	// Further directories
	if len(segments) > 1 {
		for _, seg := range segments[:len(segments)-1] {

			if parent.childs == nil {
				return nil, nil, os.ErrNotExist
			}
			if entry, ok := parent.childs[seg]; ok && entry.dir {
				parent = entry
			} else {
				return nil, nil, os.ErrNotExist
			}
		}
	}

	lastSeg := segments[len(segments)-1]
	if parent.childs != nil {
		if node, ok := parent.childs[lastSeg]; ok {
			return parent, node, nil
		}
	} else {
		parent.childs = make(map[string]*fileInfo)
	}

	return parent, nil, nil
}

func hasFlag(flag int, flags int) bool {
	return flags&flag == flag
}

// OpenFile opens a file handle with a specified flag (os.O_RDONLY etc.) and perm (e.g. 0666).
// If success the returned File can be used for I/O. Otherwise an error is returned, which
// is a *os.PathError and can be extracted for further information.
func (fs *MemFS) OpenFile(name string, flag int, perm os.FileMode) (vfs.File, error) {
	fs.lock.Lock()
	defer fs.lock.Unlock()

	name = filepath.Clean(name)
	base := filepath.Base(name)
	fiParent, fiNode, err := fs.fileInfo(name)
	if err != nil {
		return nil, &os.PathError{"open", name, err}
	}

	if hasFlag(os.O_CREATE, flag) {
		if fiNode != nil {

			// If O_TRUNC is set, existing file is overwritten
			if !hasFlag(os.O_TRUNC, flag) {
				return nil, &os.PathError{"open", name, os.ErrExist}
			}
		}
		fiNode = &fileInfo{
			name:    base,
			dir:     false,
			mode:    perm,
			parent:  fiParent,
			modTime: time.Now(),
			fs:      fs,
		}
		fiParent.childs[base] = fiNode
	} else { // find existing
		if fiNode == nil {
			return nil, &os.PathError{"open", name, os.ErrNotExist}
		}
		if fiNode.dir {
			return nil, &os.PathError{"open", name, ErrIsDirectory}
		}
	}
	fiNode.modTime = time.Now()
	return fiNode.file(flag)
}

func (fi *fileInfo) file(flag int) (vfs.File, error) {
	if fi.buf == nil || hasFlag(os.O_TRUNC, flag) {
		buf := make([]byte, 0, MinBufferSize)
		fi.buf = &buf
		fi.mutex = &sync.RWMutex{}
	}
	var f vfs.File = NewMemFile(fi.AbsPath(), fi.mutex, fi.buf)
	if hasFlag(os.O_APPEND, flag) {
		f.Seek(0, os.SEEK_END)
	}
	if hasFlag(os.O_RDWR, flag) {
		return f, nil
	} else if hasFlag(os.O_WRONLY, flag) {
		f = &woFile{f}
	} else {
		f = &roFile{f}
	}

	return f, nil
}

// roFile wraps the given file and disables Write(..) operation.
type roFile struct {
	vfs.File
}

// Write is disabled and returns ErrorReadOnly
func (f *roFile) Write(p []byte) (n int, err error) {
	return 0, ErrReadOnly
}

// woFile wraps the given file and disables Read(..) operation.
type woFile struct {
	vfs.File
}

// Read is disabled and returns ErrorWroteOnly
func (f *woFile) Read(p []byte) (n int, err error) {
	return 0, ErrWriteOnly
}

// Remove removes the named file or directory.
// If there is an error, it will be of type *PathError.
func (fs *MemFS) Remove(name string) error {
	fs.lock.Lock()
	defer fs.lock.Unlock()

	name = filepath.Clean(name)
	fiParent, fiNode, err := fs.fileInfo(name)
	if err != nil {
		return &os.PathError{"remove", name, err}
	}
	if fiNode == nil {
		return &os.PathError{"remove", name, os.ErrNotExist}
	}

	delete(fiParent.childs, fiNode.name)
	return nil
}

// Rename renames (moves) a file.
// Handles to the oldpath persist but might return oldpath if Name() is called.
func (fs *MemFS) Rename(oldpath, newpath string) error {
	fs.lock.Lock()
	defer fs.lock.Unlock()

	// OldPath
	oldpath = filepath.Clean(oldpath)
	fiOldParent, fiOld, err := fs.fileInfo(oldpath)
	if err != nil {
		return &os.PathError{"rename", oldpath, err}
	}
	if fiOld == nil {
		return &os.PathError{"rename", oldpath, os.ErrNotExist}
	}

	newpath = filepath.Clean(newpath)
	fiNewParent, fiNew, err := fs.fileInfo(newpath)
	if err != nil {
		return &os.PathError{"rename", newpath, err}
	}

	if fiNew != nil {
		return &os.PathError{"rename", newpath, os.ErrExist}
	}

	newBase := filepath.Base(newpath)

	// Relink
	delete(fiOldParent.childs, fiOld.name)
	fiOld.parent = fiNewParent
	fiOld.name = newBase
	fiOld.modTime = time.Now()
	fiNewParent.childs[fiOld.name] = fiOld
	return nil
}

// Stat returns the FileInfo structure describing the named file.
// If there is an error, it will be of type *PathError.
func (fs *MemFS) Stat(name string) (os.FileInfo, error) {
	fs.lock.RLock()
	defer fs.lock.RUnlock()

	name = filepath.Clean(name)
	// dir, base := filepath.Split(name)
	_, fi, err := fs.fileInfo(name)
	if err != nil {
		return nil, &os.PathError{"stat", name, err}
	}
	if fi == nil {
		return nil, &os.PathError{"stat", name, os.ErrNotExist}
	}
	return fi, nil
}

// Lstat returns a FileInfo describing the named file.
// MemFS does not support symbolic links.
// Alias for fs.Stat(name)
func (fs *MemFS) Lstat(name string) (os.FileInfo, error) {
	return fs.Stat(name)
}
<commit_msg>Add missing RemoveAll() to memfs -- temporary implementation.<commit_after>package memfs

import (
	"errors"
	"fmt"
	"os"
	filepath "path"
	"sort"
	"sync"
	"time"

	"github.com/jbuchbinder/vfs"
)

var (
	// ErrReadOnly is returned if the file is read-only and write operations are disabled.
	ErrReadOnly = errors.New("File is read-only")
	// ErrWriteOnly is returned if the file is write-only and read operations are disabled.
	ErrWriteOnly = errors.New("File is write-only")
	// ErrIsDirectory is returned if the file under operation is not a regular file but a directory.
	ErrIsDirectory = errors.New("Is directory")
)

// PathSeparator used to separate path segments
const PathSeparator = "/"

// MemFS is a in-memory filesystem
type MemFS struct {
	root *fileInfo
	wd   *fileInfo
	lock *sync.RWMutex
}

// Create a new MemFS filesystem which entirely resides in memory
func Create() *MemFS {
	root := &fileInfo{
		name: "/",
		dir:  true,
	}
	return &MemFS{
		root: root,
		wd:   root,
		lock: &sync.RWMutex{},
	}
}

type fileInfo struct {
	name    string
	dir     bool
	mode    os.FileMode
	parent  *fileInfo
	size    int64
	modTime time.Time
	fs      vfs.Filesystem
	childs  map[string]*fileInfo
	buf     *[]byte
	mutex   *sync.RWMutex
}

func (fi fileInfo) Sys() interface{} {
	return fi.fs
}

func (fi fileInfo) Size() int64 {
	if fi.dir {
		return 0
	}
	fi.mutex.RLock()
	l := len(*(fi.buf))
	fi.mutex.RUnlock()
	return int64(l)
}

func (fi fileInfo) IsDir() bool {
	return fi.dir
}

// ModTime returns the modification time.
// Modification time is updated on:
// 	- Creation
// 	- Rename
// 	- Open
func (fi fileInfo) ModTime() time.Time {
	return fi.modTime
}

func (fi fileInfo) Mode() os.FileMode {
	return fi.mode
}

func (fi fileInfo) Name() string {
	return fi.name
}

func (fi fileInfo) AbsPath() string {
	if fi.parent != nil {
		return filepath.Join(fi.parent.AbsPath(), fi.name)
	}
	return "/"
}

// PathSeparator returns the path separator
func (fs *MemFS) PathSeparator() uint8 {
	return '/'
}

// Mkdir creates a new directory with given permissions
func (fs *MemFS) Mkdir(name string, perm os.FileMode) error {
	fs.lock.Lock()
	defer fs.lock.Unlock()
	name = filepath.Clean(name)
	base := filepath.Base(name)
	parent, fi, err := fs.fileInfo(name)
	if err != nil {
		return &os.PathError{"mkdir", name, err}
	}
	if fi != nil {
		return &os.PathError{"mkdir", name, fmt.Errorf("Directory %q already exists", name)}
	}

	fi = &fileInfo{
		name:    base,
		dir:     true,
		mode:    perm,
		parent:  parent,
		modTime: time.Now(),
		fs:      fs,
	}
	parent.childs[base] = fi
	return nil
}

// byName implements sort.Interface
type byName []os.FileInfo

// Len returns the length of the slice
func (f byName) Len() int { return len(f) }

// Less sorts slice by Name
func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() }

// Swap two elements by index
func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] }

// ReadDir reads the directory named by path and returns a list of sorted directory entries.
func (fs *MemFS) ReadDir(path string) ([]os.FileInfo, error) {
	fs.lock.RLock()
	defer fs.lock.RUnlock()

	path = filepath.Clean(path)
	_, fi, err := fs.fileInfo(path)
	if err != nil {
		return nil, &os.PathError{"readdir", path, err}
	}
	if fi == nil || !fi.dir {
		return nil, &os.PathError{"readdir", path, vfs.ErrNotDirectory}
	}

	fis := make([]os.FileInfo, 0, len(fi.childs))
	for _, e := range fi.childs {
		fis = append(fis, e)
	}
	sort.Sort(byName(fis))
	return fis, nil
}

func (fs *MemFS) fileInfo(path string) (parent *fileInfo, node *fileInfo, err error) {
	path = filepath.Clean(path)
	segments := vfs.SplitPath(path, PathSeparator)

	// Shortcut for working directory and root
	if len(segments) == 1 {
		if segments[0] == "" {
			return nil, fs.root, nil
		} else if segments[0] == "." {
			return fs.wd.parent, fs.wd, nil
		}
	}

	// Determine root to traverse
	parent = fs.root
	if segments[0] == "." {
		parent = fs.wd
	}
	segments = segments[1:]

	// Further directories
	if len(segments) > 1 {
		for _, seg := range segments[:len(segments)-1] {

			if parent.childs == nil {
				return nil, nil, os.ErrNotExist
			}
			if entry, ok := parent.childs[seg]; ok && entry.dir {
				parent = entry
			} else {
				return nil, nil, os.ErrNotExist
			}
		}
	}

	lastSeg := segments[len(segments)-1]
	if parent.childs != nil {
		if node, ok := parent.childs[lastSeg]; ok {
			return parent, node, nil
		}
	} else {
		parent.childs = make(map[string]*fileInfo)
	}

	return parent, nil, nil
}

func hasFlag(flag int, flags int) bool {
	return flags&flag == flag
}

// OpenFile opens a file handle with a specified flag (os.O_RDONLY etc.) and perm (e.g. 0666).
// If success the returned File can be used for I/O. Otherwise an error is returned, which
// is a *os.PathError and can be extracted for further information.
func (fs *MemFS) OpenFile(name string, flag int, perm os.FileMode) (vfs.File, error) {
	fs.lock.Lock()
	defer fs.lock.Unlock()

	name = filepath.Clean(name)
	base := filepath.Base(name)
	fiParent, fiNode, err := fs.fileInfo(name)
	if err != nil {
		return nil, &os.PathError{"open", name, err}
	}

	if hasFlag(os.O_CREATE, flag) {
		if fiNode != nil {

			// If O_TRUNC is set, existing file is overwritten
			if !hasFlag(os.O_TRUNC, flag) {
				return nil, &os.PathError{"open", name, os.ErrExist}
			}
		}
		fiNode = &fileInfo{
			name:    base,
			dir:     false,
			mode:    perm,
			parent:  fiParent,
			modTime: time.Now(),
			fs:      fs,
		}
		fiParent.childs[base] = fiNode
	} else { // find existing
		if fiNode == nil {
			return nil, &os.PathError{"open", name, os.ErrNotExist}
		}
		if fiNode.dir {
			return nil, &os.PathError{"open", name, ErrIsDirectory}
		}
	}
	fiNode.modTime = time.Now()
	return fiNode.file(flag)
}

func (fi *fileInfo) file(flag int) (vfs.File, error) {
	if fi.buf == nil || hasFlag(os.O_TRUNC, flag) {
		buf := make([]byte, 0, MinBufferSize)
		fi.buf = &buf
		fi.mutex = &sync.RWMutex{}
	}
	var f vfs.File = NewMemFile(fi.AbsPath(), fi.mutex, fi.buf)
	if hasFlag(os.O_APPEND, flag) {
		f.Seek(0, os.SEEK_END)
	}
	if hasFlag(os.O_RDWR, flag) {
		return f, nil
	} else if hasFlag(os.O_WRONLY, flag) {
		f = &woFile{f}
	} else {
		f = &roFile{f}
	}

	return f, nil
}

// roFile wraps the given file and disables Write(..) operation.
type roFile struct {
	vfs.File
}

// Write is disabled and returns ErrorReadOnly
func (f *roFile) Write(p []byte) (n int, err error) {
	return 0, ErrReadOnly
}

// woFile wraps the given file and disables Read(..) operation.
type woFile struct {
	vfs.File
}

// Read is disabled and returns ErrorWroteOnly
func (f *woFile) Read(p []byte) (n int, err error) {
	return 0, ErrWriteOnly
}

// Remove removes the named file or directory.
// If there is an error, it will be of type *PathError.
func (fs *MemFS) Remove(name string) error {
	fs.lock.Lock()
	defer fs.lock.Unlock()

	name = filepath.Clean(name)
	fiParent, fiNode, err := fs.fileInfo(name)
	if err != nil {
		return &os.PathError{"remove", name, err}
	}
	if fiNode == nil {
		return &os.PathError{"remove", name, os.ErrNotExist}
	}

	delete(fiParent.childs, fiNode.name)
	return nil
}

// RemoveAll removes the named file or directory and all descendents.
// If there is an error, it will be of type *PathError.
func (fs *MemFS) RemoveAll(name string) error {
	// TODO: FIXME: Placeholder leaving orphans
	return fs.Remove(name)
}

// Rename renames (moves) a file.
// Handles to the oldpath persist but might return oldpath if Name() is called.
func (fs *MemFS) Rename(oldpath, newpath string) error {
	fs.lock.Lock()
	defer fs.lock.Unlock()

	// OldPath
	oldpath = filepath.Clean(oldpath)
	fiOldParent, fiOld, err := fs.fileInfo(oldpath)
	if err != nil {
		return &os.PathError{"rename", oldpath, err}
	}
	if fiOld == nil {
		return &os.PathError{"rename", oldpath, os.ErrNotExist}
	}

	newpath = filepath.Clean(newpath)
	fiNewParent, fiNew, err := fs.fileInfo(newpath)
	if err != nil {
		return &os.PathError{"rename", newpath, err}
	}

	if fiNew != nil {
		return &os.PathError{"rename", newpath, os.ErrExist}
	}

	newBase := filepath.Base(newpath)

	// Relink
	delete(fiOldParent.childs, fiOld.name)
	fiOld.parent = fiNewParent
	fiOld.name = newBase
	fiOld.modTime = time.Now()
	fiNewParent.childs[fiOld.name] = fiOld
	return nil
}

// Stat returns the FileInfo structure describing the named file.
// If there is an error, it will be of type *PathError.
func (fs *MemFS) Stat(name string) (os.FileInfo, error) {
	fs.lock.RLock()
	defer fs.lock.RUnlock()

	name = filepath.Clean(name)
	// dir, base := filepath.Split(name)
	_, fi, err := fs.fileInfo(name)
	if err != nil {
		return nil, &os.PathError{"stat", name, err}
	}
	if fi == nil {
		return nil, &os.PathError{"stat", name, os.ErrNotExist}
	}
	return fi, nil
}

// Lstat returns a FileInfo describing the named file.
// MemFS does not support symbolic links.
// Alias for fs.Stat(name)
func (fs *MemFS) Lstat(name string) (os.FileInfo, error) {
	return fs.Stat(name)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
)

func Generate(ch chan<- int) {
	for i := 2; ; i++ {
		time.Sleep(10 * time.Millisecond)
		ch <- i
	}
}

func Filter(ch <-chan int, out chan<- int, prime int) {
	for {
		i := <-ch
		if i%prime != 0 {
			out <- i
		}
	}
}

func main() {
	ch := make(chan int)
	go Generate(ch)
	for i := 0; i < 20; i++ {
		prime := <-ch
		fmt.Println(prime)
		out := make(chan int)
		go Filter(ch, out, prime)
		ch = out
	}
}
<commit_msg>Changed primesieve example<commit_after>package main

import (
	"fmt"
	"time"
)

func Generate(ch chan<- int) {
	for i := 2; ; i++ {
		time.Sleep(10 * time.Millisecond)
		ch <- i
	}
}

func Filter(ch <-chan int, out chan<- int, prime int) {
	for {
		i := <-ch
		if i%prime != 0 {
			out <- i
		}
	}
}

func main() {
	ch := make(chan int)
	go Generate(ch)
	for i := 0; i < 10; i++ {
		prime := <-ch
		fmt.Println(prime)
		out := make(chan int)
		go Filter(ch, out, prime)
		ch = out
	}
}
<|endoftext|>


<commit_before>package gotest

import (
	"strings"
	"time"

	"github.com/jstemmer/go-junit-report/v2/gtr"
)

// reportBuilder helps build a test Report from a collection of events.
//
// The reportBuilder keeps track of the active context whenever a test,
// benchmark or build error is created. This is necessary because the test
// parser do not contain any state themselves and simply just emit an event for
// every line that is read. By tracking the active context, any output that is
// appended to the reportBuilder gets attributed to the correct test, benchmark
// or build error.
type reportBuilder struct {
	packages    []gtr.Package
	tests       map[int]gtr.Test
	benchmarks  map[int]gtr.Benchmark
	buildErrors map[int]gtr.Error
	runErrors   map[int]gtr.Error

	// state
	nextID    int              // next free unused id
	lastID    int              // most recently created id
	output    []string         // output that does not belong to any test
	coverage  float64          // coverage percentage
	parentIDs map[int]struct{} // set of test id's that contain subtests

	// options
	packageName   string
	subtestMode   SubtestMode
	timestampFunc func() time.Time
}

// newReportBuilder creates a new reportBuilder.
func newReportBuilder() *reportBuilder {
	return &reportBuilder{
		tests:         make(map[int]gtr.Test),
		benchmarks:    make(map[int]gtr.Benchmark),
		buildErrors:   make(map[int]gtr.Error),
		runErrors:     make(map[int]gtr.Error),
		nextID:        1,
		parentIDs:     make(map[int]struct{}),
		timestampFunc: time.Now,
	}
}

// newID returns a new unique id and sets the active context this id.
func (b *reportBuilder) newID() int {
	id := b.nextID
	b.lastID = id
	b.nextID += 1
	return id
}

// flush creates a new package in this report containing any tests or
// benchmarks we've collected so far. This is necessary when a test or
// benchmark did not end with a summary.
func (b *reportBuilder) flush() {
	if len(b.tests) > 0 || len(b.benchmarks) > 0 {
		b.CreatePackage(b.packageName, "", 0, "")
	}
}

// Build returns the new Report containing all the tests, benchmarks and output
// created so far.
func (b *reportBuilder) Build() gtr.Report {
	b.flush()
	return gtr.Report{Packages: b.packages}
}

// CreateTest adds a test with the given name to the report, and marks it as
// active.
func (b *reportBuilder) CreateTest(name string) {
	if parentID, ok := b.findTestParentID(name); ok {
		b.parentIDs[parentID] = struct{}{}
	}
	id := b.newID()
	b.tests[id] = gtr.Test{ID: id, Name: name}
}

// PauseTest marks the active context as no longer active. Any results or
// output added to the report after calling PauseTest will no longer be assumed
// to belong to this test.
func (b *reportBuilder) PauseTest(name string) {
	b.lastID = 0
}

// ContinueTest finds the test with the given name and marks it as active. If
// more than one test exist with this name, the most recently created test will
// be used.
func (b *reportBuilder) ContinueTest(name string) {
	b.lastID, _ = b.findTest(name)
}

// EndTest finds the test with the given name, sets the result, duration and
// level. If more than one test exists with this name, the most recently
// created test will be used. If no test exists with this name, a new test is
// created.
func (b *reportBuilder) EndTest(name, result string, duration time.Duration, level int) {
	id, ok := b.findTest(name)
	if !ok {
		// test did not exist, create one
		// TODO: Likely reason is that the user ran go test without the -v
		// flag, should we report this somewhere?
		b.CreateTest(name)
		id = b.lastID
	}

	t := b.tests[id]
	t.Result = parseResult(result)
	t.Duration = duration
	t.Level = level
	b.tests[id] = t
	b.lastID = 0
}

// End marks the active context as no longer active.
func (b *reportBuilder) End() {
	b.lastID = 0
}

// CreateBenchmark adds a benchmark with the given name to the report, and
// marks it as active. If more than one benchmark exists with this name, the
// most recently created benchmark will be updated. If no benchmark exists with
// this name, a new benchmark is created.
func (b *reportBuilder) CreateBenchmark(name string) {
	id := b.newID()
	b.benchmarks[id] = gtr.Benchmark{ID: id, Name: name}
}

// BenchmarkResult updates an existing or adds a new benchmark with the given
// results and marks it as active. If an existing benchmark with this name
// exists but without result, then that one is updated. Otherwise a new one is
// added to the report.
func (b *reportBuilder) BenchmarkResult(name string, iterations int64, nsPerOp, mbPerSec float64, bytesPerOp, allocsPerOp int64) {
	id, ok := b.findBenchmark(name)
	if !ok || b.benchmarks[id].Result != gtr.Unknown {
		b.CreateBenchmark(name)
		id = b.lastID
	}

	b.benchmarks[id] = gtr.Benchmark{
		ID:          id,
		Name:        name,
		Result:      gtr.Pass,
		Iterations:  iterations,
		NsPerOp:     nsPerOp,
		MBPerSec:    mbPerSec,
		BytesPerOp:  bytesPerOp,
		AllocsPerOp: allocsPerOp,
	}
}

// EndBenchmark finds the benchmark with the given name and sets the result. If
// more than one benchmark exists with this name, the most recently created
// benchmark will be used. If no benchmark exists with this name, a new
// benchmark is created.
func (b *reportBuilder) EndBenchmark(name, result string) {
	id, ok := b.findBenchmark(name)
	if !ok {
		b.CreateBenchmark(name)
		id = b.lastID
	}

	bm := b.benchmarks[id]
	bm.Result = parseResult(result)
	b.benchmarks[id] = bm
	b.lastID = 0
}

// CreateBuildError creates a new build error and marks it as active.
func (b *reportBuilder) CreateBuildError(packageName string) {
	id := b.newID()
	b.buildErrors[id] = gtr.Error{ID: id, Name: packageName}
}

// CreatePackage adds a new package with the given name to the Report. This
// package contains all the build errors, output, tests and benchmarks created
// so far. Afterwards all state is reset.
func (b *reportBuilder) CreatePackage(name, result string, duration time.Duration, data string) {
	pkg := gtr.Package{
		Name:     name,
		Duration: duration,
	}

	if b.timestampFunc != nil {
		pkg.Timestamp = b.timestampFunc()
	}

	// Build errors are treated somewhat differently. Rather than having a
	// single package with all build errors collected so far, we only care
	// about the build errors for this particular package.
	for id, buildErr := range b.buildErrors {
		if buildErr.Name == name {
			if len(b.tests) > 0 || len(b.benchmarks) > 0 {
				panic("unexpected tests and/or benchmarks found in build error package")
			}
			buildErr.ID = id
			buildErr.Duration = duration
			buildErr.Cause = data
			pkg.BuildError = buildErr
			b.packages = append(b.packages, pkg)

			delete(b.buildErrors, id)
			// TODO: reset state
			// TODO: buildErrors shouldn't reset/use nextID/lastID, they're more like a global cache
			return
		}
	}

	// If we've collected output, but there were no tests or benchmarks then
	// either there were no tests, or there was some other non-build error.
	if len(b.output) > 0 && len(b.tests) == 0 && len(b.benchmarks) == 0 {
		if parseResult(result) == gtr.Fail {
			pkg.RunError = gtr.Error{
				Name:   name,
				Output: b.output,
			}
		}
		b.packages = append(b.packages, pkg)

		// TODO: reset state
		b.output = nil
		return
	}

	// If the summary result says we failed, but there were no failing tests
	// then something else must have failed.
	if parseResult(result) == gtr.Fail && (len(b.tests) > 0 || len(b.benchmarks) > 0) && !b.containsFailingTest() {
		pkg.RunError = gtr.Error{
			Name:   name,
			Output: b.output,
		}
		b.output = nil
	}

	// Collect tests and benchmarks for this package, maintaining insertion order.
	var tests []gtr.Test
	var benchmarks []gtr.Benchmark
	for id := 1; id < b.nextID; id++ {
		if t, ok := b.tests[id]; ok {
			if b.isParent(id) {
				if b.subtestMode == IgnoreParentResults {
					t.Result = gtr.Pass
				} else if b.subtestMode == ExcludeParents {
					continue
				}
			}
			tests = append(tests, t)
			continue
		}

		if bm, ok := b.benchmarks[id]; ok {
			benchmarks = append(benchmarks, bm)
			continue
		}
	}

	pkg.Coverage = b.coverage
	pkg.Output = b.output
	pkg.Tests = tests
	pkg.Benchmarks = groupBenchmarksByName(benchmarks)
	b.packages = append(b.packages, pkg)

	// reset state, except for nextID to ensure all id's are unique.
	b.lastID = 0
	b.output = nil
	b.coverage = 0
	b.tests = make(map[int]gtr.Test)
	b.benchmarks = make(map[int]gtr.Benchmark)
	b.parentIDs = make(map[int]struct{})
}

// Coverage sets the code coverage percentage.
func (b *reportBuilder) Coverage(pct float64, packages []string) {
	b.coverage = pct
}

// AppendOutput appends the given line to the currently active context. If no
// active context exists, the output is assumed to belong to the package.
func (b *reportBuilder) AppendOutput(line string) {
	if b.lastID <= 0 {
		b.output = append(b.output, line)
		return
	}

	if t, ok := b.tests[b.lastID]; ok {
		t.Output = append(t.Output, line)
		b.tests[b.lastID] = t
	} else if bm, ok := b.benchmarks[b.lastID]; ok {
		bm.Output = append(bm.Output, line)
		b.benchmarks[b.lastID] = bm
	} else if be, ok := b.buildErrors[b.lastID]; ok {
		be.Output = append(be.Output, line)
		b.buildErrors[b.lastID] = be
	} else {
		b.output = append(b.output, line)
	}
}

// findTest returns the id of the most recently created test with the given
// name if it exists.
func (b *reportBuilder) findTest(name string) (int, bool) {
	// check if this test was lastID
	if t, ok := b.tests[b.lastID]; ok && t.Name == name {
		return b.lastID, true
	}
	for i := b.nextID; i >= 0; i-- {
		if test, ok := b.tests[i]; ok && test.Name == name {
			return i, true
		}
	}
	return 0, false
}

func (b *reportBuilder) findTestParentID(name string) (int, bool) {
	parent := dropLastSegment(name)
	for parent != "" {
		if id, ok := b.findTest(parent); ok {
			return id, true
		}
		parent = dropLastSegment(parent)
	}
	return 0, false
}

func (b *reportBuilder) isParent(id int) bool {
	_, ok := b.parentIDs[id]
	return ok
}

func dropLastSegment(name string) string {
	if idx := strings.LastIndexByte(name, '/'); idx >= 0 {
		return name[:idx]
	}
	return ""
}

// findBenchmark returns the id of the most recently created benchmark with the
// given name if it exists.
func (b *reportBuilder) findBenchmark(name string) (int, bool) {
	// check if this benchmark was lastID
	if bm, ok := b.benchmarks[b.lastID]; ok && bm.Name == name {
		return b.lastID, true
	}
	for id := len(b.benchmarks); id >= 0; id-- {
		if b.benchmarks[id].Name == name {
			return id, true
		}
	}
	return 0, false
}

// containsFailingTest return true if the current list of tests contains at
// least one failing test or an unknown result.
func (b *reportBuilder) containsFailingTest() bool {
	for _, test := range b.tests {
		if test.Result == gtr.Fail || test.Result == gtr.Unknown {
			return true
		}
	}
	return false
}

// parseResult returns a Result for the given string r.
func parseResult(r string) gtr.Result {
	switch r {
	case "PASS":
		return gtr.Pass
	case "FAIL":
		return gtr.Fail
	case "SKIP":
		return gtr.Skip
	case "BENCH":
		return gtr.Pass
	default:
		return gtr.Unknown
	}
}

func groupBenchmarksByName(benchmarks []gtr.Benchmark) []gtr.Benchmark {
	if len(benchmarks) == 0 {
		return nil
	}

	var grouped []gtr.Benchmark
	byName := make(map[string][]gtr.Benchmark)
	for _, bm := range benchmarks {
		if _, ok := byName[bm.Name]; !ok {
			grouped = append(grouped, gtr.Benchmark{ID: bm.ID, Name: bm.Name})
		}
		byName[bm.Name] = append(byName[bm.Name], bm)
	}

	for i, group := range grouped {
		count := 0
		for _, bm := range byName[group.Name] {
			if bm.Result != gtr.Pass {
				continue
			}
			group.Iterations += bm.Iterations
			group.NsPerOp += bm.NsPerOp
			group.MBPerSec += bm.MBPerSec
			group.BytesPerOp += bm.BytesPerOp
			group.AllocsPerOp += bm.AllocsPerOp
			count++
		}

		group.Result = groupResults(byName[group.Name])
		if count > 0 {
			group.NsPerOp /= float64(count)
			group.MBPerSec /= float64(count)
			group.BytesPerOp /= int64(count)
			group.AllocsPerOp /= int64(count)
		}
		grouped[i] = group
	}
	return grouped
}

func groupResults(benchmarks []gtr.Benchmark) gtr.Result {
	var result gtr.Result
	for _, bm := range benchmarks {
		if bm.Result == gtr.Fail {
			return gtr.Fail
		}
		if result != gtr.Pass {
			result = bm.Result
		}
	}
	return result
}
<commit_msg>parser/gotest: Skip ID 0 when searching for existing tests or benchmarks<commit_after>package gotest

import (
	"strings"
	"time"

	"github.com/jstemmer/go-junit-report/v2/gtr"
)

// reportBuilder helps build a test Report from a collection of events.
//
// The reportBuilder keeps track of the active context whenever a test,
// benchmark or build error is created. This is necessary because the test
// parser do not contain any state themselves and simply just emit an event for
// every line that is read. By tracking the active context, any output that is
// appended to the reportBuilder gets attributed to the correct test, benchmark
// or build error.
type reportBuilder struct {
	packages    []gtr.Package
	tests       map[int]gtr.Test
	benchmarks  map[int]gtr.Benchmark
	buildErrors map[int]gtr.Error
	runErrors   map[int]gtr.Error

	// state
	nextID    int              // next free unused id
	lastID    int              // most recently created id
	output    []string         // output that does not belong to any test
	coverage  float64          // coverage percentage
	parentIDs map[int]struct{} // set of test id's that contain subtests

	// options
	packageName   string
	subtestMode   SubtestMode
	timestampFunc func() time.Time
}

// newReportBuilder creates a new reportBuilder.
func newReportBuilder() *reportBuilder {
	return &reportBuilder{
		tests:         make(map[int]gtr.Test),
		benchmarks:    make(map[int]gtr.Benchmark),
		buildErrors:   make(map[int]gtr.Error),
		runErrors:     make(map[int]gtr.Error),
		nextID:        1,
		parentIDs:     make(map[int]struct{}),
		timestampFunc: time.Now,
	}
}

// newID returns a new unique id and sets the active context this id.
func (b *reportBuilder) newID() int {
	id := b.nextID
	b.lastID = id
	b.nextID += 1
	return id
}

// flush creates a new package in this report containing any tests or
// benchmarks we've collected so far. This is necessary when a test or
// benchmark did not end with a summary.
func (b *reportBuilder) flush() {
	if len(b.tests) > 0 || len(b.benchmarks) > 0 {
		b.CreatePackage(b.packageName, "", 0, "")
	}
}

// Build returns the new Report containing all the tests, benchmarks and output
// created so far.
func (b *reportBuilder) Build() gtr.Report {
	b.flush()
	return gtr.Report{Packages: b.packages}
}

// CreateTest adds a test with the given name to the report, and marks it as
// active.
func (b *reportBuilder) CreateTest(name string) {
	if parentID, ok := b.findTestParentID(name); ok {
		b.parentIDs[parentID] = struct{}{}
	}
	id := b.newID()
	b.tests[id] = gtr.Test{ID: id, Name: name}
}

// PauseTest marks the active context as no longer active. Any results or
// output added to the report after calling PauseTest will no longer be assumed
// to belong to this test.
func (b *reportBuilder) PauseTest(name string) {
	b.lastID = 0
}

// ContinueTest finds the test with the given name and marks it as active. If
// more than one test exist with this name, the most recently created test will
// be used.
func (b *reportBuilder) ContinueTest(name string) {
	b.lastID, _ = b.findTest(name)
}

// EndTest finds the test with the given name, sets the result, duration and
// level. If more than one test exists with this name, the most recently
// created test will be used. If no test exists with this name, a new test is
// created.
func (b *reportBuilder) EndTest(name, result string, duration time.Duration, level int) {
	id, ok := b.findTest(name)
	if !ok {
		// test did not exist, create one
		// TODO: Likely reason is that the user ran go test without the -v
		// flag, should we report this somewhere?
		b.CreateTest(name)
		id = b.lastID
	}

	t := b.tests[id]
	t.Result = parseResult(result)
	t.Duration = duration
	t.Level = level
	b.tests[id] = t
	b.lastID = 0
}

// End marks the active context as no longer active.
func (b *reportBuilder) End() {
	b.lastID = 0
}

// CreateBenchmark adds a benchmark with the given name to the report, and
// marks it as active. If more than one benchmark exists with this name, the
// most recently created benchmark will be updated. If no benchmark exists with
// this name, a new benchmark is created.
func (b *reportBuilder) CreateBenchmark(name string) {
	id := b.newID()
	b.benchmarks[id] = gtr.Benchmark{ID: id, Name: name}
}

// BenchmarkResult updates an existing or adds a new benchmark with the given
// results and marks it as active. If an existing benchmark with this name
// exists but without result, then that one is updated. Otherwise a new one is
// added to the report.
func (b *reportBuilder) BenchmarkResult(name string, iterations int64, nsPerOp, mbPerSec float64, bytesPerOp, allocsPerOp int64) {
	id, ok := b.findBenchmark(name)
	if !ok || b.benchmarks[id].Result != gtr.Unknown {
		b.CreateBenchmark(name)
		id = b.lastID
	}

	b.benchmarks[id] = gtr.Benchmark{
		ID:          id,
		Name:        name,
		Result:      gtr.Pass,
		Iterations:  iterations,
		NsPerOp:     nsPerOp,
		MBPerSec:    mbPerSec,
		BytesPerOp:  bytesPerOp,
		AllocsPerOp: allocsPerOp,
	}
}

// EndBenchmark finds the benchmark with the given name and sets the result. If
// more than one benchmark exists with this name, the most recently created
// benchmark will be used. If no benchmark exists with this name, a new
// benchmark is created.
func (b *reportBuilder) EndBenchmark(name, result string) {
	id, ok := b.findBenchmark(name)
	if !ok {
		b.CreateBenchmark(name)
		id = b.lastID
	}

	bm := b.benchmarks[id]
	bm.Result = parseResult(result)
	b.benchmarks[id] = bm
	b.lastID = 0
}

// CreateBuildError creates a new build error and marks it as active.
func (b *reportBuilder) CreateBuildError(packageName string) {
	id := b.newID()
	b.buildErrors[id] = gtr.Error{ID: id, Name: packageName}
}

// CreatePackage adds a new package with the given name to the Report. This
// package contains all the build errors, output, tests and benchmarks created
// so far. Afterwards all state is reset.
func (b *reportBuilder) CreatePackage(name, result string, duration time.Duration, data string) {
	pkg := gtr.Package{
		Name:     name,
		Duration: duration,
	}

	if b.timestampFunc != nil {
		pkg.Timestamp = b.timestampFunc()
	}

	// Build errors are treated somewhat differently. Rather than having a
	// single package with all build errors collected so far, we only care
	// about the build errors for this particular package.
	for id, buildErr := range b.buildErrors {
		if buildErr.Name == name {
			if len(b.tests) > 0 || len(b.benchmarks) > 0 {
				panic("unexpected tests and/or benchmarks found in build error package")
			}
			buildErr.ID = id
			buildErr.Duration = duration
			buildErr.Cause = data
			pkg.BuildError = buildErr
			b.packages = append(b.packages, pkg)

			delete(b.buildErrors, id)
			// TODO: reset state
			// TODO: buildErrors shouldn't reset/use nextID/lastID, they're more like a global cache
			return
		}
	}

	// If we've collected output, but there were no tests or benchmarks then
	// either there were no tests, or there was some other non-build error.
	if len(b.output) > 0 && len(b.tests) == 0 && len(b.benchmarks) == 0 {
		if parseResult(result) == gtr.Fail {
			pkg.RunError = gtr.Error{
				Name:   name,
				Output: b.output,
			}
		}
		b.packages = append(b.packages, pkg)

		// TODO: reset state
		b.output = nil
		return
	}

	// If the summary result says we failed, but there were no failing tests
	// then something else must have failed.
	if parseResult(result) == gtr.Fail && (len(b.tests) > 0 || len(b.benchmarks) > 0) && !b.containsFailingTest() {
		pkg.RunError = gtr.Error{
			Name:   name,
			Output: b.output,
		}
		b.output = nil
	}

	// Collect tests and benchmarks for this package, maintaining insertion order.
	var tests []gtr.Test
	var benchmarks []gtr.Benchmark
	for id := 1; id < b.nextID; id++ {
		if t, ok := b.tests[id]; ok {
			if b.isParent(id) {
				if b.subtestMode == IgnoreParentResults {
					t.Result = gtr.Pass
				} else if b.subtestMode == ExcludeParents {
					continue
				}
			}
			tests = append(tests, t)
			continue
		}

		if bm, ok := b.benchmarks[id]; ok {
			benchmarks = append(benchmarks, bm)
			continue
		}
	}

	pkg.Coverage = b.coverage
	pkg.Output = b.output
	pkg.Tests = tests
	pkg.Benchmarks = groupBenchmarksByName(benchmarks)
	b.packages = append(b.packages, pkg)

	// reset state, except for nextID to ensure all id's are unique.
	b.lastID = 0
	b.output = nil
	b.coverage = 0
	b.tests = make(map[int]gtr.Test)
	b.benchmarks = make(map[int]gtr.Benchmark)
	b.parentIDs = make(map[int]struct{})
}

// Coverage sets the code coverage percentage.
func (b *reportBuilder) Coverage(pct float64, packages []string) {
	b.coverage = pct
}

// AppendOutput appends the given line to the currently active context. If no
// active context exists, the output is assumed to belong to the package.
func (b *reportBuilder) AppendOutput(line string) {
	if b.lastID <= 0 {
		b.output = append(b.output, line)
		return
	}

	if t, ok := b.tests[b.lastID]; ok {
		t.Output = append(t.Output, line)
		b.tests[b.lastID] = t
	} else if bm, ok := b.benchmarks[b.lastID]; ok {
		bm.Output = append(bm.Output, line)
		b.benchmarks[b.lastID] = bm
	} else if be, ok := b.buildErrors[b.lastID]; ok {
		be.Output = append(be.Output, line)
		b.buildErrors[b.lastID] = be
	} else {
		b.output = append(b.output, line)
	}
}

// findTest returns the id of the most recently created test with the given
// name if it exists.
func (b *reportBuilder) findTest(name string) (int, bool) {
	// check if this test was lastID
	if t, ok := b.tests[b.lastID]; ok && t.Name == name {
		return b.lastID, true
	}
	for i := b.nextID; i > 0; i-- {
		if test, ok := b.tests[i]; ok && test.Name == name {
			return i, true
		}
	}
	return 0, false
}

func (b *reportBuilder) findTestParentID(name string) (int, bool) {
	parent := dropLastSegment(name)
	for parent != "" {
		if id, ok := b.findTest(parent); ok {
			return id, true
		}
		parent = dropLastSegment(parent)
	}
	return 0, false
}

func (b *reportBuilder) isParent(id int) bool {
	_, ok := b.parentIDs[id]
	return ok
}

func dropLastSegment(name string) string {
	if idx := strings.LastIndexByte(name, '/'); idx >= 0 {
		return name[:idx]
	}
	return ""
}

// findBenchmark returns the id of the most recently created benchmark with the
// given name if it exists.
func (b *reportBuilder) findBenchmark(name string) (int, bool) {
	// check if this benchmark was lastID
	if bm, ok := b.benchmarks[b.lastID]; ok && bm.Name == name {
		return b.lastID, true
	}
	for id := len(b.benchmarks); id > 0; id-- {
		if b.benchmarks[id].Name == name {
			return id, true
		}
	}
	return 0, false
}

// containsFailingTest return true if the current list of tests contains at
// least one failing test or an unknown result.
func (b *reportBuilder) containsFailingTest() bool {
	for _, test := range b.tests {
		if test.Result == gtr.Fail || test.Result == gtr.Unknown {
			return true
		}
	}
	return false
}

// parseResult returns a Result for the given string r.
func parseResult(r string) gtr.Result {
	switch r {
	case "PASS":
		return gtr.Pass
	case "FAIL":
		return gtr.Fail
	case "SKIP":
		return gtr.Skip
	case "BENCH":
		return gtr.Pass
	default:
		return gtr.Unknown
	}
}

func groupBenchmarksByName(benchmarks []gtr.Benchmark) []gtr.Benchmark {
	if len(benchmarks) == 0 {
		return nil
	}

	var grouped []gtr.Benchmark
	byName := make(map[string][]gtr.Benchmark)
	for _, bm := range benchmarks {
		if _, ok := byName[bm.Name]; !ok {
			grouped = append(grouped, gtr.Benchmark{ID: bm.ID, Name: bm.Name})
		}
		byName[bm.Name] = append(byName[bm.Name], bm)
	}

	for i, group := range grouped {
		count := 0
		for _, bm := range byName[group.Name] {
			if bm.Result != gtr.Pass {
				continue
			}
			group.Iterations += bm.Iterations
			group.NsPerOp += bm.NsPerOp
			group.MBPerSec += bm.MBPerSec
			group.BytesPerOp += bm.BytesPerOp
			group.AllocsPerOp += bm.AllocsPerOp
			count++
		}

		group.Result = groupResults(byName[group.Name])
		if count > 0 {
			group.NsPerOp /= float64(count)
			group.MBPerSec /= float64(count)
			group.BytesPerOp /= int64(count)
			group.AllocsPerOp /= int64(count)
		}
		grouped[i] = group
	}
	return grouped
}

func groupResults(benchmarks []gtr.Benchmark) gtr.Result {
	var result gtr.Result
	for _, bm := range benchmarks {
		if bm.Result == gtr.Fail {
			return gtr.Fail
		}
		if result != gtr.Pass {
			result = bm.Result
		}
	}
	return result
}
<|endoftext|>