<commit_before>// +build linux freebsd

package zfs

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/docker/pkg/parsers"
	zfs "github.com/mistifyio/go-zfs"
	"github.com/opencontainers/runc/libcontainer/label"
)

type zfsOptions struct {
	fsName    string
	mountPath string
}

func init() {
	graphdriver.Register("zfs", Init)
}

// Logger returns a zfs logger implementation.
type Logger struct{}

// Log wraps log message from ZFS driver with a prefix '[zfs]'.
func (*Logger) Log(cmd []string) {
	logrus.Debugf("[zfs] %s", strings.Join(cmd, " "))
}

// Init returns a new ZFS driver.
// It takes base mount path and a array of options which are represented as key value pairs.
// Each option is in the for key=value. 'zfs.fsname' is expected to be a valid key in the options.
func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
	var err error

	if _, err := exec.LookPath("zfs"); err != nil {
		logrus.Debugf("[zfs] zfs command is not available: %v", err)
		return nil, graphdriver.ErrPrerequisites
	}

	file, err := os.OpenFile("/dev/zfs", os.O_RDWR, 600)
	if err != nil {
		logrus.Debugf("[zfs] cannot open /dev/zfs: %v", err)
		return nil, graphdriver.ErrPrerequisites
	}
	defer file.Close()

	options, err := parseOptions(opt)
	if err != nil {
		return nil, err
	}
	options.mountPath = base

	rootdir := path.Dir(base)

	if options.fsName == "" {
		err = checkRootdirFs(rootdir)
		if err != nil {
			return nil, err
		}
	}

	if options.fsName == "" {
		options.fsName, err = lookupZfsDataset(rootdir)
		if err != nil {
			return nil, err
		}
	}

	zfs.SetLogger(new(Logger))

	filesystems, err := zfs.Filesystems(options.fsName)
	if err != nil {
		return nil, fmt.Errorf("Cannot find root filesystem %s: %v", options.fsName, err)
	}

	filesystemsCache := make(map[string]bool, len(filesystems))
	var rootDataset *zfs.Dataset
	for _, fs := range filesystems {
		if fs.Name == options.fsName {
			rootDataset = fs
		}
		filesystemsCache[fs.Name] = true
	}

	if rootDataset == nil {
		return nil, fmt.Errorf("BUG: zfs get all -t filesystem -rHp '%s' should contain '%s'", options.fsName, options.fsName)
	}

	d := &Driver{
		dataset:          rootDataset,
		options:          options,
		filesystemsCache: filesystemsCache,
		uidMaps:          uidMaps,
		gidMaps:          gidMaps,
	}
	return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil
}

func parseOptions(opt []string) (zfsOptions, error) {
	var options zfsOptions
	options.fsName = ""
	for _, option := range opt {
		key, val, err := parsers.ParseKeyValueOpt(option)
		if err != nil {
			return options, err
		}
		key = strings.ToLower(key)
		switch key {
		case "zfs.fsname":
			options.fsName = val
		default:
			return options, fmt.Errorf("Unknown option %s", key)
		}
	}
	return options, nil
}

func lookupZfsDataset(rootdir string) (string, error) {
	var stat syscall.Stat_t
	if err := syscall.Stat(rootdir, &stat); err != nil {
		return "", fmt.Errorf("Failed to access '%s': %s", rootdir, err)
	}
	wantedDev := stat.Dev

	mounts, err := mount.GetMounts()
	if err != nil {
		return "", err
	}
	for _, m := range mounts {
		if err := syscall.Stat(m.Mountpoint, &stat); err != nil {
			logrus.Debugf("[zfs] failed to stat '%s' while scanning for zfs mount: %v", m.Mountpoint, err)
			continue // may fail on fuse file systems
		}

		if stat.Dev == wantedDev && m.Fstype == "zfs" {
			return m.Source, nil
		}
	}

	return "", fmt.Errorf("Failed to find zfs dataset mounted on '%s' in /proc/mounts", rootdir)
}

// Driver holds information about the driver, such as zfs dataset, options and cache.
type Driver struct {
	dataset          *zfs.Dataset
	options          zfsOptions
	sync.Mutex       // protects filesystem cache against concurrent access
	filesystemsCache map[string]bool
	uidMaps          []idtools.IDMap
	gidMaps          []idtools.IDMap
}

func (d *Driver) String() string {
	return "zfs"
}

// Cleanup is used to implement graphdriver.ProtoDriver. There is no cleanup required for this driver.
func (d *Driver) Cleanup() error {
	return nil
}

// Status returns information about the ZFS filesystem. It returns a two dimensional array of information
// such as pool name, dataset name, disk usage, parent quota and compression used.
// Currently it return 'Zpool', 'Zpool Health', 'Parent Dataset', 'Space Used By Parent',
// 'Space Available', 'Parent Quota' and 'Compression'.
func (d *Driver) Status() [][2]string {
	parts := strings.Split(d.dataset.Name, "/")
	pool, err := zfs.GetZpool(parts[0])

	var poolName, poolHealth string
	if err == nil {
		poolName = pool.Name
		poolHealth = pool.Health
	} else {
		poolName = fmt.Sprintf("error while getting pool information %v", err)
		poolHealth = "not available"
	}

	quota := "no"
	if d.dataset.Quota != 0 {
		quota = strconv.FormatUint(d.dataset.Quota, 10)
	}

	return [][2]string{
		{"Zpool", poolName},
		{"Zpool Health", poolHealth},
		{"Parent Dataset", d.dataset.Name},
		{"Space Used By Parent", strconv.FormatUint(d.dataset.Used, 10)},
		{"Space Available", strconv.FormatUint(d.dataset.Avail, 10)},
		{"Parent Quota", quota},
		{"Compression", d.dataset.Compression},
	}
}

// GetMetadata returns image/container metadata related to graph driver
func (d *Driver) GetMetadata(id string) (map[string]string, error) {
	return nil, nil
}

func (d *Driver) cloneFilesystem(name, parentName string) error {
	snapshotName := fmt.Sprintf("%d", time.Now().Nanosecond())
	parentDataset := zfs.Dataset{Name: parentName}
	snapshot, err := parentDataset.Snapshot(snapshotName /*recursive */, false)
	if err != nil {
		return err
	}

	_, err = snapshot.Clone(name, map[string]string{"mountpoint": "legacy"})
	if err == nil {
		d.Lock()
		d.filesystemsCache[name] = true
		d.Unlock()
	}

	if err != nil {
		snapshot.Destroy(zfs.DestroyDeferDeletion)
		return err
	}
	return snapshot.Destroy(zfs.DestroyDeferDeletion)
}

func (d *Driver) zfsPath(id string) string {
	return d.options.fsName + "/" + id
}

func (d *Driver) mountPath(id string) string {
	return path.Join(d.options.mountPath, "graph", getMountpoint(id))
}

// Create prepares the dataset and filesystem for the ZFS driver for the given id under the parent.
func (d *Driver) Create(id string, parent string, mountLabel string) error {
	err := d.create(id, parent)
	if err == nil {
		return nil
	}
	if zfsError, ok := err.(*zfs.Error); ok {
		if !strings.HasSuffix(zfsError.Stderr, "dataset already exists\n") {
			return err
		}
		// aborted build -> cleanup
	} else {
		return err
	}

	dataset := zfs.Dataset{Name: d.zfsPath(id)}
	if err := dataset.Destroy(zfs.DestroyRecursiveClones); err != nil {
		return err
	}

	// retry
	return d.create(id, parent)
}

func (d *Driver) create(id, parent string) error {
	name := d.zfsPath(id)
	if parent == "" {
		mountoptions := map[string]string{"mountpoint": "legacy"}
		fs, err := zfs.CreateFilesystem(name, mountoptions)
		if err == nil {
			d.Lock()
			d.filesystemsCache[fs.Name] = true
			d.Unlock()
		}
		return err
	}
	return d.cloneFilesystem(name, d.zfsPath(parent))
}

// Remove deletes the dataset, filesystem and the cache for the given id.
func (d *Driver) Remove(id string) error {
	name := d.zfsPath(id)
	dataset := zfs.Dataset{Name: name}
	err := dataset.Destroy(zfs.DestroyRecursive)
	if err == nil {
		d.Lock()
		delete(d.filesystemsCache, name)
		d.Unlock()
	}
	return err
}

// Get returns the mountpoint for the given id after creating the target directories if necessary.
func (d *Driver) Get(id, mountLabel string) (string, error) {
	mountpoint := d.mountPath(id)
	filesystem := d.zfsPath(id)
	options := label.FormatMountLabel("", mountLabel)
	logrus.Debugf(`[zfs] mount("%s", "%s", "%s")`, filesystem, mountpoint, options)

	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
	if err != nil {
		return "", err
	}
	// Create the target directories if they don't exist
	if err := idtools.MkdirAllAs(mountpoint, 0755, rootUID, rootGID); err != nil {
		return "", err
	}

	err = mount.Mount(filesystem, mountpoint, "zfs", options)
	if err != nil {
		return "", fmt.Errorf("error creating zfs mount of %s to %s: %v", filesystem, mountpoint, err)
	}

	return mountpoint, nil
}

// Put removes the existing mountpoint for the given id if it exists.
func (d *Driver) Put(id string) error {
	mountpoint := d.mountPath(id)
	logrus.Debugf(`[zfs] unmount("%s")`, mountpoint)

	if err := mount.Unmount(mountpoint); err != nil {
		return fmt.Errorf("error unmounting to %s: %v", mountpoint, err)
	}
	return nil
}

// Exists checks to see if the cache entry exists for the given id.
func (d *Driver) Exists(id string) bool {
	return d.filesystemsCache[d.zfsPath(id)] == true
}
<commit_msg>Fix ZFS permissions bug with user namespaces<commit_after>// +build linux freebsd

package zfs

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/docker/pkg/parsers"
	zfs "github.com/mistifyio/go-zfs"
	"github.com/opencontainers/runc/libcontainer/label"
)

type zfsOptions struct {
	fsName    string
	mountPath string
}

func init() {
	graphdriver.Register("zfs", Init)
}

// Logger returns a zfs logger implementation.
type Logger struct{}

// Log wraps log message from ZFS driver with a prefix '[zfs]'.
func (*Logger) Log(cmd []string) {
	logrus.Debugf("[zfs] %s", strings.Join(cmd, " "))
}

// Init returns a new ZFS driver.
// It takes base mount path and a array of options which are represented as key value pairs.
// Each option is in the for key=value. 'zfs.fsname' is expected to be a valid key in the options.
func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
	var err error

	if _, err := exec.LookPath("zfs"); err != nil {
		logrus.Debugf("[zfs] zfs command is not available: %v", err)
		return nil, graphdriver.ErrPrerequisites
	}

	file, err := os.OpenFile("/dev/zfs", os.O_RDWR, 600)
	if err != nil {
		logrus.Debugf("[zfs] cannot open /dev/zfs: %v", err)
		return nil, graphdriver.ErrPrerequisites
	}
	defer file.Close()

	options, err := parseOptions(opt)
	if err != nil {
		return nil, err
	}
	options.mountPath = base

	rootdir := path.Dir(base)

	if options.fsName == "" {
		err = checkRootdirFs(rootdir)
		if err != nil {
			return nil, err
		}
	}

	if options.fsName == "" {
		options.fsName, err = lookupZfsDataset(rootdir)
		if err != nil {
			return nil, err
		}
	}

	zfs.SetLogger(new(Logger))

	filesystems, err := zfs.Filesystems(options.fsName)
	if err != nil {
		return nil, fmt.Errorf("Cannot find root filesystem %s: %v", options.fsName, err)
	}

	filesystemsCache := make(map[string]bool, len(filesystems))
	var rootDataset *zfs.Dataset
	for _, fs := range filesystems {
		if fs.Name == options.fsName {
			rootDataset = fs
		}
		filesystemsCache[fs.Name] = true
	}

	if rootDataset == nil {
		return nil, fmt.Errorf("BUG: zfs get all -t filesystem -rHp '%s' should contain '%s'", options.fsName, options.fsName)
	}

	d := &Driver{
		dataset:          rootDataset,
		options:          options,
		filesystemsCache: filesystemsCache,
		uidMaps:          uidMaps,
		gidMaps:          gidMaps,
	}
	return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil
}

func parseOptions(opt []string) (zfsOptions, error) {
	var options zfsOptions
	options.fsName = ""
	for _, option := range opt {
		key, val, err := parsers.ParseKeyValueOpt(option)
		if err != nil {
			return options, err
		}
		key = strings.ToLower(key)
		switch key {
		case "zfs.fsname":
			options.fsName = val
		default:
			return options, fmt.Errorf("Unknown option %s", key)
		}
	}
	return options, nil
}

func lookupZfsDataset(rootdir string) (string, error) {
	var stat syscall.Stat_t
	if err := syscall.Stat(rootdir, &stat); err != nil {
		return "", fmt.Errorf("Failed to access '%s': %s", rootdir, err)
	}
	wantedDev := stat.Dev

	mounts, err := mount.GetMounts()
	if err != nil {
		return "", err
	}
	for _, m := range mounts {
		if err := syscall.Stat(m.Mountpoint, &stat); err != nil {
			logrus.Debugf("[zfs] failed to stat '%s' while scanning for zfs mount: %v", m.Mountpoint, err)
			continue // may fail on fuse file systems
		}

		if stat.Dev == wantedDev && m.Fstype == "zfs" {
			return m.Source, nil
		}
	}

	return "", fmt.Errorf("Failed to find zfs dataset mounted on '%s' in /proc/mounts", rootdir)
}

// Driver holds information about the driver, such as zfs dataset, options and cache.
type Driver struct {
	dataset          *zfs.Dataset
	options          zfsOptions
	sync.Mutex       // protects filesystem cache against concurrent access
	filesystemsCache map[string]bool
	uidMaps          []idtools.IDMap
	gidMaps          []idtools.IDMap
}

func (d *Driver) String() string {
	return "zfs"
}

// Cleanup is used to implement graphdriver.ProtoDriver. There is no cleanup required for this driver.
func (d *Driver) Cleanup() error {
	return nil
}

// Status returns information about the ZFS filesystem. It returns a two dimensional array of information
// such as pool name, dataset name, disk usage, parent quota and compression used.
// Currently it return 'Zpool', 'Zpool Health', 'Parent Dataset', 'Space Used By Parent',
// 'Space Available', 'Parent Quota' and 'Compression'.
func (d *Driver) Status() [][2]string {
	parts := strings.Split(d.dataset.Name, "/")
	pool, err := zfs.GetZpool(parts[0])

	var poolName, poolHealth string
	if err == nil {
		poolName = pool.Name
		poolHealth = pool.Health
	} else {
		poolName = fmt.Sprintf("error while getting pool information %v", err)
		poolHealth = "not available"
	}

	quota := "no"
	if d.dataset.Quota != 0 {
		quota = strconv.FormatUint(d.dataset.Quota, 10)
	}

	return [][2]string{
		{"Zpool", poolName},
		{"Zpool Health", poolHealth},
		{"Parent Dataset", d.dataset.Name},
		{"Space Used By Parent", strconv.FormatUint(d.dataset.Used, 10)},
		{"Space Available", strconv.FormatUint(d.dataset.Avail, 10)},
		{"Parent Quota", quota},
		{"Compression", d.dataset.Compression},
	}
}

// GetMetadata returns image/container metadata related to graph driver
func (d *Driver) GetMetadata(id string) (map[string]string, error) {
	return nil, nil
}

func (d *Driver) cloneFilesystem(name, parentName string) error {
	snapshotName := fmt.Sprintf("%d", time.Now().Nanosecond())
	parentDataset := zfs.Dataset{Name: parentName}
	snapshot, err := parentDataset.Snapshot(snapshotName /*recursive */, false)
	if err != nil {
		return err
	}

	_, err = snapshot.Clone(name, map[string]string{"mountpoint": "legacy"})
	if err == nil {
		d.Lock()
		d.filesystemsCache[name] = true
		d.Unlock()
	}

	if err != nil {
		snapshot.Destroy(zfs.DestroyDeferDeletion)
		return err
	}
	return snapshot.Destroy(zfs.DestroyDeferDeletion)
}

func (d *Driver) zfsPath(id string) string {
	return d.options.fsName + "/" + id
}

func (d *Driver) mountPath(id string) string {
	return path.Join(d.options.mountPath, "graph", getMountpoint(id))
}

// Create prepares the dataset and filesystem for the ZFS driver for the given id under the parent.
func (d *Driver) Create(id string, parent string, mountLabel string) error {
	err := d.create(id, parent)
	if err == nil {
		return nil
	}
	if zfsError, ok := err.(*zfs.Error); ok {
		if !strings.HasSuffix(zfsError.Stderr, "dataset already exists\n") {
			return err
		}
		// aborted build -> cleanup
	} else {
		return err
	}

	dataset := zfs.Dataset{Name: d.zfsPath(id)}
	if err := dataset.Destroy(zfs.DestroyRecursiveClones); err != nil {
		return err
	}

	// retry
	return d.create(id, parent)
}

func (d *Driver) create(id, parent string) error {
	name := d.zfsPath(id)
	if parent == "" {
		mountoptions := map[string]string{"mountpoint": "legacy"}
		fs, err := zfs.CreateFilesystem(name, mountoptions)
		if err == nil {
			d.Lock()
			d.filesystemsCache[fs.Name] = true
			d.Unlock()
		}
		return err
	}
	return d.cloneFilesystem(name, d.zfsPath(parent))
}

// Remove deletes the dataset, filesystem and the cache for the given id.
func (d *Driver) Remove(id string) error {
	name := d.zfsPath(id)
	dataset := zfs.Dataset{Name: name}
	err := dataset.Destroy(zfs.DestroyRecursive)
	if err == nil {
		d.Lock()
		delete(d.filesystemsCache, name)
		d.Unlock()
	}
	return err
}

// Get returns the mountpoint for the given id after creating the target directories if necessary.
func (d *Driver) Get(id, mountLabel string) (string, error) {
	mountpoint := d.mountPath(id)
	filesystem := d.zfsPath(id)
	options := label.FormatMountLabel("", mountLabel)
	logrus.Debugf(`[zfs] mount("%s", "%s", "%s")`, filesystem, mountpoint, options)

	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
	if err != nil {
		return "", err
	}
	// Create the target directories if they don't exist
	if err := idtools.MkdirAllAs(mountpoint, 0755, rootUID, rootGID); err != nil {
		return "", err
	}

	if err := mount.Mount(filesystem, mountpoint, "zfs", options); err != nil {
		return "", fmt.Errorf("error creating zfs mount of %s to %s: %v", filesystem, mountpoint, err)
	}
	// this could be our first mount after creation of the filesystem, and the root dir may still have root
	// permissions instead of the remapped root uid:gid (if user namespaces are enabled):
	if err := os.Chown(mountpoint, rootUID, rootGID); err != nil {
		return "", fmt.Errorf("error modifying zfs mountpoint (%s) directory ownership: %v", mountpoint, err)
	}

	return mountpoint, nil
}

// Put removes the existing mountpoint for the given id if it exists.
func (d *Driver) Put(id string) error {
	mountpoint := d.mountPath(id)
	logrus.Debugf(`[zfs] unmount("%s")`, mountpoint)

	if err := mount.Unmount(mountpoint); err != nil {
		return fmt.Errorf("error unmounting to %s: %v", mountpoint, err)
	}
	return nil
}

// Exists checks to see if the cache entry exists for the given id.
func (d *Driver) Exists(id string) bool {
	return d.filesystemsCache[d.zfsPath(id)] == true
}
<|endoftext|>


<commit_before>package core

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"time"

	"github.com/eure/kamimai/internal/cast"
	"github.com/eure/kamimai/internal/direction"
	"github.com/eure/kamimai/internal/version"
)

const (
	notFoundIndex = 0xffff
)

var (
	errOutOfBoundsMigrations = errors.New("out of bounds migration")
)

type (
	// A Service manages for kamimai.
	Service struct {
		config    *Config
		driver    Driver
		version   uint64
		direction int
		data      Migrations
	}
)

func (s Service) walker(indexPath map[uint64]*Migration) func(path string, info os.FileInfo, err error) error {
	wd, _ := os.Getwd()

	return func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}

		name := info.Name()
		ver := cast.Uint64(version.Get(name))
		mig, found := indexPath[ver]
		if !found || mig == nil {
			mig = &Migration{
				version: ver,
			}
			indexPath[ver] = mig
		}

		if s.direction == direction.Get(name) {
			mig.name = filepath.Clean(filepath.Join(wd, path))
		} else if s.direction == direction.Unknown {
			mig.name = filepath.Clean(filepath.Join(wd, path))
		}

		return nil
	}
}

// NewService returns a new Service pointer that can be chained with builder methods to
// set multiple configuration values inline without using pointers.
func NewService(c *Config) *Service {
	svc := &Service{
		config: c,
	}
	return svc
}

// WithVersion sets a config version value returning a Service pointer
// for chaining.
func (s *Service) WithVersion(v interface{}) *Service {
	s.version = cast.Uint64(v)
	return s
}

// WithDriver sets a driver returning a Service pointer for chaining.
func (s *Service) WithDriver(d Driver) *Service {
	s.driver = d
	return s
}

// MakeMigrationsDir creates a directory named path, along with any necessary
// parents, and returns nil, or else returns an error. If path is
// already a directory, MkdirAll does nothing and returns nil.
func (s *Service) MakeMigrationsDir() error {
	return os.MkdirAll(s.config.migrationsDir(), 0777)
}

func (s *Service) apply() {
	index := map[uint64]*Migration{}
	filepath.Walk(s.config.migrationsDir(), s.walker(index))

	list := make([]*Migration, len(index))
	i := 0
	for _, m := range index {
		list[i] = m
		i++
	}

	migs := Migrations(list)
	sort.Sort(migs)
	s.data = migs
}

func (s *Service) do(idx int) error {
	drv := s.driver

	migs := s.data
	if !(0 <= idx && idx < migs.Len()) {
		return errOutOfBoundsMigrations
	}

	mig := migs[idx]
	if err := drv.Migrate(mig); err != nil {
		return err
	}

	switch s.direction {
	case direction.Up:
		if err := drv.Version().Insert(mig.version); err != nil {
			return err
		}
	case direction.Down:
		if err := drv.Version().Delete(mig.version); err != nil {
			return err
		}
	}

	log.Println("applied", mig.Name())

	return nil
}

func (s *Service) step(n int) error {
	s.apply()

	if s.version == 0 {
		// init
		if n > 0 {
			// only up
			for i := 0; i < n; i++ {
				if err := s.do(i); err != nil {
					return err
				}
			}
		}
		return nil
	}

	// gets current index of migrations
	idx := s.data.index(Migration{version: s.version})

	if n > 0 {
		// up
		for i := 0; i < n; i++ {
			if err := s.do(idx + i + 1); err != nil {
				return err
			}
		}
	} else {
		// down
		for i := 0; i < -n; i++ {
			if err := s.do(idx - i); err != nil {
				return err
			}
		}
	}

	return nil
}

// Apply applies the given migration version.
func (s *Service) Apply(d int, version uint64) error {
	s.direction = d
	s.apply()

	// gets current index of migrations
	idx := s.data.index(Migration{version: version})
	return s.do(idx)
}

func (s *Service) up(n int) error {
	s.direction = direction.Up
	err := s.step(n)
	switch err {
	case errOutOfBoundsMigrations:
		return nil
	}
	return err
}

func (s *Service) down(n int) error {
	s.direction = direction.Down
	err := s.step(-n)
	switch err {
	case errOutOfBoundsMigrations:
		return nil
	}
	return err
}

// Up upgrades migration version.
func (s *Service) Up() error {
	return s.up(notFoundIndex)
}

// Down downgrades migration version.
func (s *Service) Down() error {
	return s.down(notFoundIndex)
}

// Next upgrades migration version.
func (s *Service) Next(n int) error {
	return s.up(n)
}

// Prev downgrades migration version.
func (s *Service) Prev(n int) error {
	return s.down(n)
}

// Sync applies all migrations.
func (s *Service) Sync() error {
	s.direction = direction.Up
	s.apply()

	version := s.driver.Version()

	for _, mig := range s.data {
		if count := version.Count(mig.version); count == 0 {
			// gets current index of migrations
			idx := s.data.index(*mig)
			if err := s.do(idx); err != nil {
				return err
			}
		}
	}

	return nil
}

// NextMigration returns next version migrations.
func (s *Service) NextMigration(name string) (up *Migration, down *Migration, err error) {
	s.apply()

	// initialize default variables for making migrations.
	up, down = &Migration{version: 1, name: ""}, &Migration{version: 1, name: ""}
	ver := "001"

	// gets the oldest migration version file.
	if obj := s.data.last(); obj != nil {
		// for version number
		v := obj.version + 1
		up.version, down.version = v, v
	}
	if obj := s.data.first(); obj != nil {
		// for version format
		_, file := filepath.Split(obj.name)
		ver = version.Format(file)

		if _, err := time.Parse("20060102150405", version.Get(file)); err == nil {
			v := cast.Uint64(time.Now())
			up.version, down.version = v, v
		}

	}

	// [ver]_[name]_[direction-suffix][.ext]
	base := fmt.Sprintf("%s_%s_%%s%%s", ver, name)
	// including dot
	ext := s.driver.Ext()

	// up
	n := fmt.Sprintf(base, up.version, direction.Suffix(direction.Up), ext)
	up.name = filepath.Join(s.config.migrationsDir(), n)
	// down
	n = fmt.Sprintf(base, down.version, direction.Suffix(direction.Down), ext)
	down.name = filepath.Join(s.config.migrationsDir(), n)

	return
}
<commit_msg>Modify version format<commit_after>package core

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"time"

	"github.com/eure/kamimai/internal/cast"
	"github.com/eure/kamimai/internal/direction"
	"github.com/eure/kamimai/internal/version"
)

const (
	notFoundIndex = 0xffff
)

var (
	errOutOfBoundsMigrations = errors.New("out of bounds migration")
)

type (
	// A Service manages for kamimai.
	Service struct {
		config    *Config
		driver    Driver
		version   uint64
		direction int
		data      Migrations
	}
)

func (s Service) walker(indexPath map[uint64]*Migration) func(path string, info os.FileInfo, err error) error {
	wd, _ := os.Getwd()

	return func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}

		name := info.Name()
		ver := cast.Uint64(version.Get(name))
		mig, found := indexPath[ver]
		if !found || mig == nil {
			mig = &Migration{
				version: ver,
			}
			indexPath[ver] = mig
		}

		if s.direction == direction.Get(name) {
			mig.name = filepath.Clean(filepath.Join(wd, path))
		} else if s.direction == direction.Unknown {
			mig.name = filepath.Clean(filepath.Join(wd, path))
		}

		return nil
	}
}

// NewService returns a new Service pointer that can be chained with builder methods to
// set multiple configuration values inline without using pointers.
func NewService(c *Config) *Service {
	svc := &Service{
		config: c,
	}
	return svc
}

// WithVersion sets a config version value returning a Service pointer
// for chaining.
func (s *Service) WithVersion(v interface{}) *Service {
	s.version = cast.Uint64(v)
	return s
}

// WithDriver sets a driver returning a Service pointer for chaining.
func (s *Service) WithDriver(d Driver) *Service {
	s.driver = d
	return s
}

// MakeMigrationsDir creates a directory named path, along with any necessary
// parents, and returns nil, or else returns an error. If path is
// already a directory, MkdirAll does nothing and returns nil.
func (s *Service) MakeMigrationsDir() error {
	return os.MkdirAll(s.config.migrationsDir(), 0777)
}

func (s *Service) apply() {
	index := map[uint64]*Migration{}
	filepath.Walk(s.config.migrationsDir(), s.walker(index))

	list := make([]*Migration, len(index))
	i := 0
	for _, m := range index {
		list[i] = m
		i++
	}

	migs := Migrations(list)
	sort.Sort(migs)
	s.data = migs
}

func (s *Service) do(idx int) error {
	drv := s.driver

	migs := s.data
	if !(0 <= idx && idx < migs.Len()) {
		return errOutOfBoundsMigrations
	}

	mig := migs[idx]
	if err := drv.Migrate(mig); err != nil {
		return err
	}

	switch s.direction {
	case direction.Up:
		if err := drv.Version().Insert(mig.version); err != nil {
			return err
		}
	case direction.Down:
		if err := drv.Version().Delete(mig.version); err != nil {
			return err
		}
	}

	log.Println("applied", mig.Name())

	return nil
}

func (s *Service) step(n int) error {
	s.apply()

	if s.version == 0 {
		// init
		if n > 0 {
			// only up
			for i := 0; i < n; i++ {
				if err := s.do(i); err != nil {
					return err
				}
			}
		}
		return nil
	}

	// gets current index of migrations
	idx := s.data.index(Migration{version: s.version})

	if n > 0 {
		// up
		for i := 0; i < n; i++ {
			if err := s.do(idx + i + 1); err != nil {
				return err
			}
		}
	} else {
		// down
		for i := 0; i < -n; i++ {
			if err := s.do(idx - i); err != nil {
				return err
			}
		}
	}

	return nil
}

// Apply applies the given migration version.
func (s *Service) Apply(d int, version uint64) error {
	s.direction = d
	s.apply()

	// gets current index of migrations
	idx := s.data.index(Migration{version: version})
	return s.do(idx)
}

func (s *Service) up(n int) error {
	s.direction = direction.Up
	err := s.step(n)
	switch err {
	case errOutOfBoundsMigrations:
		return nil
	}
	return err
}

func (s *Service) down(n int) error {
	s.direction = direction.Down
	err := s.step(-n)
	switch err {
	case errOutOfBoundsMigrations:
		return nil
	}
	return err
}

// Up upgrades migration version.
func (s *Service) Up() error {
	return s.up(notFoundIndex)
}

// Down downgrades migration version.
func (s *Service) Down() error {
	return s.down(notFoundIndex)
}

// Next upgrades migration version.
func (s *Service) Next(n int) error {
	return s.up(n)
}

// Prev downgrades migration version.
func (s *Service) Prev(n int) error {
	return s.down(n)
}

// Sync applies all migrations.
func (s *Service) Sync() error {
	s.direction = direction.Up
	s.apply()

	version := s.driver.Version()

	for _, mig := range s.data {
		if count := version.Count(mig.version); count == 0 {
			// gets current index of migrations
			idx := s.data.index(*mig)
			if err := s.do(idx); err != nil {
				return err
			}
		}
	}

	return nil
}

// NextMigration returns next version migrations.
func (s *Service) NextMigration(name string) (up *Migration, down *Migration, err error) {
	s.apply()

	// initialize default variables for making migrations.
	up, down = &Migration{version: 1, name: ""}, &Migration{version: 1, name: ""}
	verFormat := "%03d"

	// gets the oldest migration version file.
	if obj := s.data.last(); obj != nil {
		// for version number
		v := obj.version + 1
		up.version, down.version = v, v
	}
	if obj := s.data.first(); obj != nil {
		// for version format
		_, file := filepath.Split(obj.name)
		verFormat = version.Format(file)

		if _, err := time.Parse("20060102150405", version.Get(file)); err == nil {
			v := cast.Uint64(time.Now())
			up.version, down.version = v, v
		}

	}

	// [ver]_[name]_[direction-suffix][.ext]
	base := fmt.Sprintf("%s_%s_%%s%%s", verFormat, name)
	// including dot
	ext := s.driver.Ext()

	// up
	n := fmt.Sprintf(base, up.version, direction.Suffix(direction.Up), ext)
	up.name = filepath.Join(s.config.migrationsDir(), n)
	// down
	n = fmt.Sprintf(base, down.version, direction.Suffix(direction.Down), ext)
	down.name = filepath.Join(s.config.migrationsDir(), n)

	return
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bpfdebug

import (
	"fmt"
)

const (
	// DropNotifyLen is the amount of packet data provided in a drop notification
	DropNotifyLen = 32
)

// DropNotify is the message format of a drop notification in the BPF ring buffer
type DropNotify struct {
	Type     uint8
	SubType  uint8
	Source   uint16
	Hash     uint32
	OrigLen  uint32
	CapLen   uint32
	SrcLabel uint32
	DstLabel uint32
	DstID    uint32
	Ifindex  uint32
	// data
}

var errors = map[uint8]string{
	0:   "Success",
	2:   "Invalid packet",
	130: "Invalid source mac",
	131: "Invalid destination mac",
	132: "Invalid source ip",
	133: "Policy denied",
	134: "Invalid packet",
	135: "CT: Truncated or invalid header",
	136: "CT: Missing TCP ACK flag",
	137: "CT: Unknown L4 protocol",
	138: "CT: Can't create entry from packet",
	139: "Unsupported L3 protocol",
	140: "Missed tail call",
	141: "Error writing to packet",
	142: "Unknown L4 protocol",
	143: "Unknown ICMPv4 code",
	144: "Unknown ICMPv4 type",
	145: "Unknown ICMPv6 code",
	146: "Unknown ICMPv6 type",
	147: "Error retrieving tunnel key",
	148: "Error retrieving tunnel options",
	149: "Invalid Geneve option",
	150: "Unknown L3 target address",
	151: "Not a local target address",
	152: "No matching local container found",
	153: "Error while correcting L3 checksum",
	154: "Error while correcting L4 checksum",
	155: "CT: Map insertion failed",
	156: "Invalid IPv6 extension header",
	157: "IPv6 fragmentation not supported",
	158: "Service backend not found",
	159: "Policy denied (L4)",
	160: "No tunnel/encapsulation endpoint",
}

func dropReason(reason uint8) string {
	if err, ok := errors[reason]; ok {
		return err
	}
	return fmt.Sprintf("%d", reason)
}

// DumpInfo prints a summary of the drop messages.
func (n *DropNotify) DumpInfo(data []byte) {
	fmt.Printf("xx drop (%s) to endpoint %d, identity %d->%d: %s\n",
		dropReason(n.SubType), n.DstID, n.SrcLabel, n.DstLabel,
		GetConnectionSummary(data[DropNotifyLen:]))
}

// Dump prints the drop notification in human readable form
func (n *DropNotify) DumpVerbose(dissect bool, data []byte, prefix string) {
	fmt.Printf("%s MARK %#x FROM %d Packet dropped %d (%s) %d bytes ifindex=%d",
		prefix, n.Hash, n.Source, n.SubType, dropReason(n.SubType), n.OrigLen, n.Ifindex)

	if n.SrcLabel != 0 || n.DstLabel != 0 {
		fmt.Printf(" %d->%d", n.SrcLabel, n.DstLabel)
	}

	if n.DstID != 0 {
		fmt.Printf(" to lxc %d\n", n.DstID)
	} else {
		fmt.Printf("\n")
	}

	if n.CapLen > 0 && len(data) > DropNotifyLen {
		Dissect(dissect, data[DropNotifyLen:])
	}
}
<commit_msg>monitor: Be specific about policy drop reason<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bpfdebug

import (
	"fmt"
)

const (
	// DropNotifyLen is the amount of packet data provided in a drop notification
	DropNotifyLen = 32
)

// DropNotify is the message format of a drop notification in the BPF ring buffer
type DropNotify struct {
	Type     uint8
	SubType  uint8
	Source   uint16
	Hash     uint32
	OrigLen  uint32
	CapLen   uint32
	SrcLabel uint32
	DstLabel uint32
	DstID    uint32
	Ifindex  uint32
	// data
}

var errors = map[uint8]string{
	0:   "Success",
	2:   "Invalid packet",
	130: "Invalid source mac",
	131: "Invalid destination mac",
	132: "Invalid source ip",
	133: "Policy denied (L3)",
	134: "Invalid packet",
	135: "CT: Truncated or invalid header",
	136: "CT: Missing TCP ACK flag",
	137: "CT: Unknown L4 protocol",
	138: "CT: Can't create entry from packet",
	139: "Unsupported L3 protocol",
	140: "Missed tail call",
	141: "Error writing to packet",
	142: "Unknown L4 protocol",
	143: "Unknown ICMPv4 code",
	144: "Unknown ICMPv4 type",
	145: "Unknown ICMPv6 code",
	146: "Unknown ICMPv6 type",
	147: "Error retrieving tunnel key",
	148: "Error retrieving tunnel options",
	149: "Invalid Geneve option",
	150: "Unknown L3 target address",
	151: "Not a local target address",
	152: "No matching local container found",
	153: "Error while correcting L3 checksum",
	154: "Error while correcting L4 checksum",
	155: "CT: Map insertion failed",
	156: "Invalid IPv6 extension header",
	157: "IPv6 fragmentation not supported",
	158: "Service backend not found",
	159: "Policy denied (L4)",
	160: "No tunnel/encapsulation endpoint",
}

func dropReason(reason uint8) string {
	if err, ok := errors[reason]; ok {
		return err
	}
	return fmt.Sprintf("%d", reason)
}

// DumpInfo prints a summary of the drop messages.
func (n *DropNotify) DumpInfo(data []byte) {
	fmt.Printf("xx drop (%s) to endpoint %d, identity %d->%d: %s\n",
		dropReason(n.SubType), n.DstID, n.SrcLabel, n.DstLabel,
		GetConnectionSummary(data[DropNotifyLen:]))
}

// Dump prints the drop notification in human readable form
func (n *DropNotify) DumpVerbose(dissect bool, data []byte, prefix string) {
	fmt.Printf("%s MARK %#x FROM %d Packet dropped %d (%s) %d bytes ifindex=%d",
		prefix, n.Hash, n.Source, n.SubType, dropReason(n.SubType), n.OrigLen, n.Ifindex)

	if n.SrcLabel != 0 || n.DstLabel != 0 {
		fmt.Printf(" %d->%d", n.SrcLabel, n.DstLabel)
	}

	if n.DstID != 0 {
		fmt.Printf(" to lxc %d\n", n.DstID)
	} else {
		fmt.Printf("\n")
	}

	if n.CapLen > 0 && len(data) > DropNotifyLen {
		Dissect(dissect, data[DropNotifyLen:])
	}
}
<|endoftext|>


<commit_before>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
	"fmt"
	"os"
	"path"

	"istio.io/release-builder/pkg/model"
	"istio.io/release-builder/pkg/util"
)

// Archive creates the release archive that users will download. This includes the installation templates,
// istioctl, and various tools.
func Archive(manifest model.Manifest) error {
	// First, build all variants of istioctl (linux, osx, windows). gen-charts is required for manifests compiled in to istioctl.
	if err := util.RunMake(manifest, "istio", nil, "gen-charts", "istioctl-all", "istioctl.completion"); err != nil {
		return fmt.Errorf("failed to make istioctl: %v", err)
	}

	// We build archives for each arch. These contain the same thing except arch specific istioctl
	for _, arch := range []string{"linux-amd64", "linux-armv7", "linux-arm64", "osx", "win"} {
		out := path.Join(manifest.Directory, "work", "archive", arch, fmt.Sprintf("istio-%s", manifest.Version))
		if err := os.MkdirAll(out, 0750); err != nil {
			return err
		}

		// Some files we just directly copy into the release archive
		directCopies := []string{
			"LICENSE",
			"README.md",

			// Setup tools. The tools/ folder contains a bunch of extra junk, so just select exactly what we want
			"tools/certs/Makefile",
			"tools/certs/README.md",
			"tools/dump_kubernetes.sh",
		}
		for _, file := range directCopies {
			if err := util.CopyFile(path.Join(manifest.RepoDir("istio"), file), path.Join(out, file)); err != nil {
				return err
			}
		}

		// Set up install and samples. We filter down to only some file patterns
		// TODO - clean this up. We probably include files we don't want and exclude files we do want.
		includePatterns := []string{"*.yaml", "*.md", "cleanup.sh", "*.txt", "*.pem", "*.conf", "*.tpl", "*.json", "Makefile"}
		if err := util.CopyDirFiltered(path.Join(manifest.RepoDir("istio"), "samples"), path.Join(out, "samples"), includePatterns); err != nil {
			return err
		}

		manifestsDir := path.Join(out, "manifests")
		if err := os.MkdirAll(manifestsDir, 0755); err != nil {
			return err
		}
		if err := util.CopyDir(path.Join(manifest.RepoDir("istio"), "manifests", "charts"), manifestsDir); err != nil {
			return err
		}
		if err := util.CopyDir(path.Join(manifest.RepoDir("istio"), "manifests", "examples"), manifestsDir); err != nil {
			return err
		}
		if err := util.CopyDir(path.Join(manifest.RepoDir("istio"), "manifests", "profiles"), manifestsDir); err != nil {
			return err
		}

		if err := sanitizeTemplate(manifest, path.Join(out, "manifests/profiles/default.yaml")); err != nil {
			return fmt.Errorf("failed to sanitize operator charts")
		}
		if err := util.CopyDir(path.Join(manifest.RepoDir("istio"), "operator", "samples"), path.Join(out, "samples/operator")); err != nil {
			return err
		}

		// Write manifest
		if err := writeManifest(manifest, out); err != nil {
			return fmt.Errorf("failed to write manifest: %v", err)
		}

		// Copy the istioctl binary over
		istioctlBinary := fmt.Sprintf("istioctl-%s", arch)
		istioctlDest := "istioctl"
		if arch == "win" {
			istioctlBinary += ".exe"
			istioctlDest += ".exe"
		}
		if err := util.CopyFile(path.Join(manifest.RepoOutDir("istio"), istioctlBinary), path.Join(out, "bin", istioctlDest)); err != nil {
			return err
		}
		if err := os.Chmod(path.Join(out, "bin", istioctlDest), 0755); err != nil {
			return err
		}

		// Copy the istioctl completions files to the tools directory
		completionFiles := []string{"istioctl.bash", "_istioctl"}
		for _, file := range completionFiles {
			if err := util.CopyFile(path.Join(manifest.RepoOutDir("istio"), file), path.Join(out, "tools", file)); err != nil {
				return err
			}
		}

		if err := createArchive(arch, manifest, out); err != nil {
			return err
		}

		if err := createStandaloneIstioctl(arch, manifest, out); err != nil {
			return err
		}
	}
	return nil
}

func createStandaloneIstioctl(arch string, manifest model.Manifest, out string) error {
	var istioctlArchive string
	// Create a stand alone archive for istioctl
	// Windows should use zip, linux and osx tar
	if arch == "win" {
		istioctlArchive = fmt.Sprintf("istioctl-%s-%s.zip", manifest.Version, arch)
		if err := util.ZipFolder(path.Join(out, "bin", "istioctl.exe"), path.Join(out, "bin", istioctlArchive)); err != nil {
			return fmt.Errorf("failed to zip istioctl: %v", err)
		}
	} else {
		istioctlArchive = fmt.Sprintf("istioctl-%s-%s.tar.gz", manifest.Version, arch)
		icmd := util.VerboseCommand("tar", "-czf", istioctlArchive, "istioctl")
		icmd.Dir = path.Join(out, "bin")
		if err := icmd.Run(); err != nil {
			return fmt.Errorf("failed to tar istioctl: %v", err)
		}
	}
	// Copy files over to the output directory
	archivePath := path.Join(out, "bin", istioctlArchive)
	dest := path.Join(manifest.OutDir(), istioctlArchive)
	if err := util.CopyFile(archivePath, dest); err != nil {
		return fmt.Errorf("failed to package %v release archive: %v", arch, err)
	}

	// Create a SHA of the archive
	if err := util.CreateSha(dest); err != nil {
		return fmt.Errorf("failed to package %v: %v", dest, err)
	}
	return nil
}

func createArchive(arch string, manifest model.Manifest, out string) error {
	var archive string
	// Create the archive from all the above files
	// Windows should use zip, linux and osx tar
	if arch == "win" {
		archive = fmt.Sprintf("istio-%s-%s.zip", manifest.Version, arch)
		if err := util.ZipFolder(path.Join(out, "..", fmt.Sprintf("istio-%s", manifest.Version)), path.Join(out, "..", archive)); err != nil {
			return fmt.Errorf("failed to zip istioctl: %v", err)
		}
	} else {
		archive = fmt.Sprintf("istio-%s-%s.tar.gz", manifest.Version, arch)
		cmd := util.VerboseCommand("tar", "-czf", archive, fmt.Sprintf("istio-%s", manifest.Version))
		cmd.Dir = path.Join(out, "..")
		if err := cmd.Run(); err != nil {
			return err
		}
	}

	// Copy files over to the output directory
	archivePath := path.Join(manifest.WorkDir(), "archive", arch, archive)
	dest := path.Join(manifest.OutDir(), archive)
	if err := util.CopyFile(archivePath, dest); err != nil {
		return fmt.Errorf("failed to package %v release archive: %v", arch, err)
	}
	// Create a SHA of the archive
	if err := util.CreateSha(dest); err != nil {
		return fmt.Errorf("failed to package %v: %v", dest, err)
	}
	return nil
}
<commit_msg>Copy all tools/certs/Makefile* (#409)<commit_after>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
	"fmt"
	"os"
	"path"

	"istio.io/release-builder/pkg/model"
	"istio.io/release-builder/pkg/util"
)

// Archive creates the release archive that users will download. This includes the installation templates,
// istioctl, and various tools.
func Archive(manifest model.Manifest) error {
	// First, build all variants of istioctl (linux, osx, windows). gen-charts is required for manifests compiled in to istioctl.
	if err := util.RunMake(manifest, "istio", nil, "gen-charts", "istioctl-all", "istioctl.completion"); err != nil {
		return fmt.Errorf("failed to make istioctl: %v", err)
	}

	// We build archives for each arch. These contain the same thing except arch specific istioctl
	for _, arch := range []string{"linux-amd64", "linux-armv7", "linux-arm64", "osx", "win"} {
		out := path.Join(manifest.Directory, "work", "archive", arch, fmt.Sprintf("istio-%s", manifest.Version))
		if err := os.MkdirAll(out, 0750); err != nil {
			return err
		}

		// Some files we just directly copy into the release archive
		directCopies := []string{
			"LICENSE",
			"README.md",

			// Setup tools. The tools/ folder contains a bunch of extra junk, so just select exactly what we want
			"tools/dump_kubernetes.sh",
		}
		for _, file := range directCopies {
			if err := util.CopyFile(path.Join(manifest.RepoDir("istio"), file), path.Join(out, file)); err != nil {
				return err
			}
		}

		// Set up tools/certs. We filter down to only some file patterns
		includePatterns := []string{"README.md", "Makefile*"}
		if err := util.CopyDirFiltered(path.Join(manifest.RepoDir("istio"), "tools", "certs"), path.Join(out, "tools", "certs"), includePatterns); err != nil {
			return err
		}

		// Set up samples. We filter down to only some file patterns
		// TODO - clean this up. We probably include files we don't want and exclude files we do want.
		includePatterns = []string{"*.yaml", "*.md", "cleanup.sh", "*.txt", "*.pem", "*.conf", "*.tpl", "*.json", "Makefile"}
		if err := util.CopyDirFiltered(path.Join(manifest.RepoDir("istio"), "samples"), path.Join(out, "samples"), includePatterns); err != nil {
			return err
		}

		manifestsDir := path.Join(out, "manifests")
		if err := os.MkdirAll(manifestsDir, 0755); err != nil {
			return err
		}
		if err := util.CopyDir(path.Join(manifest.RepoDir("istio"), "manifests", "charts"), manifestsDir); err != nil {
			return err
		}
		if err := util.CopyDir(path.Join(manifest.RepoDir("istio"), "manifests", "examples"), manifestsDir); err != nil {
			return err
		}
		if err := util.CopyDir(path.Join(manifest.RepoDir("istio"), "manifests", "profiles"), manifestsDir); err != nil {
			return err
		}

		if err := sanitizeTemplate(manifest, path.Join(out, "manifests/profiles/default.yaml")); err != nil {
			return fmt.Errorf("failed to sanitize operator charts")
		}
		if err := util.CopyDir(path.Join(manifest.RepoDir("istio"), "operator", "samples"), path.Join(out, "samples/operator")); err != nil {
			return err
		}

		// Write manifest
		if err := writeManifest(manifest, out); err != nil {
			return fmt.Errorf("failed to write manifest: %v", err)
		}

		// Copy the istioctl binary over
		istioctlBinary := fmt.Sprintf("istioctl-%s", arch)
		istioctlDest := "istioctl"
		if arch == "win" {
			istioctlBinary += ".exe"
			istioctlDest += ".exe"
		}
		if err := util.CopyFile(path.Join(manifest.RepoOutDir("istio"), istioctlBinary), path.Join(out, "bin", istioctlDest)); err != nil {
			return err
		}
		if err := os.Chmod(path.Join(out, "bin", istioctlDest), 0755); err != nil {
			return err
		}

		// Copy the istioctl completions files to the tools directory
		completionFiles := []string{"istioctl.bash", "_istioctl"}
		for _, file := range completionFiles {
			if err := util.CopyFile(path.Join(manifest.RepoOutDir("istio"), file), path.Join(out, "tools", file)); err != nil {
				return err
			}
		}

		if err := createArchive(arch, manifest, out); err != nil {
			return err
		}

		if err := createStandaloneIstioctl(arch, manifest, out); err != nil {
			return err
		}
	}
	return nil
}

func createStandaloneIstioctl(arch string, manifest model.Manifest, out string) error {
	var istioctlArchive string
	// Create a stand alone archive for istioctl
	// Windows should use zip, linux and osx tar
	if arch == "win" {
		istioctlArchive = fmt.Sprintf("istioctl-%s-%s.zip", manifest.Version, arch)
		if err := util.ZipFolder(path.Join(out, "bin", "istioctl.exe"), path.Join(out, "bin", istioctlArchive)); err != nil {
			return fmt.Errorf("failed to zip istioctl: %v", err)
		}
	} else {
		istioctlArchive = fmt.Sprintf("istioctl-%s-%s.tar.gz", manifest.Version, arch)
		icmd := util.VerboseCommand("tar", "-czf", istioctlArchive, "istioctl")
		icmd.Dir = path.Join(out, "bin")
		if err := icmd.Run(); err != nil {
			return fmt.Errorf("failed to tar istioctl: %v", err)
		}
	}
	// Copy files over to the output directory
	archivePath := path.Join(out, "bin", istioctlArchive)
	dest := path.Join(manifest.OutDir(), istioctlArchive)
	if err := util.CopyFile(archivePath, dest); err != nil {
		return fmt.Errorf("failed to package %v release archive: %v", arch, err)
	}

	// Create a SHA of the archive
	if err := util.CreateSha(dest); err != nil {
		return fmt.Errorf("failed to package %v: %v", dest, err)
	}
	return nil
}

func createArchive(arch string, manifest model.Manifest, out string) error {
	var archive string
	// Create the archive from all the above files
	// Windows should use zip, linux and osx tar
	if arch == "win" {
		archive = fmt.Sprintf("istio-%s-%s.zip", manifest.Version, arch)
		if err := util.ZipFolder(path.Join(out, "..", fmt.Sprintf("istio-%s", manifest.Version)), path.Join(out, "..", archive)); err != nil {
			return fmt.Errorf("failed to zip istioctl: %v", err)
		}
	} else {
		archive = fmt.Sprintf("istio-%s-%s.tar.gz", manifest.Version, arch)
		cmd := util.VerboseCommand("tar", "-czf", archive, fmt.Sprintf("istio-%s", manifest.Version))
		cmd.Dir = path.Join(out, "..")
		if err := cmd.Run(); err != nil {
			return err
		}
	}

	// Copy files over to the output directory
	archivePath := path.Join(manifest.WorkDir(), "archive", arch, archive)
	dest := path.Join(manifest.OutDir(), archive)
	if err := util.CopyFile(archivePath, dest); err != nil {
		return fmt.Errorf("failed to package %v release archive: %v", arch, err)
	}
	// Create a SHA of the archive
	if err := util.CreateSha(dest); err != nil {
		return fmt.Errorf("failed to package %v: %v", dest, err)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package clusterd

import (
	"errors"
	"fmt"
	"path"
	"regexp"
	"strconv"

	"github.com/coreos/pkg/capnslog"

	"github.com/rook/rook/pkg/util/exec"
	"github.com/rook/rook/pkg/util/sys"
)

var (
	logger = capnslog.NewPackageLogger("github.com/rook/rook", "inventory")
	isRBD  = regexp.MustCompile("^rbd[0-9]+p?[0-9]{0,}$")
)

// GetDeviceEmpty check whether a device is completely empty
func GetDeviceEmpty(device *sys.LocalDisk) bool {
	return device.Parent == "" && (device.Type == sys.DiskType || device.Type == sys.SSDType || device.Type == sys.CryptType || device.Type == sys.LVMType) && len(device.Partitions) == 0 && device.Filesystem == ""
}

func ignoreDevice(d string) bool {
	return isRBD.MatchString(d)
}

// DiscoverDevices returns all the details of devices available on the local node
func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) {
	var disks []*sys.LocalDisk
	devices, err := sys.ListDevices(executor)
	if err != nil {
		return nil, err
	}

	for _, d := range devices {
		// Ignore RBD device
		if ignoreDevice(d) {
			// skip device
			logger.Warningf("skipping rbd device %q", d)
			continue
		}

		// Populate device information coming from lsblk
		disk, err := PopulateDeviceInfo(d, executor)
		if err != nil {
			logger.Warningf("skipping device %q. %v", d, err)
			continue
		}

		// Populate udev information coming from udev
		disk, err = PopulateDeviceUdevInfo(d, executor, disk)
		if err != nil {
			// go on without udev info
			// not ideal for our filesystem check later but we can't really fail either...
			logger.Warningf("failed to get udev info for device %q. %v", d, err)
		}

		// Test if device has child, if so we skip it and only consider the partitions
		// which will come in later iterations of the loop
		// We only test if the type is 'disk', this is a property reported by lsblk
		// and means it's a parent block device
		if disk.Type == sys.DiskType {
			deviceChild, err := sys.ListDevicesChild(executor, d)
			if err != nil {
				logger.Warningf("failed to detect child devices for device %q, assuming they are none. %v", d, err)
			}
			// lsblk will output at least 2 lines if they are partitions, one for the parent
			// and N for the child
			if len(deviceChild) > 1 {
				logger.Infof("skipping device %q because it has child, considering the child instead.", d)
				continue
			}
		}

		disks = append(disks, disk)
	}
	logger.Debugf("discovered disks are %v", disks)

	return disks, nil
}

// PopulateDeviceInfo returns the information of the specified block device
func PopulateDeviceInfo(d string, executor exec.Executor) (*sys.LocalDisk, error) {
	diskProps, err := sys.GetDeviceProperties(d, executor)
	if err != nil {
		return nil, err
	}

	diskType, ok := diskProps["TYPE"]
	if !ok || (diskType != sys.SSDType && diskType != sys.CryptType && diskType != sys.DiskType && diskType != sys.PartType && diskType != sys.LinearType && diskType != sys.LVMType) {
		if !ok {
			return nil, errors.New("diskType is empty")
		} else {
			return nil, fmt.Errorf("unsupported diskType %+s", diskType)
		}
	}

	// get the UUID for disks
	var diskUUID string
	if diskType != sys.PartType {
		diskUUID, err = sys.GetDiskUUID(d, executor)
		if err != nil {
			return nil, err
		}
	}

	disk := &sys.LocalDisk{Name: d, UUID: diskUUID}

	if val, ok := diskProps["TYPE"]; ok {
		disk.Type = val
	}
	if val, ok := diskProps["SIZE"]; ok {
		if size, err := strconv.ParseUint(val, 10, 64); err == nil {
			disk.Size = size
		}
	}
	if val, ok := diskProps["ROTA"]; ok {
		if rotates, err := strconv.ParseBool(val); err == nil {
			disk.Rotational = rotates
		}
	}
	if val, ok := diskProps["RO"]; ok {
		if ro, err := strconv.ParseBool(val); err == nil {
			disk.Readonly = ro
		}
	}
	if val, ok := diskProps["PKNAME"]; ok {
		disk.Parent = path.Base(val)
	}
	if val, ok := diskProps["NAME"]; ok {
		disk.RealPath = val
	}

	return disk, nil
}

// PopulateDeviceUdevInfo fills the udev info into the block device information
func PopulateDeviceUdevInfo(d string, executor exec.Executor, disk *sys.LocalDisk) (*sys.LocalDisk, error) {
	udevInfo, err := sys.GetUdevInfo(d, executor)
	if err != nil {
		return disk, err
	}
	// parse udev info output
	if val, ok := udevInfo["DEVLINKS"]; ok {
		disk.DevLinks = val
	}
	if val, ok := udevInfo["ID_FS_TYPE"]; ok {
		disk.Filesystem = val
	}
	if val, ok := udevInfo["ID_SERIAL"]; ok {
		disk.Serial = val
	}

	if val, ok := udevInfo["ID_VENDOR"]; ok {
		disk.Vendor = val
	}

	if val, ok := udevInfo["ID_MODEL"]; ok {
		disk.Model = val
	}

	if val, ok := udevInfo["ID_WWN_WITH_EXTENSION"]; ok {
		disk.WWNVendorExtension = val
	}

	if val, ok := udevInfo["ID_WWN"]; ok {
		disk.WWN = val
	}

	return disk, nil
}
<commit_msg>edgefs: raw disk empty check regression fix<commit_after>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package clusterd

import (
	"errors"
	"fmt"
	"path"
	"regexp"
	"strconv"

	"github.com/coreos/pkg/capnslog"

	"github.com/rook/rook/pkg/util/exec"
	"github.com/rook/rook/pkg/util/sys"
)

var (
	logger = capnslog.NewPackageLogger("github.com/rook/rook", "inventory")
	isRBD  = regexp.MustCompile("^rbd[0-9]+p?[0-9]{0,}$")
)

// GetDeviceEmpty check whether a device is completely empty
func GetDeviceEmpty(device *sys.LocalDisk) bool {
	return device.Parent == "" && (device.Type == sys.DiskType || device.Type == sys.SSDType || device.Type == sys.CryptType || device.Type == sys.LVMType) && len(device.Partitions) == 0 && device.Filesystem == ""
}

func ignoreDevice(d string) bool {
	return isRBD.MatchString(d)
}

// DiscoverDevices returns all the details of devices available on the local node
func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) {
	var disks []*sys.LocalDisk
	devices, err := sys.ListDevices(executor)
	if err != nil {
		return nil, err
	}

	for _, d := range devices {
		// Ignore RBD device
		if ignoreDevice(d) {
			// skip device
			logger.Warningf("skipping rbd device %q", d)
			continue
		}

		// Populate device information coming from lsblk
		disk, err := PopulateDeviceInfo(d, executor)
		if err != nil {
			logger.Warningf("skipping device %q. %v", d, err)
			continue
		}

		// Populate udev information coming from udev
		disk, err = PopulateDeviceUdevInfo(d, executor, disk)
		if err != nil {
			// go on without udev info
			// not ideal for our filesystem check later but we can't really fail either...
			logger.Warningf("failed to get udev info for device %q. %v", d, err)
		}

		// Test if device has child, if so we skip it and only consider the partitions
		// which will come in later iterations of the loop
		// We only test if the type is 'disk', this is a property reported by lsblk
		// and means it's a parent block device
		if disk.Type == sys.DiskType {
			deviceChild, err := sys.ListDevicesChild(executor, d)
			if err != nil {
				logger.Warningf("failed to detect child devices for device %q, assuming they are none. %v", d, err)
			}
			// lsblk will output at least 2 lines if they are partitions, one for the parent
			// and N for the child
			if len(deviceChild) > 1 {
				logger.Infof("skipping device %q because it has child, considering the child instead.", d)
				continue
			}
		}

		disks = append(disks, disk)
	}
	logger.Debugf("discovered disks are %v", disks)

	return disks, nil
}

// PopulateDeviceInfo returns the information of the specified block device
func PopulateDeviceInfo(d string, executor exec.Executor) (*sys.LocalDisk, error) {
	diskProps, err := sys.GetDeviceProperties(d, executor)
	if err != nil {
		return nil, err
	}

	diskType, ok := diskProps["TYPE"]
	if !ok || (diskType != sys.SSDType && diskType != sys.CryptType && diskType != sys.DiskType && diskType != sys.PartType && diskType != sys.LinearType && diskType != sys.LVMType) {
		if !ok {
			return nil, errors.New("diskType is empty")
		} else {
			return nil, fmt.Errorf("unsupported diskType %+s", diskType)
		}
	}

	// get the UUID for disks
	var diskUUID string
	if diskType != sys.PartType {
		diskUUID, err = sys.GetDiskUUID(d, executor)
		if err != nil {
			return nil, err
		}
	}

	disk := &sys.LocalDisk{Name: d, UUID: diskUUID}

	if val, ok := diskProps["TYPE"]; ok {
		disk.Type = val
	}
	if val, ok := diskProps["SIZE"]; ok {
		if size, err := strconv.ParseUint(val, 10, 64); err == nil {
			disk.Size = size
		}
	}
	if val, ok := diskProps["ROTA"]; ok {
		if rotates, err := strconv.ParseBool(val); err == nil {
			disk.Rotational = rotates
		}
	}
	if val, ok := diskProps["RO"]; ok {
		if ro, err := strconv.ParseBool(val); err == nil {
			disk.Readonly = ro
		}
	}
	if val, ok := diskProps["PKNAME"]; ok {
		if val != "" {
			disk.Parent = path.Base(val)
		}
	}
	if val, ok := diskProps["NAME"]; ok {
		disk.RealPath = val
	}

	return disk, nil
}

// PopulateDeviceUdevInfo fills the udev info into the block device information
func PopulateDeviceUdevInfo(d string, executor exec.Executor, disk *sys.LocalDisk) (*sys.LocalDisk, error) {
	udevInfo, err := sys.GetUdevInfo(d, executor)
	if err != nil {
		return disk, err
	}
	// parse udev info output
	if val, ok := udevInfo["DEVLINKS"]; ok {
		disk.DevLinks = val
	}
	if val, ok := udevInfo["ID_FS_TYPE"]; ok {
		disk.Filesystem = val
	}
	if val, ok := udevInfo["ID_SERIAL"]; ok {
		disk.Serial = val
	}

	if val, ok := udevInfo["ID_VENDOR"]; ok {
		disk.Vendor = val
	}

	if val, ok := udevInfo["ID_MODEL"]; ok {
		disk.Model = val
	}

	if val, ok := udevInfo["ID_WWN_WITH_EXTENSION"]; ok {
		disk.WWNVendorExtension = val
	}

	if val, ok := udevInfo["ID_WWN"]; ok {
		disk.WWN = val
	}

	return disk, nil
}
<|endoftext|>


<commit_before>// Package config parse and validates command flags.
package config

import (
	"flag"
	"fmt"
	"os"
	"strings"
)

// Resource can be either Redis (isRedis) or file.
// URI is either a Redis URI or a file path.
// Silent mode will disable read/write logs.
type Resource struct {
	URI     string
	IsRedis bool
	Silent bool
}

// Config represents the current source and target config.
// Source and target are Resources.
type Config struct {
	Source Resource
	Target Resource
}

// exit will exit and print the usage.
// Used in case of errors during flags parse/validate.
func exit(e error) {
	fmt.Println(e)
	flag.PrintDefaults()
	os.Exit(1)
}

// validate makes sure from and to are Redis URIs or file paths,
// and generates the final Config.
func validate(from, to string, silent bool) (Config, error) {
	cfg := Config{
		Source: Resource{
			URI: from,
			Silent: silent,
		},
		Target: Resource{
			URI: to,
			Silent: silent,
		},
	}

	if strings.HasPrefix(from, "redis://") {
		cfg.Source.IsRedis = true
	}

	if strings.HasPrefix(to, "redis://") {
		cfg.Target.IsRedis = true
	}

	// Guard from incorrect usage.
	switch {
	case cfg.Source.URI == "":
		return cfg, fmt.Errorf("from is required")
	case cfg.Target.URI == "":
		return cfg, fmt.Errorf("to is required")
	case !cfg.Source.IsRedis && !cfg.Target.IsRedis:
		return cfg, fmt.Errorf("file-only operations not supported")
	}

	return cfg, nil
}

// Parse parses the command line flags and returns a Config.
func Parse() Config {
	example := "example: redis://127.0.0.1:6379/0 or /tmp/dump.rump"
	from := flag.String("from", "", example)
	to := flag.String("to", "", example)
	silent := flag.Bool("silent", false, "example: -silent")

	flag.Parse()

	cfg, err := validate(*from, *to, *silent)
	if err != nil {
		// we exit here instead of returning so that we can show
		// the usage examples in case of an error.
		exit(err)
	}

	return cfg
}
<commit_msg>silent mode: improve help message<commit_after>// Package config parse and validates command flags.
package config

import (
	"flag"
	"fmt"
	"os"
	"strings"
)

// Resource can be either Redis (isRedis) or file.
// URI is either a Redis URI or a file path.
// Silent mode will disable read/write logs.
type Resource struct {
	URI     string
	IsRedis bool
	Silent bool
}

// Config represents the current source and target config.
// Source and target are Resources.
type Config struct {
	Source Resource
	Target Resource
}

// exit will exit and print the usage.
// Used in case of errors during flags parse/validate.
func exit(e error) {
	fmt.Println(e)
	flag.PrintDefaults()
	os.Exit(1)
}

// validate makes sure from and to are Redis URIs or file paths,
// and generates the final Config.
func validate(from, to string, silent bool) (Config, error) {
	cfg := Config{
		Source: Resource{
			URI: from,
			Silent: silent,
		},
		Target: Resource{
			URI: to,
			Silent: silent,
		},
	}

	if strings.HasPrefix(from, "redis://") {
		cfg.Source.IsRedis = true
	}

	if strings.HasPrefix(to, "redis://") {
		cfg.Target.IsRedis = true
	}

	// Guard from incorrect usage.
	switch {
	case cfg.Source.URI == "":
		return cfg, fmt.Errorf("from is required")
	case cfg.Target.URI == "":
		return cfg, fmt.Errorf("to is required")
	case !cfg.Source.IsRedis && !cfg.Target.IsRedis:
		return cfg, fmt.Errorf("file-only operations not supported")
	}

	return cfg, nil
}

// Parse parses the command line flags and returns a Config.
func Parse() Config {
	example := "example: redis://127.0.0.1:6379/0 or /tmp/dump.rump"
	from := flag.String("from", "", example)
	to := flag.String("to", "", example)
	silent := flag.Bool("silent", false, "disable verbose output")

	flag.Parse()

	cfg, err := validate(*from, *to, *silent)
	if err != nil {
		// we exit here instead of returning so that we can show
		// the usage examples in case of an error.
		exit(err)
	}

	return cfg
}
<|endoftext|>


<commit_before>package config

import (
    "github.com/gorilla/sessions"
    "strings"
    mutils "github.com/nebtex/menshend/pkg/utils"
    "net/url"
    "os"
    vault "github.com/hashicorp/vault/api"
    "github.com/Sirupsen/logrus"
    "io/ioutil"
    "github.com/ghodss/yaml"
    "fmt"
    "math"
    "github.com/markbates/goth/gothic"
    "github.com/markbates/goth/providers/github"
    "github.com/markbates/goth"
)

type GithubConfig struct {
    ClientID     string `json:"clientId"`
    ClientSecret string `json:"clientSecret"`
    Host         string `json:"host"`
}

type Uris struct {
    BaseUrl           string `json:"baseUrl"`
    MenshendSubdomain string `json:"menshendSubdomain"`
}

type Space struct {
    Logo            string `json:"logo"`
    Description     string `json:"description"`
    LongDescription string `json:"longDescription"`
    Name            string `json:"name"`
}

type MenshendConfig struct {
    HashKey     string        `json:"hashKey"`
    BlockKey    string        `json:"blockKey"`
    VaultPath   string        `json:"vaultPath"`
    Uris        Uris          `json:"uris"`
    DefaultRole string        `json:"defaultRole"`
    Github      GithubConfig  `json:"github"`
    Space       Space         `json:"space"`
}

func (k *MenshendConfig) Host() string {
    URL, err := url.Parse(k.Uris.BaseUrl)
    mutils.CheckPanic(err)
    return URL.Host
}

func (k *MenshendConfig) Scheme() string {
    URL, err := url.Parse(k.Uris.BaseUrl)
    mutils.CheckPanic(err)
    return URL.Scheme
}
func (k *MenshendConfig) HostWithoutPort() string {
    return strings.Split(k.Host(), ":")[0]
}

func (k *MenshendConfig) GetLoginPath() string {
    loginUrl := k.Scheme() + "://" + k.Uris.MenshendSubdomain + k.Host() + "/login"
    return loginUrl
}
func (k *MenshendConfig) GetServicePath() string {
    return k.Scheme() + "://" + k.Uris.MenshendSubdomain + k.Host() + "/services"
}

func (k *MenshendConfig) GetLogo() string {
    return k.Space.Logo
}

func (k *MenshendConfig) GetLongDescription() string {
    return k.Space.LongDescription
}

func (k *MenshendConfig) GetCallbackUrl(provider string, subdomain string) string {
    if subdomain != "" {
        return k.Scheme() + "://" + k.Uris.MenshendSubdomain + k.Host() + "/ui" + fmt.Sprintf("/auth/%s/callback/%s", provider, subdomain)
    }
    return k.Scheme() + "://" + k.Uris.MenshendSubdomain + k.Host() + "/ui" + fmt.Sprintf("/auth/%s/callback", provider)
}

func (k *MenshendConfig) GetShortDescription() string {
    return k.Space.Description
}

func (k *MenshendConfig) GetName() string {
    return k.Space.Name
}
func (k*MenshendConfig)GetSubdomainFullUrl(sub string) string {
    return k.Scheme() + "://" + sub + k.Host()
}

var Config *MenshendConfig
var ConfigFile *string
var FlashStore *sessions.CookieStore

// generates default config
func init() {
    Config = &MenshendConfig{}
    Config.Uris = Uris{BaseUrl:"http://test.local", MenshendSubdomain: ""}
    Config.DefaultRole = "default"
    Config.VaultPath = "secret/menshend"
    Config.HashKey = string(mutils.GenerateRandomBytes(32))
    Config.BlockKey = string(mutils.GenerateRandomBytes(32))

    FlashStore = sessions.NewCookieStore([]byte(Config.HashKey), []byte(Config.BlockKey))
    FlashStore.Options.Domain = "." + Config.HostWithoutPort()
    FlashStore.Options.Path = "/"
    if Config.Scheme() == "http" {
        FlashStore.Options.Secure = false
    }

    //store := sessions.NewFilesystemStore(os.TempDir(), []byte(Config.HashKey), []byte(Config.BlockKey))

    // set the maxLength of the cookies stored on the disk to a larger number to prevent issues with:
    // securecookie: the value is too long
    // when using OpenID Connect , since this can contain a large amount of extra information in the id_token

    // Note, when using the FilesystemStore only the session.ID is written to a browser cookie, so this is explicit for the storage on disk
    //store.MaxLength(int(math.MaxInt32))

    //gothic.Store = store
}

func LoadConfig() error {
    //check if vault token is defined
    if os.Getenv(vault.EnvVaultToken) != "" {
        return fmt.Errorf("Menshend servar is not allowed to run when %s is defined, please check you environmental variable and delete %s", vault.EnvVaultToken, vault.EnvVaultToken)
    }
    //check if file is defined
    if ConfigFile != nil && *ConfigFile != "" {
        //load config from file
        data, err := ioutil.ReadFile(*ConfigFile)
        if err != nil {
            return err
        }
        err = yaml.Unmarshal(data, &Config)
        if err != nil {
            return err
        }
    }
    //pre-flight check
    if Config.HashKey == "" {
        Config.HashKey = string(mutils.GenerateRandomBytes(32))
    }
    if Config.BlockKey == "" {
        Config.BlockKey = string(mutils.GenerateRandomBytes(32))
    }
    if len(Config.HashKey) != 32 || len(Config.BlockKey) != 32 {
        logrus.Fatal("HashKey and BlockKey only support secret phrase of 32 characters")
    }

    if Config.VaultPath == "" {
        Config.VaultPath = "secret/menshend"
    }

    if Config.Uris.BaseUrl == "" {
        logrus.Fatal("please define the base url of menshend 'Config.Uris.BaseUrl'")
    }

    //pass
    FlashStore = sessions.NewCookieStore([]byte(Config.HashKey), []byte(Config.BlockKey))
    FlashStore.Options.Domain = "." + Config.HostWithoutPort()
    FlashStore.Options.Path = "/"
    if Config.Scheme() == "http" {
        FlashStore.Options.Secure = false
    }
    if (Config.Github.ClientID != "" && Config.Github.ClientSecret != "") {
        if Config.Github.Host == "" {
            Config.Github.Host = "github.com"

        }
        github.AuthURL = fmt.Sprintf("https://%s/login/oauth/authorize", Config.Github.Host)
        github.TokenURL = fmt.Sprintf("https://%s/login/oauth/access_token", Config.Github.Host)
        github.ProfileURL = fmt.Sprintf("https://api.%s/user", Config.Github.Host)
        github.EmailURL = fmt.Sprintf("https://api.%s/user/emails", Config.Github.Host)
        callback := Config.Scheme() + "://" + Config.Uris.MenshendSubdomain + Config.Host() + "/ui" + "/auth/github/callback"
        goth.UseProviders(
            github.New(Config.Github.ClientID, Config.Github.ClientSecret, callback, "read:org"),
        )
    }
    //TODO: check menshend subdomain

    store := sessions.NewFilesystemStore(os.TempDir(), []byte(Config.HashKey), []byte(Config.BlockKey))
    store.MaxLength(int(math.MaxInt32))
    gothic.Store = store
    return nil
}
<commit_msg>fix flashes tests<commit_after>package config

import (
    "github.com/gorilla/sessions"
    "strings"
    mutils "github.com/nebtex/menshend/pkg/utils"
    "net/url"
    "os"
    vault "github.com/hashicorp/vault/api"
    "github.com/Sirupsen/logrus"
    "io/ioutil"
    "github.com/ghodss/yaml"
    "fmt"
    "math"
    "github.com/markbates/goth/gothic"
    "github.com/markbates/goth/providers/github"
    "github.com/markbates/goth"
)

type GithubConfig struct {
    ClientID     string `json:"clientId"`
    ClientSecret string `json:"clientSecret"`
    Host         string `json:"host"`
}

type Uris struct {
    BaseUrl           string `json:"baseUrl"`
    MenshendSubdomain string `json:"menshendSubdomain"`
}

type Space struct {
    Logo            string `json:"logo"`
    Description     string `json:"description"`
    LongDescription string `json:"longDescription"`
    Name            string `json:"name"`
}

type MenshendConfig struct {
    HashKey     string        `json:"hashKey"`
    BlockKey    string        `json:"blockKey"`
    VaultPath   string        `json:"vaultPath"`
    Uris        Uris          `json:"uris"`
    DefaultRole string        `json:"defaultRole"`
    Github      GithubConfig  `json:"github"`
    Space       Space         `json:"space"`
}

func (k *MenshendConfig) Host() string {
    URL, err := url.Parse(k.Uris.BaseUrl)
    mutils.CheckPanic(err)
    return URL.Host
}

func (k *MenshendConfig) Scheme() string {
    URL, err := url.Parse(k.Uris.BaseUrl)
    mutils.CheckPanic(err)
    return URL.Scheme
}
func (k *MenshendConfig) HostWithoutPort() string {
    return strings.Split(k.Host(), ":")[0]
}

func (k *MenshendConfig) GetLoginPath() string {
    loginUrl := k.Scheme() + "://" + k.Uris.MenshendSubdomain + k.Host() + "/login"
    return loginUrl
}
func (k *MenshendConfig) GetServicePath() string {
    return k.Scheme() + "://" + k.Uris.MenshendSubdomain + k.Host() + "/services"
}

func (k *MenshendConfig) GetLogo() string {
    return k.Space.Logo
}

func (k *MenshendConfig) GetLongDescription() string {
    return k.Space.LongDescription
}

func (k *MenshendConfig) GetCallbackUrl(provider string, subdomain string) string {
    if subdomain != "" {
        return k.Scheme() + "://" + k.Uris.MenshendSubdomain + k.Host() + "/ui" + fmt.Sprintf("/auth/%s/callback/%s", provider, subdomain)
    }
    return k.Scheme() + "://" + k.Uris.MenshendSubdomain + k.Host() + "/ui" + fmt.Sprintf("/auth/%s/callback", provider)
}

func (k *MenshendConfig) GetShortDescription() string {
    return k.Space.Description
}

func (k *MenshendConfig) GetName() string {
    return k.Space.Name
}
func (k*MenshendConfig)GetSubdomainFullUrl(sub string) string {
    return k.Scheme() + "://" + sub + k.Host()
}

var Config *MenshendConfig
var ConfigFile *string
var FlashStore *sessions.CookieStore

// generates default config
func init() {
    Config = &MenshendConfig{}
    Config.Uris = Uris{BaseUrl:"http://test.local", MenshendSubdomain: ""}
    Config.DefaultRole = "default"
    Config.VaultPath = "secret/menshend"
    Config.HashKey = string(mutils.GenerateRandomBytes(32))
    Config.BlockKey = string(mutils.GenerateRandomBytes(32))

    FlashStore = sessions.NewCookieStore([]byte(Config.HashKey), []byte(Config.BlockKey))
    FlashStore.Options.Domain = "." + Config.HostWithoutPort()
    FlashStore.Options.Path = "/"
    if Config.Scheme() == "http" {
        FlashStore.Options.Secure = false
    }

}

func LoadConfig() error {
    //check if vault token is defined
    if os.Getenv(vault.EnvVaultToken) != "" {
        return fmt.Errorf("Menshend servar is not allowed to run when %s is defined, please check you environmental variable and delete %s", vault.EnvVaultToken, vault.EnvVaultToken)
    }
    //check if file is defined
    if ConfigFile != nil && *ConfigFile != "" {
        //load config from file
        data, err := ioutil.ReadFile(*ConfigFile)
        if err != nil {
            return err
        }
        err = yaml.Unmarshal(data, &Config)
        if err != nil {
            return err
        }
    }
    //pre-flight check
    if Config.HashKey == "" {
        Config.HashKey = string(mutils.GenerateRandomBytes(32))
    }
    if Config.BlockKey == "" {
        Config.BlockKey = string(mutils.GenerateRandomBytes(32))
    }
    if len(Config.HashKey) != 32 || len(Config.BlockKey) != 32 {
        logrus.Fatal("HashKey and BlockKey only support secret phrase of 32 characters")
    }

    if Config.VaultPath == "" {
        Config.VaultPath = "secret/menshend"
    }

    if Config.Uris.BaseUrl == "" {
        logrus.Fatal("please define the base url of menshend 'Config.Uris.BaseUrl'")
    }

    //pass
    FlashStore = sessions.NewCookieStore([]byte(Config.HashKey), []byte(Config.BlockKey))
    FlashStore.Options.Domain = "." + Config.HostWithoutPort()
    FlashStore.Options.Path = "/"
    if Config.Scheme() == "http" {
        FlashStore.Options.Secure = false
    }
    if (Config.Github.ClientID != "" && Config.Github.ClientSecret != "") {
        if Config.Github.Host == "" {
            Config.Github.Host = "github.com"

        }
        github.AuthURL = fmt.Sprintf("https://%s/login/oauth/authorize", Config.Github.Host)
        github.TokenURL = fmt.Sprintf("https://%s/login/oauth/access_token", Config.Github.Host)
        github.ProfileURL = fmt.Sprintf("https://api.%s/user", Config.Github.Host)
        github.EmailURL = fmt.Sprintf("https://api.%s/user/emails", Config.Github.Host)
        callback := Config.Scheme() + "://" + Config.Uris.MenshendSubdomain + Config.Host() + "/ui" + "/auth/github/callback"
        goth.UseProviders(
            github.New(Config.Github.ClientID, Config.Github.ClientSecret, callback, "read:org"),
        )
    }
    //TODO: check menshend subdomain

    store := sessions.NewFilesystemStore(os.TempDir(), []byte(Config.HashKey), []byte(Config.BlockKey))
    store.MaxLength(int(math.MaxInt32))
    gothic.Store = store
    return nil
}
<|endoftext|>


<commit_before>package proxy

import (
	"context"
	"time"

	"github.com/fagongzi/gateway/pkg/pb/metapb"
	"github.com/fagongzi/gateway/pkg/store"
	"github.com/fagongzi/gateway/pkg/util"
	"github.com/fagongzi/log"
	"github.com/valyala/fasthttp"
)

func (r *dispatcher) readyToHeathChecker() {
	for i := 0; i < r.cnf.Option.LimitCountHeathCheckWorker; i++ {
		r.runner.RunCancelableTask(func(ctx context.Context) {
			log.Infof("start server check worker")

			for {
				select {
				case <-ctx.Done():
					return
				case id := <-r.checkerC:
					r.check(id)
				}
			}
		})
	}
}

func (r *dispatcher) addToCheck(svr *serverRuntime) {
	svr.circuit = metapb.Open
	if svr.meta.HeathCheck != nil {
		svr.useCheckDuration = time.Duration(svr.meta.HeathCheck.CheckInterval)
	}
	svr.heathTimeout.Stop()
	r.checkerC <- svr.meta.ID
}

func (r *dispatcher) heathCheckTimeout(arg interface{}) {
	id := arg.(uint64)
	if _, ok := r.servers[id]; ok {
		r.checkerC <- id
	}
}

func (r *dispatcher) check(id uint64) {
	svr, ok := r.servers[id]
	if !ok {
		return
	}

	defer func() {
		if svr.meta.HeathCheck != nil {
			if svr.useCheckDuration > r.cnf.Option.LimitIntervalHeathCheck {
				svr.useCheckDuration = r.cnf.Option.LimitIntervalHeathCheck
			}
			svr.heathTimeout, _ = r.tw.Schedule(svr.useCheckDuration, r.heathCheckTimeout, id)
		}
	}()

	status := metapb.Unknown
	prev := r.getServerStatus(svr.meta.ID)

	if svr.meta.HeathCheck == nil {
		log.Warnf("server <%d> heath check not setting", svr.meta.ID)
		r.watchEventC <- &store.Evt{
			Src:  eventSrcStatusChanged,
			Type: eventTypeStatusChanged,
			Value: statusChanged{
				meta:   *svr.meta,
				status: metapb.Up,
			},
		}
	} else {
		if r.doCheck(svr) {
			status = metapb.Up
		} else {
			status = metapb.Down
		}
	}

	if prev != status {
		r.watchEventC <- &store.Evt{
			Src:  eventSrcStatusChanged,
			Type: eventTypeStatusChanged,
			Value: statusChanged{
				meta:   *svr.meta,
				status: status,
			},
		}
	}
}

func (r *dispatcher) doCheck(svr *serverRuntime) bool {
	req := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(req)

	req.SetRequestURI(svr.getCheckURL())

	opt := util.DefaultHTTPOption()
	opt.ReadTimeout = time.Duration(svr.meta.HeathCheck.Timeout)
	resp, err := r.httpClient.Do(req, svr.meta.Addr, opt)
	defer fasthttp.ReleaseResponse(resp)
	if err != nil {
		log.Warnf("server <%d, %s, %d> check failed, errors:\n%+v",
			svr.meta.ID,
			svr.meta.HeathCheck.Path,
			svr.checkFailCount+1,
			err)
		svr.fail()
		return false
	}

	if fasthttp.StatusOK != resp.StatusCode() {
		log.Warnf("server <%d, %s, %d, %d> check failed",
			svr.meta.ID,
			svr.meta.HeathCheck.Path,
			resp.StatusCode(),
			svr.checkFailCount+1)
		svr.fail()
		return false
	}

	if svr.meta.HeathCheck.Body != "" &&
		svr.meta.HeathCheck.Body != string(resp.Body()) {
		log.Warnf("server <%s, %s, %d> check failed, body <%s>, expect <%s>",
			svr.meta.Addr,
			svr.meta.HeathCheck.Path,
			svr.checkFailCount+1,
			resp.Body(),
			svr.meta.HeathCheck.Body)
		svr.fail()
		return false
	}

	svr.reset()
	return true
}
<commit_msg>dev: global http options in check<commit_after>package proxy

import (
	"context"
	"time"

	"github.com/fagongzi/gateway/pkg/pb/metapb"
	"github.com/fagongzi/gateway/pkg/store"
	"github.com/fagongzi/gateway/pkg/util"
	"github.com/fagongzi/log"
	"github.com/valyala/fasthttp"
)

func (r *dispatcher) readyToHeathChecker() {
	for i := 0; i < r.cnf.Option.LimitCountHeathCheckWorker; i++ {
		r.runner.RunCancelableTask(func(ctx context.Context) {
			log.Infof("start server check worker")

			for {
				select {
				case <-ctx.Done():
					return
				case id := <-r.checkerC:
					r.check(id)
				}
			}
		})
	}
}

func (r *dispatcher) addToCheck(svr *serverRuntime) {
	svr.circuit = metapb.Open
	if svr.meta.HeathCheck != nil {
		svr.useCheckDuration = time.Duration(svr.meta.HeathCheck.CheckInterval)
	}
	svr.heathTimeout.Stop()
	r.checkerC <- svr.meta.ID
}

func (r *dispatcher) heathCheckTimeout(arg interface{}) {
	id := arg.(uint64)
	if _, ok := r.servers[id]; ok {
		r.checkerC <- id
	}
}

func (r *dispatcher) check(id uint64) {
	svr, ok := r.servers[id]
	if !ok {
		return
	}

	defer func() {
		if svr.meta.HeathCheck != nil {
			if svr.useCheckDuration > r.cnf.Option.LimitIntervalHeathCheck {
				svr.useCheckDuration = r.cnf.Option.LimitIntervalHeathCheck
			}
			svr.heathTimeout, _ = r.tw.Schedule(svr.useCheckDuration, r.heathCheckTimeout, id)
		}
	}()

	status := metapb.Unknown
	prev := r.getServerStatus(svr.meta.ID)

	if svr.meta.HeathCheck == nil {
		log.Warnf("server <%d> heath check not setting", svr.meta.ID)
		r.watchEventC <- &store.Evt{
			Src:  eventSrcStatusChanged,
			Type: eventTypeStatusChanged,
			Value: statusChanged{
				meta:   *svr.meta,
				status: metapb.Up,
			},
		}
	} else {
		if r.doCheck(svr) {
			status = metapb.Up
		} else {
			status = metapb.Down
		}
	}

	if prev != status {
		r.watchEventC <- &store.Evt{
			Src:  eventSrcStatusChanged,
			Type: eventTypeStatusChanged,
			Value: statusChanged{
				meta:   *svr.meta,
				status: status,
			},
		}
	}
}

func (r *dispatcher) doCheck(svr *serverRuntime) bool {
	req := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(req)

	req.SetRequestURI(svr.getCheckURL())

	opt := util.DefaultHTTPOption()
	*opt = *globalHTTPOptions
	opt.ReadTimeout = time.Duration(svr.meta.HeathCheck.Timeout)

	resp, err := r.httpClient.Do(req, svr.meta.Addr, opt)
	defer fasthttp.ReleaseResponse(resp)
	if err != nil {
		log.Warnf("server <%d, %s, %d> check failed, errors:\n%+v",
			svr.meta.ID,
			svr.getCheckURL(),
			svr.checkFailCount+1,
			err)
		svr.fail()
		return false
	}

	if fasthttp.StatusOK != resp.StatusCode() {
		log.Warnf("server <%d, %s, %d, %d> check failed",
			svr.meta.ID,
			svr.getCheckURL(),
			resp.StatusCode(),
			svr.checkFailCount+1)
		svr.fail()
		return false
	}

	if svr.meta.HeathCheck.Body != "" &&
		svr.meta.HeathCheck.Body != string(resp.Body()) {
		log.Warnf("server <%s, %s, %d> check failed, body <%s>, expect <%s>",
			svr.meta.Addr,
			svr.getCheckURL(),
			svr.checkFailCount+1,
			resp.Body(),
			svr.meta.HeathCheck.Body)
		svr.fail()
		return false
	}

	svr.reset()
	return true
}
<|endoftext|>