<commit_before>package build

import (
	"fmt"
	"neon/util"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"time"

	"gopkg.in/yaml.v2"
)

const (
	// DefaultRepo is local repository root directory
	DefaultRepo = "~/.neon"
	// RegexpPlugin is regexp for a plugin name
	RegexpPlugin = `[\w-]+/[\w-]+`
)

// Fields is the list of possible root fields for a build file
var Fields = []string{"doc", "default", "extends", "repository", "context",
	"singleton", "shell", "properties", "configuration", "environment", "targets"}

// Build structure
type Build struct {
	File        string
	Dir         string
	Here        string
	Default     []string
	Doc         string
	Repository  string
	Singleton   string
	Shell       map[string][]string
	Scripts     []string
	Extends     []string
	Config      []string
	Properties  util.Object
	Environment map[string]string
	Targets     map[string]*Target
	Parents     []*Build
}

// NewBuild makes a build from a build file
// - file: path of the build file
// Return:
// - Pointer to the build
// - error if something went wrong
func NewBuild(file string) (*Build, error) {
	object, build, err := parseBuildFile(file)
	if err != nil {
		return nil, err
	}
	if err := object.CheckFields(Fields); err != nil {
		return nil, fmt.Errorf("parsing build file: %v", err)
	}
	if err := ParseSingleton(object, build); err != nil {
		return nil, err
	}
	if err := ParseShell(object, build); err != nil {
		return nil, err
	}
	if err := ParseDefault(object, build); err != nil {
		return nil, err
	}
	if err := ParseDoc(object, build); err != nil {
		return nil, err
	}
	if err := ParseRepository(object, build); err != nil {
		return nil, err
	}
	if err := ParseContext(object, build); err != nil {
		return nil, err
	}
	if err := ParseExtends(object, build); err != nil {
		return nil, err
	}
	if err := ParseProperties(object, build); err != nil {
		return nil, err
	}
	if err := ParseConfiguration(object, build); err != nil {
		return nil, err
	}
	if err := ParseEnvironment(object, build); err != nil {
		return nil, err
	}
	if err := ParseTargets(object, build); err != nil {
		return nil, err
	}
	build.Properties = build.GetProperties()
	build.Environment = build.GetEnvironment()
	build.SetDir(build.Dir)
	return build, nil
}

func parseBuildFile(file string) (util.Object, *Build, error) {
	build := &Build{}
	file = util.ExpandUserHome(file)
	build.File = filepath.Base(file)
	base, err := filepath.Abs(filepath.Dir(file))
	if err != nil {
		return nil, nil, fmt.Errorf("getting build file directory: %v", err)
	}
	build.Dir = base
	here, err := os.Getwd()
	if err != nil {
		return nil, nil, fmt.Errorf("getting current directory: %v", err)
	}
	build.Here = here
	source, err := util.ReadFile(file)
	if err != nil {
		return nil, nil, fmt.Errorf("loading build file '%s': %v", file, err)
	}
	var object util.Object
	if err = yaml.Unmarshal(source, &object); err != nil {
		return nil, nil, fmt.Errorf("build must be a map with string keys: %v", err)
	}
	return object, build, nil
}

// GetProperties returns build properties, including those inherited from
// parents
// Return: build properties as an Object
func (build *Build) GetProperties() util.Object {
	var properties = make(map[string]interface{})
	for _, parent := range build.Parents {
		for name, value := range parent.GetProperties() {
			properties[name] = value
		}
	}
	for name, value := range build.Properties {
		properties[name] = value
	}
	return properties
}

// GetEnvironment returns the build environment, including the environment
// inherited from parents
// Return: environment as a map with string keys and values
func (build *Build) GetEnvironment() map[string]string {
	var environment = make(map[string]string)
	for _, parent := range build.Parents {
		for name, value := range parent.GetEnvironment() {
			environment[name] = value
		}
	}
	for name, value := range build.Environment {
		environment[name] = value
	}
	return environment
}

// GetTargets returns build targets, including those inherited from parents
// Return: targets as a map of targets with their name as keys
func (build *Build) GetTargets() map[string]*Target {
	var targets = make(map[string]*Target)
	for _, parent := range build.Parents {
		for name, target := range parent.GetTargets() {
			targets[name] = target
		}
	}
	for name, target := range build.Targets {
		targets[name] = target
	}
	return targets
}

// GetTargetByName return target with given name. If not defined in build,
// return target inherited from parent
// - name: the target name as a string
// Return: found target
func (build *Build) GetTargetByName(name string) *Target {
	target, found := build.Targets[name]
	if found {
		return target
	}
	for _, parent := range build.Parents {
		target = parent.GetTargetByName(name)
		if target != nil {
			return target
		}
	}
	return nil
}

// SetDir sets the build directory, propagating to parents
// - dir: build directory as a string
func (build *Build) SetDir(dir string) {
	build.Dir = dir
	for _, parent := range build.Parents {
		parent.SetDir(dir)
	}
}

// SetCommandLineProperties defines properties passed on command line in the
// context. These properties overwrite those define in the build file.
// - props: properties as a YAML map
// Return: error if something went wrong
func (build *Build) SetCommandLineProperties(props string) error {
	var object util.Object
	err := yaml.Unmarshal([]byte(props), &object)
	if err != nil {
		return fmt.Errorf("parsing command line properties: properties must be a map with string keys")
	}
	for name, value := range object {
		build.Properties[name] = value
	}
	return nil
}

// GetDefault returns default targets. If none is defined in build, return
// those from parent build files.
// Return: default targets a slice of strings
func (build *Build) GetDefault() []string {
	if len(build.Default) > 0 {
		return build.Default
	}
	for _, parent := range build.Parents {
		if len(parent.Default) > 0 {
			return parent.Default
		}
	}
	for _, parent := range build.Parents {
		parentDefault := parent.GetDefault()
		if len(parentDefault) > 0 {
			return parentDefault
		}
	}
	return build.Default
}

// Run runs given targets in a build context. If no target is given, runs
// default one.
// - context: the context to run into
// - targets: targets to run as a slice of strings
// Return: error if something went wrong
func (build *Build) Run(context *Context, targets []string) error {
	if err := build.EnsureSingle(context); err != nil {
		return err
	}
	if len(targets) == 0 {
		targets = build.GetDefault()
		if len(targets) == 0 {
			return fmt.Errorf("no default target")
		}
	}
	for _, target := range targets {
		context.Stack = NewStack()
		err := build.RunTarget(context, target)
		if err != nil {
			return err
		}
	}
	return nil
}

// RunTarget runs given target in a build context.
// - context: build context
// - name: name of the target to run as a string
// Return: an error if something went wrong
func (build *Build) RunTarget(context *Context, name string) error {
	target := build.GetTargetByName(name)
	if target == nil {
		return fmt.Errorf("target '%s' not found", name)
	}
	err := target.Run(context)
	if err != nil {
		return fmt.Errorf("running target '%s': %v", name, err)
	}
	return nil
}

// RunParentTarget runs parent target with given name in a build context.
// - context: build context
// - name: the name of the target to run
// Return:
// - boolean: that tells if parent target was found
// - error: if something went wrong
func (build *Build) RunParentTarget(context *Context, name string) (bool, error) {
	for _, parent := range build.Parents {
		target := parent.GetTargetByName(name)
		if target != nil {
			err := context.Stack.Push(target)
			if err != nil {
				return false, err
			}
			err = target.Steps.Run(context)
			if err != nil {
				return true, fmt.Errorf("running target '%s': %v", name, err)
			}
			return true, nil
		}
	}
	return false, nil
}

// GetShell return shell for current os.
// Return:
// - shell as a slice of strings (such as ["sh", "-c"])
// - error if something went wrong
func (build *Build) GetShell() ([]string, error) {
	for system, shell := range build.Shell {
		if system != "default" && system == runtime.GOOS {
			return shell, nil
		}
	}
	shell, ok := build.Shell["default"]
	if !ok {
		return nil, fmt.Errorf("no shell found for '%s'", runtime.GOOS)
	}
	return shell, nil
}

// EnsureSingle runs a TCP server on given port to ensure that a single
// instance is running on a machine. Fails if another instance is already
// running on same port.
// - context: build context
// Return: an error if another instance is running on same port
func (build *Build) EnsureSingle(context *Context) error {
	if build.Singleton == "" {
		return nil
	}
	expression := build.Singleton
	if IsExpression(expression) {
		expression = expression[1:]
	}
	singleton, err := context.EvaluateExpression(expression)
	if err != nil {
		return fmt.Errorf("evaluating singleton port expression '%s': %v", expression, err)
	}
	port, ok := singleton.(int64)
	if !ok {
		return fmt.Errorf("singleton port expression '%s' must return an integer", expression)
	}
	if port < 0 || port > 65535 {
		return fmt.Errorf("singleton port port must be between 0 and 65535")
	}
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		return fmt.Errorf("listening singleton port: %v", err)
	}
	go func() {
		for {
			listener.Accept()
			time.Sleep(100 * time.Millisecond)
		}
	}()
	return nil
}
<commit_msg>Release 0.12.0: Added template and parent discovery<commit_after>package build

import (
	"fmt"
	"neon/util"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"time"

	"gopkg.in/yaml.v2"
)

const (
	// DefaultRepo is local repository root directory
	DefaultRepo = "~/.neon"
	// RegexpPlugin is regexp for a plugin name
	RegexpPlugin = `[\w-]+/[\w-]+`
)

// Fields is the list of possible root fields for a build file
var Fields = []string{"doc", "default", "extends", "repository", "context",
	"singleton", "shell", "properties", "configuration", "environment", "targets"}

// Build structure
type Build struct {
	File        string
	Dir         string
	Here        string
	Default     []string
	Doc         string
	Repository  string
	Singleton   string
	Shell       map[string][]string
	Scripts     []string
	Extends     []string
	Config      []string
	Properties  util.Object
	Environment map[string]string
	Targets     map[string]*Target
	Parents     []*Build
}

// NewBuild makes a build from a build file
// - file: path of the build file
// Return:
// - Pointer to the build
// - error if something went wrong
func NewBuild(file string) (*Build, error) {
	object, build, err := parseBuildFile(file)
	if err != nil {
		return nil, err
	}
	if err := object.CheckFields(Fields); err != nil {
		return nil, fmt.Errorf("parsing build file: %v", err)
	}
	if err := ParseSingleton(object, build); err != nil {
		return nil, err
	}
	if err := ParseShell(object, build); err != nil {
		return nil, err
	}
	if err := ParseDefault(object, build); err != nil {
		return nil, err
	}
	if err := ParseDoc(object, build); err != nil {
		return nil, err
	}
	if err := ParseRepository(object, build); err != nil {
		return nil, err
	}
	if err := ParseContext(object, build); err != nil {
		return nil, err
	}
	if err := ParseExtends(object, build); err != nil {
		return nil, err
	}
	if err := ParseProperties(object, build); err != nil {
		return nil, err
	}
	if err := ParseConfiguration(object, build); err != nil {
		return nil, err
	}
	if err := ParseEnvironment(object, build); err != nil {
		return nil, err
	}
	if err := ParseTargets(object, build); err != nil {
		return nil, err
	}
	build.Properties = build.GetProperties()
	build.Environment = build.GetEnvironment()
	build.SetDir(build.Dir)
	return build, nil
}

func parseBuildFile(file string) (util.Object, *Build, error) {
	build := &Build{}
	file = util.ExpandUserHome(file)
	build.File = filepath.Base(file)
	base, err := filepath.Abs(filepath.Dir(file))
	if err != nil {
		return nil, nil, fmt.Errorf("getting build file directory: %v", err)
	}
	build.Dir = base
	here, err := os.Getwd()
	if err != nil {
		return nil, nil, fmt.Errorf("getting current directory: %v", err)
	}
	build.Here = here
	source, err := util.ReadFile(file)
	if err != nil {
		return nil, nil, fmt.Errorf("loading build file '%s': %v", file, err)
	}
	var object util.Object
	if err = yaml.Unmarshal(source, &object); err != nil {
		return nil, nil, fmt.Errorf("build must be a map with string keys: %v", err)
	}
	return object, build, nil
}

// GetProperties returns build properties, including those inherited from
// parents
// Return: build properties as an Object
func (build *Build) GetProperties() util.Object {
	var properties = make(map[string]interface{})
	for _, parent := range build.Parents {
		for name, value := range parent.GetProperties() {
			properties[name] = value
		}
	}
	for name, value := range build.Properties {
		properties[name] = value
	}
	return properties
}

// GetEnvironment returns the build environment, including the environment
// inherited from parents
// Return: environment as a map with string keys and values
func (build *Build) GetEnvironment() map[string]string {
	var environment = make(map[string]string)
	for _, parent := range build.Parents {
		for name, value := range parent.GetEnvironment() {
			environment[name] = value
		}
	}
	for name, value := range build.Environment {
		environment[name] = value
	}
	return environment
}

// GetTargets returns build targets, including those inherited from parents
// Return: targets as a map of targets with their name as keys
func (build *Build) GetTargets() map[string]*Target {
	var targets = make(map[string]*Target)
	for _, parent := range build.Parents {
		for name, target := range parent.GetTargets() {
			targets[name] = target
		}
	}
	for name, target := range build.Targets {
		targets[name] = target
	}
	return targets
}

// GetTargetByName return target with given name. If not defined in build,
// return target inherited from parent
// - name: the target name as a string
// Return: found target
func (build *Build) GetTargetByName(name string) *Target {
	target, found := build.Targets[name]
	if found {
		return target
	}
	for _, parent := range build.Parents {
		target = parent.GetTargetByName(name)
		if target != nil {
			return target
		}
	}
	return nil
}

// SetDir sets the build directory, propagating to parents
// - dir: build directory as a string
func (build *Build) SetDir(dir string) {
	build.Dir = dir
	for _, parent := range build.Parents {
		parent.SetDir(dir)
	}
}

// SetCommandLineProperties defines properties passed on command line in the
// context. These properties overwrite those define in the build file.
// - props: properties as a YAML map
// Return: error if something went wrong
func (build *Build) SetCommandLineProperties(props string) error {
	var object util.Object
	err := yaml.Unmarshal([]byte(props), &object)
	if err != nil {
		return fmt.Errorf("parsing command line properties: properties must be a map with string keys")
	}
	for name, value := range object {
		build.Properties[name] = value
	}
	return nil
}

// GetDefault returns default targets. If none is defined in build, return
// those from parent build files.
// Return: default targets a slice of strings
func (build *Build) GetDefault() []string {
	if len(build.Default) > 0 {
		return build.Default
	}
	for _, parent := range build.Parents {
		if len(parent.Default) > 0 {
			return parent.Default
		}
	}
	for _, parent := range build.Parents {
		parentDefault := parent.GetDefault()
		if len(parentDefault) > 0 {
			return parentDefault
		}
	}
	return build.Default
}

// Run runs given targets in a build context. If no target is given, runs
// default one.
// - context: the context to run into
// - targets: targets to run as a slice of strings
// Return: error if something went wrong
func (build *Build) Run(context *Context, targets []string) error {
	if err := build.EnsureSingle(context); err != nil {
		return err
	}
	if len(targets) == 0 {
		targets = build.GetDefault()
		if len(targets) == 0 {
			return fmt.Errorf("no default target")
		}
	}
	for _, target := range targets {
		context.Stack = NewStack()
		err := build.RunTarget(context, target)
		if err != nil {
			return err
		}
	}
	return nil
}

// RunTarget runs given target in a build context.
// - context: build context
// - name: name of the target to run as a string
// Return: an error if something went wrong
func (build *Build) RunTarget(context *Context, name string) error {
	target := build.GetTargetByName(name)
	if target == nil {
		return fmt.Errorf("target '%s' not found", name)
	}
	err := target.Run(context)
	if err != nil {
		return fmt.Errorf("running target '%s': %v", name, err)
	}
	return nil
}

// RunParentTarget runs parent target with given name in a build context.
// - context: build context
// - name: the name of the target to run
// Return:
// - boolean: that tells if parent target was found
// - error: if something went wrong
func (build *Build) RunParentTarget(context *Context, name string) (bool, error) {
	for _, parent := range build.Parents {
		target := parent.GetTargetByName(name)
		if target != nil {
			err := context.Stack.Push(target)
			if err != nil {
				return false, err
			}
			err = target.Steps.Run(context)
			if err != nil {
				return true, fmt.Errorf("running target '%s': %v", name, err)
			}
			return true, nil
		}
	}
	return false, nil
}

// GetShell return shell for current os.
// Return:
// - shell as a slice of strings (such as ["sh", "-c"])
// - error if something went wrong
func (build *Build) GetShell() ([]string, error) {
	for system, shell := range build.Shell {
		if system != "default" && system == runtime.GOOS {
			return shell, nil
		}
	}
	shell, ok := build.Shell["default"]
	if !ok {
		return nil, fmt.Errorf("no shell found for '%s'", runtime.GOOS)
	}
	return shell, nil
}

// EnsureSingle runs a TCP server on given port to ensure that a single
// instance is running on a machine. Fails if another instance is already
// running on same port.
// - context: build context
// Return: an error if another instance is running on same port
func (build *Build) EnsureSingle(context *Context) error {
	if build.Singleton == "" {
		return nil
	}
	expression := build.Singleton
	if IsExpression(expression) {
		expression = expression[1:]
	}
	singleton, err := context.EvaluateExpression(expression)
	if err != nil {
		return fmt.Errorf("evaluating singleton port expression '%s': %v", expression, err)
	}
	port, ok := singleton.(int64)
	if !ok {
		portInt, ok := singleton.(int)
		if !ok {
			return fmt.Errorf("singleton port expression '%s' must return an integer", expression)
		}
		port = int64(portInt)
	}
	if port < 0 || port > 65535 {
		return fmt.Errorf("singleton port port must be between 0 and 65535")
	}
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		return fmt.Errorf("listening singleton port: %v", err)
	}
	go func() {
		for {
			listener.Accept()
			time.Sleep(100 * time.Millisecond)
		}
	}()
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"internal/syscall/windows"
	"sync"
	"syscall"
	"time"
	"unsafe"
)

// A fileStat is the implementation of FileInfo returned by Stat and Lstat.
type fileStat struct {
	name string

	// from ByHandleFileInformation, Win32FileAttributeData and Win32finddata
	FileAttributes uint32
	CreationTime   syscall.Filetime
	LastAccessTime syscall.Filetime
	LastWriteTime  syscall.Filetime
	FileSizeHigh   uint32
	FileSizeLow    uint32

	// from Win32finddata
	Reserved0 uint32

	// what syscall.GetFileType returns
	filetype uint32

	// used to implement SameFile
	sync.Mutex
	path             string
	vol              uint32
	idxhi            uint32
	idxlo            uint32
	appendNameToPath bool
}

// newFileStatFromGetFileInformationByHandle calls GetFileInformationByHandle
// to gather all required information about the file handle h.
func newFileStatFromGetFileInformationByHandle(path string, h syscall.Handle) (fs *fileStat, err error) {
	var d syscall.ByHandleFileInformation
	err = syscall.GetFileInformationByHandle(h, &d)
	if err != nil {
		return nil, &PathError{"GetFileInformationByHandle", path, err}
	}

	var ti windows.FILE_ATTRIBUTE_TAG_INFO
	err = windows.GetFileInformationByHandleEx(h, windows.FileAttributeTagInfo, (*byte)(unsafe.Pointer(&ti)), uint32(unsafe.Sizeof(ti)))
	if err != nil {
		return nil, &PathError{"GetFileInformationByHandleEx", path, err}
	}

	return &fileStat{
		name:           basename(path),
		FileAttributes: d.FileAttributes,
		CreationTime:   d.CreationTime,
		LastAccessTime: d.LastAccessTime,
		LastWriteTime:  d.LastWriteTime,
		FileSizeHigh:   d.FileSizeHigh,
		FileSizeLow:    d.FileSizeLow,
		vol:            d.VolumeSerialNumber,
		idxhi:          d.FileIndexHigh,
		idxlo:          d.FileIndexLow,
		Reserved0:      ti.ReparseTag,
		// fileStat.path is used by os.SameFile to decide if it needs
		// to fetch vol, idxhi and idxlo. But these are already set,
		// so set fileStat.path to "" to prevent os.SameFile doing it again.
	}, nil
}

// newFileStatFromWin32finddata copies all required information
// from syscall.Win32finddata d into the newly created fileStat.
func newFileStatFromWin32finddata(d *syscall.Win32finddata) *fileStat {
	return &fileStat{
		FileAttributes: d.FileAttributes,
		CreationTime:   d.CreationTime,
		LastAccessTime: d.LastAccessTime,
		LastWriteTime:  d.LastWriteTime,
		FileSizeHigh:   d.FileSizeHigh,
		FileSizeLow:    d.FileSizeLow,
		Reserved0:      d.Reserved0,
	}
}

func (fs *fileStat) isSymlink() bool {
	// Use instructions described at
	// https://blogs.msdn.microsoft.com/oldnewthing/20100212-00/?p=14963/
	// to recognize whether it's a symlink.
	if fs.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 {
		return false
	}
	return fs.Reserved0 == syscall.IO_REPARSE_TAG_SYMLINK ||
		fs.Reserved0 == windows.IO_REPARSE_TAG_MOUNT_POINT
}

func (fs *fileStat) Size() int64 {
	return int64(fs.FileSizeHigh)<<32 + int64(fs.FileSizeLow)
}

func (fs *fileStat) Mode() (m FileMode) {
	if fs == &devNullStat {
		return ModeDevice | ModeCharDevice | 0666
	}
	if fs.FileAttributes&syscall.FILE_ATTRIBUTE_READONLY != 0 {
		m |= 0444
	} else {
		m |= 0666
	}
	if fs.isSymlink() {
		return m | ModeSymlink
	}
	if fs.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
		m |= ModeDir | 0111
	}
	switch fs.filetype {
	case syscall.FILE_TYPE_PIPE:
		m |= ModeNamedPipe
	case syscall.FILE_TYPE_CHAR:
		m |= ModeDevice | ModeCharDevice
	}
	return m
}

func (fs *fileStat) ModTime() time.Time {
	return time.Unix(0, fs.LastWriteTime.Nanoseconds())
}

// Sys returns syscall.Win32FileAttributeData for file fs.
func (fs *fileStat) Sys() interface{} {
	return &syscall.Win32FileAttributeData{
		FileAttributes: fs.FileAttributes,
		CreationTime:   fs.CreationTime,
		LastAccessTime: fs.LastAccessTime,
		LastWriteTime:  fs.LastWriteTime,
		FileSizeHigh:   fs.FileSizeHigh,
		FileSizeLow:    fs.FileSizeLow,
	}
}

func (fs *fileStat) loadFileId() error {
	fs.Lock()
	defer fs.Unlock()
	if fs.path == "" {
		// already done
		return nil
	}
	var path string
	if fs.appendNameToPath {
		path = fs.path + `\` + fs.name
	} else {
		path = fs.path
	}
	pathp, err := syscall.UTF16PtrFromString(path)
	if err != nil {
		return err
	}
	attrs := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS)
	if fs.isSymlink() {
		// Use FILE_FLAG_OPEN_REPARSE_POINT, otherwise CreateFile will follow symlink.
		// See https://docs.microsoft.com/en-us/windows/desktop/FileIO/symbolic-link-effects-on-file-systems-functions#createfile-and-createfiletransacted
		attrs |= syscall.FILE_FLAG_OPEN_REPARSE_POINT
	}
	h, err := syscall.CreateFile(pathp, 0, 0, nil, syscall.OPEN_EXISTING, attrs, 0)
	if err != nil {
		return err
	}
	defer syscall.CloseHandle(h)
	var i syscall.ByHandleFileInformation
	err = syscall.GetFileInformationByHandle(h, &i)
	if err != nil {
		return err
	}
	fs.path = ""
	fs.vol = i.VolumeSerialNumber
	fs.idxhi = i.FileIndexHigh
	fs.idxlo = i.FileIndexLow
	return nil
}

// devNullStat is fileStat structure describing DevNull file ("NUL").
var devNullStat = fileStat{
	name: DevNull,
	// hopefully this will work for SameFile
	vol:   0,
	idxhi: 0,
	idxlo: 0,
}

func sameFile(fs1, fs2 *fileStat) bool {
	e := fs1.loadFileId()
	if e != nil {
		return false
	}
	e = fs2.loadFileId()
	if e != nil {
		return false
	}
	return fs1.vol == fs2.vol && fs1.idxhi == fs2.idxhi && fs1.idxlo == fs2.idxlo
}

// For testing.
func atime(fi FileInfo) time.Time {
	return time.Unix(0, fi.Sys().(*syscall.Win32FileAttributeData).LastAccessTime.Nanoseconds())
}
<commit_msg>os: make Stat work on FAT file system<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"internal/syscall/windows"
	"sync"
	"syscall"
	"time"
	"unsafe"
)

// A fileStat is the implementation of FileInfo returned by Stat and Lstat.
type fileStat struct {
	name string

	// from ByHandleFileInformation, Win32FileAttributeData and Win32finddata
	FileAttributes uint32
	CreationTime   syscall.Filetime
	LastAccessTime syscall.Filetime
	LastWriteTime  syscall.Filetime
	FileSizeHigh   uint32
	FileSizeLow    uint32

	// from Win32finddata
	Reserved0 uint32

	// what syscall.GetFileType returns
	filetype uint32

	// used to implement SameFile
	sync.Mutex
	path             string
	vol              uint32
	idxhi            uint32
	idxlo            uint32
	appendNameToPath bool
}

// newFileStatFromGetFileInformationByHandle calls GetFileInformationByHandle
// to gather all required information about the file handle h.
func newFileStatFromGetFileInformationByHandle(path string, h syscall.Handle) (fs *fileStat, err error) {
	var d syscall.ByHandleFileInformation
	err = syscall.GetFileInformationByHandle(h, &d)
	if err != nil {
		return nil, &PathError{"GetFileInformationByHandle", path, err}
	}

	var ti windows.FILE_ATTRIBUTE_TAG_INFO
	err = windows.GetFileInformationByHandleEx(h, windows.FileAttributeTagInfo, (*byte)(unsafe.Pointer(&ti)), uint32(unsafe.Sizeof(ti)))
	if err != nil {
		if errno, ok := err.(syscall.Errno); ok && errno == windows.ERROR_INVALID_PARAMETER {
			// It appears calling GetFileInformationByHandleEx with
			// FILE_ATTRIBUTE_TAG_INFO fails on FAT file system with
			// ERROR_INVALID_PARAMETER. Clear ti.ReparseTag in that
			// instance to indicate no symlinks are possible.
			ti.ReparseTag = 0
		} else {
			return nil, &PathError{"GetFileInformationByHandleEx", path, err}
		}
	}

	return &fileStat{
		name:           basename(path),
		FileAttributes: d.FileAttributes,
		CreationTime:   d.CreationTime,
		LastAccessTime: d.LastAccessTime,
		LastWriteTime:  d.LastWriteTime,
		FileSizeHigh:   d.FileSizeHigh,
		FileSizeLow:    d.FileSizeLow,
		vol:            d.VolumeSerialNumber,
		idxhi:          d.FileIndexHigh,
		idxlo:          d.FileIndexLow,
		Reserved0:      ti.ReparseTag,
		// fileStat.path is used by os.SameFile to decide if it needs
		// to fetch vol, idxhi and idxlo. But these are already set,
		// so set fileStat.path to "" to prevent os.SameFile doing it again.
	}, nil
}

// newFileStatFromWin32finddata copies all required information
// from syscall.Win32finddata d into the newly created fileStat.
func newFileStatFromWin32finddata(d *syscall.Win32finddata) *fileStat {
	return &fileStat{
		FileAttributes: d.FileAttributes,
		CreationTime:   d.CreationTime,
		LastAccessTime: d.LastAccessTime,
		LastWriteTime:  d.LastWriteTime,
		FileSizeHigh:   d.FileSizeHigh,
		FileSizeLow:    d.FileSizeLow,
		Reserved0:      d.Reserved0,
	}
}

func (fs *fileStat) isSymlink() bool {
	// Use instructions described at
	// https://blogs.msdn.microsoft.com/oldnewthing/20100212-00/?p=14963/
	// to recognize whether it's a symlink.
	if fs.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 {
		return false
	}
	return fs.Reserved0 == syscall.IO_REPARSE_TAG_SYMLINK ||
		fs.Reserved0 == windows.IO_REPARSE_TAG_MOUNT_POINT
}

func (fs *fileStat) Size() int64 {
	return int64(fs.FileSizeHigh)<<32 + int64(fs.FileSizeLow)
}

func (fs *fileStat) Mode() (m FileMode) {
	if fs == &devNullStat {
		return ModeDevice | ModeCharDevice | 0666
	}
	if fs.FileAttributes&syscall.FILE_ATTRIBUTE_READONLY != 0 {
		m |= 0444
	} else {
		m |= 0666
	}
	if fs.isSymlink() {
		return m | ModeSymlink
	}
	if fs.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
		m |= ModeDir | 0111
	}
	switch fs.filetype {
	case syscall.FILE_TYPE_PIPE:
		m |= ModeNamedPipe
	case syscall.FILE_TYPE_CHAR:
		m |= ModeDevice | ModeCharDevice
	}
	return m
}

func (fs *fileStat) ModTime() time.Time {
	return time.Unix(0, fs.LastWriteTime.Nanoseconds())
}

// Sys returns syscall.Win32FileAttributeData for file fs.
func (fs *fileStat) Sys() interface{} {
	return &syscall.Win32FileAttributeData{
		FileAttributes: fs.FileAttributes,
		CreationTime:   fs.CreationTime,
		LastAccessTime: fs.LastAccessTime,
		LastWriteTime:  fs.LastWriteTime,
		FileSizeHigh:   fs.FileSizeHigh,
		FileSizeLow:    fs.FileSizeLow,
	}
}

func (fs *fileStat) loadFileId() error {
	fs.Lock()
	defer fs.Unlock()
	if fs.path == "" {
		// already done
		return nil
	}
	var path string
	if fs.appendNameToPath {
		path = fs.path + `\` + fs.name
	} else {
		path = fs.path
	}
	pathp, err := syscall.UTF16PtrFromString(path)
	if err != nil {
		return err
	}
	attrs := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS)
	if fs.isSymlink() {
		// Use FILE_FLAG_OPEN_REPARSE_POINT, otherwise CreateFile will follow symlink.
		// See https://docs.microsoft.com/en-us/windows/desktop/FileIO/symbolic-link-effects-on-file-systems-functions#createfile-and-createfiletransacted
		attrs |= syscall.FILE_FLAG_OPEN_REPARSE_POINT
	}
	h, err := syscall.CreateFile(pathp, 0, 0, nil, syscall.OPEN_EXISTING, attrs, 0)
	if err != nil {
		return err
	}
	defer syscall.CloseHandle(h)
	var i syscall.ByHandleFileInformation
	err = syscall.GetFileInformationByHandle(h, &i)
	if err != nil {
		return err
	}
	fs.path = ""
	fs.vol = i.VolumeSerialNumber
	fs.idxhi = i.FileIndexHigh
	fs.idxlo = i.FileIndexLow
	return nil
}

// devNullStat is fileStat structure describing DevNull file ("NUL").
var devNullStat = fileStat{
	name: DevNull,
	// hopefully this will work for SameFile
	vol:   0,
	idxhi: 0,
	idxlo: 0,
}

func sameFile(fs1, fs2 *fileStat) bool {
	e := fs1.loadFileId()
	if e != nil {
		return false
	}
	e = fs2.loadFileId()
	if e != nil {
		return false
	}
	return fs1.vol == fs2.vol && fs1.idxhi == fs2.idxhi && fs1.idxlo == fs2.idxlo
}

// For testing.
func atime(fi FileInfo) time.Time {
	return time.Unix(0, fi.Sys().(*syscall.Win32FileAttributeData).LastAccessTime.Nanoseconds())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"sync"
	"time"
)

// A MaintenanceDaemon is a structure to continually update the existence of a
// file depending on whether or not maintenance mode is enabled.
type MaintenanceDaemon struct {
	fullpath string
	ch       chan interface{}
	wg       *sync.WaitGroup
	rwm      *sync.RWMutex
}

func (md *MaintenanceDaemon) on() {
	_, err := os.Create(md.fullpath)
	if err != nil {
		log.Printf("maintenance: could not create maintenance file")
	}
}

func (md *MaintenanceDaemon) off() {
	err := os.Remove(md.fullpath)
	if err != nil && !os.IsNotExist(err) {
		log.Printf("maintenance: could not remove maintenance file")
	}
}

func (md *MaintenanceDaemon) update(value *float64) {
	// if there is no valid data, don't risk changing the maintenace state
	if value == nil {
		return
	}
	if *value > 0 {
		md.on()
	} else {
		md.off()
	}
}

// NewMaintenanceDaemon creates a daemon that controls the given file.
func NewMaintenanceDaemon(fullpath string, monitor Monitor, rollout Rollout) (*MaintenanceDaemon, error) {
	if fullpath == "" {
		return nil, fmt.Errorf("fullpath string is empty")
	}
	md := &MaintenanceDaemon{
		fullpath: fullpath,
		ch:       make(chan interface{}),
		wg:       &sync.WaitGroup{},
		rwm:      &sync.RWMutex{},
	}
	md.update(rollout.Get("maintenance"))
	go func() {
		tick := time.NewTicker(time.Second)
		defer tick.Stop()
		select {
		case <-md.ch:
			return
		case <-tick.C:
			md.update(rollout.Get("maintenance"))
		}
	}()
	return md, nil
}

// IsOn returns the status of maintenance (on or off).
func (md *MaintenanceDaemon) IsOn() bool {
	_, err := os.Stat(md.fullpath)
	return err == nil
}

// Stop stops the daemon forever.
func (md *MaintenanceDaemon) Stop() {
	md.rwm.Lock()
	if md.ch != nil {
		md.ch <- nil
	}
	md.ch = nil
	md.rwm.Unlock()
	if md.wg != nil {
		md.wg.Wait()
	}
}
<commit_msg>Fix maintenance falling out of its loop<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"sync"
	"time"
)

// A MaintenanceDaemon is a structure to continually update the existence of a
// file depending on whether or not maintenance mode is enabled.
type MaintenanceDaemon struct {
	fullpath string
	ch       chan interface{}
	wg       *sync.WaitGroup
	rwm      *sync.RWMutex
}

func (md *MaintenanceDaemon) on() {
	_, err := os.Create(md.fullpath)
	if err != nil {
		log.Printf("maintenance: could not create maintenance file")
	}
}

func (md *MaintenanceDaemon) off() {
	err := os.Remove(md.fullpath)
	if err != nil && !os.IsNotExist(err) {
		log.Printf("maintenance: could not remove maintenance file")
	}
}

func (md *MaintenanceDaemon) update(value *float64) {
	// if there is no valid data, don't risk changing the maintenace state
	if value == nil {
		return
	}
	if *value > 0 {
		md.on()
	} else {
		md.off()
	}
}

// NewMaintenanceDaemon creates a daemon that controls the given file.
func NewMaintenanceDaemon(fullpath string, monitor Monitor, rollout Rollout) (*MaintenanceDaemon, error) {
	if fullpath == "" {
		return nil, fmt.Errorf("fullpath string is empty")
	}
	md := &MaintenanceDaemon{
		fullpath: fullpath,
		ch:       make(chan interface{}),
		wg:       &sync.WaitGroup{},
		rwm:      &sync.RWMutex{},
	}
	md.update(rollout.Get("maintenance"))
	go func() {
		tick := time.NewTicker(time.Second)
		defer tick.Stop()
		for {
			select {
			case <-md.ch:
				return
			case <-tick.C:
				md.update(rollout.Get("maintenance"))
			}
		}
	}()
	return md, nil
}

// IsOn returns the status of maintenance (on or off).
func (md *MaintenanceDaemon) IsOn() bool {
	_, err := os.Stat(md.fullpath)
	return err == nil
}

// Stop stops the daemon forever.
func (md *MaintenanceDaemon) Stop() {
	md.rwm.Lock()
	if md.ch != nil {
		md.ch <- nil
	}
	md.ch = nil
	md.rwm.Unlock()
	if md.wg != nil {
		md.wg.Wait()
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel

import (
	"fmt"
	"sync"

	"golang.org/x/net/context"
)

// SubChannelOption are used to set options for subchannels.
type SubChannelOption func(*SubChannel)

// Isolated is a SubChannelOption that creates an isolated subchannel.
func Isolated(s *SubChannel) {
	s.Lock()
	s.peers = s.topChannel.peers.newSibling()
	s.peers.SetStrategy(newLeastPendingCalculator())
	s.Unlock()
}

// SubChannel allows calling a specific service on a channel.
// TODO(prashant): Allow creating a subchannel with default call options.
// TODO(prashant): Allow registering handlers on a subchannel.
type SubChannel struct {
	sync.RWMutex
	serviceName        string
	topChannel         *Channel
	defaultCallOptions *CallOptions
	peers              *PeerList
	handler            Handler
	logger             Logger
	statsReporter      StatsReporter
}

// Map of subchannel and the corresponding service
type subChannelMap struct {
	sync.RWMutex
	subchannels map[string]*SubChannel
}

func newSubChannel(serviceName string, ch *Channel) *SubChannel {
	logger := ch.Logger().WithFields(LogField{"subchannel", serviceName})
	return &SubChannel{
		serviceName:   serviceName,
		peers:         ch.peers,
		topChannel:    ch,
		handler:       &handlerMap{}, // use handlerMap by default
		logger:        logger,
		statsReporter: ch.StatsReporter(),
	}
}

// ServiceName returns the service name that this subchannel is for.
func (c *SubChannel) ServiceName() string {
	return c.serviceName
}

// BeginCall starts a new call to a remote peer, returning an OutboundCall that can
// be used to write the arguments of the call.
func (c *SubChannel) BeginCall(ctx context.Context, methodName string, callOptions *CallOptions) (*OutboundCall, error) {
	if callOptions == nil {
		callOptions = defaultCallOptions
	}

	peer, err := c.peers.Get(callOptions.RequestState.PrevSelectedPeers())
	if err != nil {
		return nil, err
	}

	return peer.BeginCall(ctx, c.ServiceName(), methodName, callOptions)
}

// Peers returns the PeerList for this subchannel.
func (c *SubChannel) Peers() *PeerList {
	return c.peers
}

// Isolated returns whether this subchannel is an isolated subchannel.
func (c *SubChannel) Isolated() bool {
	c.RLock()
	defer c.RUnlock()
	return c.topChannel.Peers() != c.peers
}

// Register registers a handler on the subchannel for the given method.
//
// This function panics if the Handler for the SubChannel was overwritten with
// SetHandler.
func (c *SubChannel) Register(h Handler, methodName string) {
	handlers, ok := c.handler.(*handlerMap)
	if !ok {
		panic(fmt.Sprintf(
			"handler for SubChannel(%v) was changed to disallow method registration", c.ServiceName,
		))
	}
	handlers.register(h, methodName)
}

// SetHandler changes the SubChannel's underlying handler. This may be used to
// set up a catch-all Handler for all requests received by this SubChannel.
//
// Methods registered on this SubChannel using Register() before calling
// SetHandler() will be forgotten. Further calls to Register() on this
// SubChannel after SetHandler() is called will cause panics.
func (c *SubChannel) SetHandler(h Handler) {
	c.Lock()
	c.handler = h
	c.Unlock()
}

// Logger returns the logger for this subchannel.
func (c *SubChannel) Logger() Logger {
	return c.logger
}

// StatsReporter returns the stats reporter for this subchannel.
func (c *SubChannel) StatsReporter() StatsReporter {
	return c.topChannel.StatsReporter()
}

// StatsTags returns the stats tags for this subchannel.
func (c *SubChannel) StatsTags() map[string]string {
	tags := c.topChannel.StatsTags()
	tags["subchannel"] = c.serviceName
	return tags
}

// Register a new subchannel for the given serviceName
func (subChMap *subChannelMap) registerNewSubChannel(serviceName string, ch *Channel) *SubChannel {
	subChMap.Lock()
	defer subChMap.Unlock()

	if subChMap.subchannels == nil {
		subChMap.subchannels = make(map[string]*SubChannel)
	}

	if sc, ok := subChMap.subchannels[serviceName]; ok {
		return sc
	}

	sc := newSubChannel(serviceName, ch)
	subChMap.subchannels[serviceName] = sc
	return sc
}

// Get subchannel if, we have one
func (subChMap *subChannelMap) get(serviceName string) (*SubChannel, bool) {
	subChMap.RLock()
	sc, ok := subChMap.subchannels[serviceName]
	subChMap.RUnlock()
	return sc, ok
}

// GetOrAdd a subchannel for the given serviceName on the map
func (subChMap *subChannelMap) getOrAdd(serviceName string, ch *Channel) *SubChannel {
	if sc, ok := subChMap.get(serviceName); ok {
		return sc
	}

	return subChMap.registerNewSubChannel(serviceName, ch)
}

func (subChMap *subChannelMap) updatePeer(p *Peer) {
	subChMap.RLock()
	for _, subCh := range subChMap.subchannels {
		if subCh.Isolated() {
			subCh.RLock()
			subCh.Peers().updatePeer(p)
			subCh.RUnlock()
		}
	}
	subChMap.RUnlock()
}
<commit_msg>SubChannel.SetHandler: Don't use locks<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel

import (
	"fmt"
	"sync"

	"golang.org/x/net/context"
)

// SubChannelOption are used to set options for subchannels.
type SubChannelOption func(*SubChannel)

// Isolated is a SubChannelOption that creates an isolated subchannel.
func Isolated(s *SubChannel) {
	s.Lock()
	s.peers = s.topChannel.peers.newSibling()
	s.peers.SetStrategy(newLeastPendingCalculator())
	s.Unlock()
}

// SubChannel allows calling a specific service on a channel.
// TODO(prashant): Allow creating a subchannel with default call options.
// TODO(prashant): Allow registering handlers on a subchannel.
type SubChannel struct {
	sync.RWMutex
	serviceName        string
	topChannel         *Channel
	defaultCallOptions *CallOptions
	peers              *PeerList
	handler            Handler
	logger             Logger
	statsReporter      StatsReporter
}

// Map of subchannel and the corresponding service
type subChannelMap struct {
	sync.RWMutex
	subchannels map[string]*SubChannel
}

func newSubChannel(serviceName string, ch *Channel) *SubChannel {
	logger := ch.Logger().WithFields(LogField{"subchannel", serviceName})
	return &SubChannel{
		serviceName:   serviceName,
		peers:         ch.peers,
		topChannel:    ch,
		handler:       &handlerMap{}, // use handlerMap by default
		logger:        logger,
		statsReporter: ch.StatsReporter(),
	}
}

// ServiceName returns the service name that this subchannel is for.
func (c *SubChannel) ServiceName() string {
	return c.serviceName
}

// BeginCall starts a new call to a remote peer, returning an OutboundCall that can
// be used to write the arguments of the call.
func (c *SubChannel) BeginCall(ctx context.Context, methodName string, callOptions *CallOptions) (*OutboundCall, error) {
	if callOptions == nil {
		callOptions = defaultCallOptions
	}

	peer, err := c.peers.Get(callOptions.RequestState.PrevSelectedPeers())
	if err != nil {
		return nil, err
	}

	return peer.BeginCall(ctx, c.ServiceName(), methodName, callOptions)
}

// Peers returns the PeerList for this subchannel.
func (c *SubChannel) Peers() *PeerList {
	return c.peers
}

// Isolated returns whether this subchannel is an isolated subchannel.
func (c *SubChannel) Isolated() bool {
	c.RLock()
	defer c.RUnlock()
	return c.topChannel.Peers() != c.peers
}

// Register registers a handler on the subchannel for the given method.
//
// This function panics if the Handler for the SubChannel was overwritten with
// SetHandler.
func (c *SubChannel) Register(h Handler, methodName string) {
	handlers, ok := c.handler.(*handlerMap)
	if !ok {
		panic(fmt.Sprintf(
			"handler for SubChannel(%v) was changed to disallow method registration", c.ServiceName,
		))
	}
	handlers.register(h, methodName)
}

// SetHandler changes the SubChannel's underlying handler. This may be used to
// set up a catch-all Handler for all requests received by this SubChannel.
//
// Methods registered on this SubChannel using Register() before calling
// SetHandler() will be forgotten. Further calls to Register() on this
// SubChannel after SetHandler() is called will cause panics.
func (c *SubChannel) SetHandler(h Handler) {
	c.handler = h
}

// Logger returns the logger for this subchannel.
func (c *SubChannel) Logger() Logger {
	return c.logger
}

// StatsReporter returns the stats reporter for this subchannel.
func (c *SubChannel) StatsReporter() StatsReporter {
	return c.topChannel.StatsReporter()
}

// StatsTags returns the stats tags for this subchannel.
func (c *SubChannel) StatsTags() map[string]string {
	tags := c.topChannel.StatsTags()
	tags["subchannel"] = c.serviceName
	return tags
}

// Register a new subchannel for the given serviceName
func (subChMap *subChannelMap) registerNewSubChannel(serviceName string, ch *Channel) *SubChannel {
	subChMap.Lock()
	defer subChMap.Unlock()

	if subChMap.subchannels == nil {
		subChMap.subchannels = make(map[string]*SubChannel)
	}

	if sc, ok := subChMap.subchannels[serviceName]; ok {
		return sc
	}

	sc := newSubChannel(serviceName, ch)
	subChMap.subchannels[serviceName] = sc
	return sc
}

// Get subchannel if, we have one
func (subChMap *subChannelMap) get(serviceName string) (*SubChannel, bool) {
	subChMap.RLock()
	sc, ok := subChMap.subchannels[serviceName]
	subChMap.RUnlock()
	return sc, ok
}

// GetOrAdd a subchannel for the given serviceName on the map
func (subChMap *subChannelMap) getOrAdd(serviceName string, ch *Channel) *SubChannel {
	if sc, ok := subChMap.get(serviceName); ok {
		return sc
	}

	return subChMap.registerNewSubChannel(serviceName, ch)
}

func (subChMap *subChannelMap) updatePeer(p *Peer) {
	subChMap.RLock()
	for _, subCh := range subChMap.subchannels {
		if subCh.Isolated() {
			subCh.RLock()
			subCh.Peers().updatePeer(p)
			subCh.RUnlock()
		}
	}
	subChMap.RUnlock()
}
<|endoftext|>


<commit_before>package gexpect

import (
	"bufio"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"regexp"
	"syscall"
	"time"

	"github.com/blacknon/gexpect/pty"
)

var (
	err error
)

type SubProcess struct {
	Term            *pty.Terminal
	cmd             *exec.Cmd
	DelayBeforeSend time.Duration
	CheckInterval   time.Duration
	Before          []byte
	After           []byte
	Match           []byte
	echo            bool
}

func (sp *SubProcess) Start() (err error) {
	return sp.Term.Start(sp.cmd)
}

func (sp *SubProcess) Close() (err error) {
	sp.Terminate()
	return sp.Term.Close()
}

func (sp *SubProcess) WaitTimeout(d time.Duration) (err error) {
	if sp.echo {
		go func() {
			io.Copy(os.Stdout, sp)
		}()
	}
	execerr := make(chan error, 1)
	if d > 0 {
		go func() {
			time.Sleep(d)
			execerr <- TIMEOUT
		}()
	}
	go func() {
		execerr <- sp.cmd.Wait()
	}()
	return <-execerr
}

func (sp *SubProcess) Wait() error {
	return sp.WaitTimeout(0)
}

func (sp *SubProcess) Terminate() error {
	return sp.cmd.Process.Kill()
}

func (sp *SubProcess) Expect(expreg ...*regexp.Regexp) (matchIndex int, err error) {
	return sp.ExpectTimeout(0, expreg...)
}

func (sp *SubProcess) ExpectTimeout(timeout time.Duration, expreg ...*regexp.Regexp) (matchIndex int, err error) {
	buf := make([]byte, 2048)
	c := make(chan byte, 1)
	checkpoint := make(chan int, 1)
	rerr := make(chan error, 1)
	go func() {
		for {
			if _, err := io.ReadAtLeast(sp, buf, 1); err != nil {
				rerr <- err
				continue
			}
			for _, b := range buf {
				c <- b
			}
			if sp.echo {
				os.Stdout.Write(buf)
				os.Stdout.Sync()
			}
		}
	}()
	if timeout > 0 {
		go func() {
			time.Sleep(timeout)
			rerr <- TIMEOUT
		}()
	}
	go func() {
		for i := 1; ; i++ {
			time.Sleep(sp.CheckInterval)
			checkpoint <- i
		}
	}()
	for {
		select {
		case c1 := <-c:
			buf = append(buf, c1)
		case e := <-rerr:
			sp.Before = append(sp.Before, buf...)
			return -1, e
		case <-checkpoint:
			for idx, re := range expreg {
				if loc := re.FindIndex(buf); loc != nil {
					sp.Match = buf[loc[0]:loc[1]]
					sp.Before = append(sp.Before, buf[0:loc[0]]...)
					buf = make([]byte, 2048)
					return idx, nil
				}
			} // no match
		}
	}
	return -1, nil
}

func (sp *SubProcess) Read(b []byte) (n int, err error) {
	return sp.Term.Read(b)
}

func (sp *SubProcess) Write(b []byte) (n int, err error) {
	time.Sleep(sp.DelayBeforeSend)
	return sp.Term.Write(b)
}

func (sp *SubProcess) Writeln(b []byte) (n int, err error) {
	bn := append(b, []byte("\r\n")...)
	return sp.Write(bn)
}

func (sp *SubProcess) Send(response string) (err error) {
	_, err = sp.Write([]byte(response))
	return
}

func (sp *SubProcess) SendLine(response string) (err error) {
	return sp.Send(response + "\r\n")
}

func (sp *SubProcess) Interact() (err error) {
	return sp.InteractTimeout(0)
}

func (sp *SubProcess) InteractTimeout(d time.Duration) (err error) {
	sp.Write(sp.After)
	sp.After = []byte{}
	oldState, _ := pty.Tcgetattr(os.Stdin)
	pty.SetRaw(os.Stdin)
	defer pty.Tcsetattr(os.Stdin, oldState)
	s := make(chan os.Signal, 1)
	signal.Notify(s, syscall.SIGINT, syscall.SIGWINCH, syscall.SIGTSTP)
	go func() {
		for sig := range s {
			switch sig {
			case syscall.SIGINT:
				sp.Term.SendIntr()
			case syscall.SIGWINCH:
				if x, y, err := pty.GetWinSize(os.Stdout); err == nil {
					sp.Term.SetWinSize(x, y)
				}
			default:
				continue
			}
		}
	}()
	execerr := make(chan error, 1)
	if d > 0 {
		go func() {
			time.Sleep(d)
			execerr <- TIMEOUT
		}()
	}
	go func() {
		execerr <- sp.cmd.Wait()
	}()
	in := make(chan byte, 1)
	stdin := bufio.NewReader(os.Stdin)
	go func() error {
		var b byte
		for {
			if b, err = stdin.ReadByte(); err != nil {
				if err == io.EOF {
					sp.Term.SendEOF()
					continue
				} else {
					return err
				}
			}
			in <- b
		}
	}()
	go func() {
		io.Copy(os.Stdout, sp)
		return
	}()
	for {

		select {
		case err := <-execerr:
			return err
		case b := <-in:
			_, err = sp.Write([]byte{b})
		}
	}
	return
}

func (sp *SubProcess) Echo() {
	sp.echo = true
}

func (sp *SubProcess) NoEcho() {
	sp.echo = false
}

func NewSubProcess(name string, arg ...string) (sp *SubProcess, err error) {
	sp = new(SubProcess)
	if sp.Term, err = pty.NewTerminal(); err != nil {
		return
	}
	sp.cmd = exec.Command(name, arg...)
	sp.DelayBeforeSend = 50 * time.Microsecond
	sp.CheckInterval = time.Microsecond
	if x, y, err := pty.GetWinSize(os.Stdout); err == nil {
		sp.Term.SetWinSize(x, y)

	}
	return
}
<commit_msg>change import.<commit_after>package gexpect

import (
	"bufio"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"regexp"
	"syscall"
	"time"

	"github.com/shavac/gexpect/pty"
)

var (
	err error
)

type SubProcess struct {
	Term            *pty.Terminal
	cmd             *exec.Cmd
	DelayBeforeSend time.Duration
	CheckInterval   time.Duration
	Before          []byte
	After           []byte
	Match           []byte
	echo            bool
}

func (sp *SubProcess) Start() (err error) {
	return sp.Term.Start(sp.cmd)
}

func (sp *SubProcess) Close() (err error) {
	sp.Terminate()
	return sp.Term.Close()
}

func (sp *SubProcess) WaitTimeout(d time.Duration) (err error) {
	if sp.echo {
		go func() {
			io.Copy(os.Stdout, sp)
		}()
	}
	execerr := make(chan error, 1)
	if d > 0 {
		go func() {
			time.Sleep(d)
			execerr <- TIMEOUT
		}()
	}
	go func() {
		execerr <- sp.cmd.Wait()
	}()
	return <-execerr
}

func (sp *SubProcess) Wait() error {
	return sp.WaitTimeout(0)
}

func (sp *SubProcess) Terminate() error {
	return sp.cmd.Process.Kill()
}

func (sp *SubProcess) Expect(expreg ...*regexp.Regexp) (matchIndex int, err error) {
	return sp.ExpectTimeout(0, expreg...)
}

func (sp *SubProcess) ExpectTimeout(timeout time.Duration, expreg ...*regexp.Regexp) (matchIndex int, err error) {
	buf := make([]byte, 2048)
	c := make(chan byte, 1)
	checkpoint := make(chan int, 1)
	rerr := make(chan error, 1)
	go func() {
		for {
			if _, err := io.ReadAtLeast(sp, buf, 1); err != nil {
				rerr <- err
				continue
			}
			for _, b := range buf {
				c <- b
			}
			if sp.echo {
				os.Stdout.Write(buf)
				os.Stdout.Sync()
			}
		}
	}()
	if timeout > 0 {
		go func() {
			time.Sleep(timeout)
			rerr <- TIMEOUT
		}()
	}
	go func() {
		for i := 1; ; i++ {
			time.Sleep(sp.CheckInterval)
			checkpoint <- i
		}
	}()
	for {
		select {
		case c1 := <-c:
			buf = append(buf, c1)
		case e := <-rerr:
			sp.Before = append(sp.Before, buf...)
			return -1, e
		case <-checkpoint:
			for idx, re := range expreg {
				if loc := re.FindIndex(buf); loc != nil {
					sp.Match = buf[loc[0]:loc[1]]
					sp.Before = append(sp.Before, buf[0:loc[0]]...)
					buf = make([]byte, 2048)
					return idx, nil
				}
			} // no match
		}
	}
	return -1, nil
}

func (sp *SubProcess) Read(b []byte) (n int, err error) {
	return sp.Term.Read(b)
}

func (sp *SubProcess) Write(b []byte) (n int, err error) {
	time.Sleep(sp.DelayBeforeSend)
	return sp.Term.Write(b)
}

func (sp *SubProcess) Writeln(b []byte) (n int, err error) {
	bn := append(b, []byte("\r\n")...)
	return sp.Write(bn)
}

func (sp *SubProcess) Send(response string) (err error) {
	_, err = sp.Write([]byte(response))
	return
}

func (sp *SubProcess) SendLine(response string) (err error) {
	return sp.Send(response + "\r\n")
}

func (sp *SubProcess) Interact() (err error) {
	return sp.InteractTimeout(0)
}

func (sp *SubProcess) InteractTimeout(d time.Duration) (err error) {
	sp.Write(sp.After)
	sp.After = []byte{}
	oldState, _ := pty.Tcgetattr(os.Stdin)
	pty.SetRaw(os.Stdin)
	defer pty.Tcsetattr(os.Stdin, oldState)
	s := make(chan os.Signal, 1)
	signal.Notify(s, syscall.SIGINT, syscall.SIGWINCH, syscall.SIGTSTP)
	go func() {
		for sig := range s {
			switch sig {
			case syscall.SIGINT:
				sp.Term.SendIntr()
			case syscall.SIGWINCH:
				if x, y, err := pty.GetWinSize(os.Stdout); err == nil {
					sp.Term.SetWinSize(x, y)
				}
			default:
				continue
			}
		}
	}()
	execerr := make(chan error, 1)
	if d > 0 {
		go func() {
			time.Sleep(d)
			execerr <- TIMEOUT
		}()
	}
	go func() {
		execerr <- sp.cmd.Wait()
	}()
	in := make(chan byte, 1)
	stdin := bufio.NewReader(os.Stdin)
	go func() error {
		var b byte
		for {
			if b, err = stdin.ReadByte(); err != nil {
				if err == io.EOF {
					sp.Term.SendEOF()
					continue
				} else {
					return err
				}
			}
			in <- b
		}
	}()
	go func() {
		io.Copy(os.Stdout, sp)
		return
	}()
	for {

		select {
		case err := <-execerr:
			return err
		case b := <-in:
			_, err = sp.Write([]byte{b})
		}
	}
	return
}

func (sp *SubProcess) Echo() {
	sp.echo = true
}

func (sp *SubProcess) NoEcho() {
	sp.echo = false
}

func NewSubProcess(name string, arg ...string) (sp *SubProcess, err error) {
	sp = new(SubProcess)
	if sp.Term, err = pty.NewTerminal(); err != nil {
		return
	}
	sp.cmd = exec.Command(name, arg...)
	sp.DelayBeforeSend = 50 * time.Microsecond
	sp.CheckInterval = time.Microsecond
	if x, y, err := pty.GetWinSize(os.Stdout); err == nil {
		sp.Term.SetWinSize(x, y)

	}
	return
}
<|endoftext|>


<commit_before>package copier

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/juju/errors"
)

type (
	// A Supervisor is batch copier handler.
	Supervisor struct {
		context.Context
		base          string
		logger        *Logger
		FilePaths     []string
		Destination   string
		ExecTimeout   time.Duration
		Progress      chan ProgressInfo
		Speed         float64
		Retries       int
		RetryInterval time.Duration
		summary       map[string]int
	}

	// ProgressInfo handle data for building a progress bar.
	ProgressInfo struct {
		Name        string
		Size        int64
		ProxyReader *ProxyReader
		Status      string
	}
)

// NewSupervisor returns a new Supervisor.
func NewSupervisor(paths []string, destination string) (*Supervisor, error) {
	base, err := GetBaseDirectory(paths)
	ctx := context.Background()
	return &Supervisor{
		Context: ctx,
		base:    base,
		logger:  NewLoggerWithContext(ctx),
		summary: map[string]int{
			StatusAlreadyExist: 0,
			StatusOverwritten:  0,
			StatusCopied:       0,
			StatusFailed:       0,
		},
		FilePaths:     paths,
		Destination:   destination,
		ExecTimeout:   10 * time.Minute,
		Progress:      make(chan ProgressInfo),
		Speed:         float64(512 * KBps),
		Retries:       5,
		RetryInterval: 2 * time.Second,
	}, err
}

// Logger returns the Supervisor logger.
func (s *Supervisor) Logger() *Logger {
	return s.logger
}

// Summary returns the summary of an execution.
func (s *Supervisor) Summary() map[string]int {
	return s.summary
}

// Execute runs the Supervisor copy batch.
func (s *Supervisor) Execute() error {
	fmt.Println("Start copy")
	fmt.Println("  - Speed:", int64(s.Speed), "Bps")
	fmt.Println("  - Timeout:", s.ExecTimeout)
	fmt.Println("  - Retries:", s.Retries)
	fmt.Println("  - Retry Interval:", s.RetryInterval)

	for _, from := range s.FilePaths {
		to := strings.Replace(from, s.base, s.Destination, 1)

		if err := s.execute(from, to, s.Retries, nil); err != nil {
			// Ignore returned error; go to next file.
			s.logger.C <- LogEntry{
				Severity:    ERROR,
				Status:      StatusFailed,
				FilepathSrc: from,
				FilepathDst: err.Error(),
			}
		}
	}
	return nil
}

func (s *Supervisor) execute(from, to string, retries int, err2 error) error {
	if retries == 0 {
		return err2 // keep last error
	}

	errc := make(chan error)
	var err error

	ctx, cancel := context.WithTimeout(s.Context, s.ExecTimeout)
	defer cancel()

	cp := NewExecWithContext(ctx, from, to, s.Speed)
	cp.Speed = s.Speed
	go func() {
		if err = cp.Execute(); err != nil {
			errc <- errors.Annotate(err, "Exec#Execute")
		}
		errc <- nil
	}()

	ready := false
	terminated := false
	for {
		select {
		case <-cp.Ready:
			s.Progress <- ProgressInfo{
				Name:        cp.Name(),
				Size:        cp.Size(),
				ProxyReader: cp.Reader(),
				Status:      cp.Status(),
			}
			ready = true
		case <-ctx.Done():
			if ctx.Err() != nil {
				time.Sleep(s.RetryInterval)
				err = s.execute(from, to, retries-1, ctx.Err())
			}

			terminated = true
		case err = <-errc:
			if err != nil {
				time.Sleep(s.RetryInterval)
				err = s.execute(from, to, retries-1, err)
			}

			terminated = true
		}

		if terminated {
			if !ready {
				s.Progress <- ProgressInfo{
					Name:        cp.Name(),
					Size:        cp.Size(),
					ProxyReader: cp.Reader(),
					Status:      StatusFailed,
				}
			}

			if err = s.logsOrReturnError(cp, err); err != nil {
				return err
			}
			break
		}
	}
	return nil
}

func (s *Supervisor) logsOrReturnError(cp *Exec, err error) error {
	// Succeed
	if err == nil {
		s.summary[cp.Status()]++
		s.logger.C <- LogEntry{
			Severity:    INFO,
			Status:      cp.Status(),
			FilepathSrc: cp.From,
			FilepathDst: cp.To,
		}
		return nil
	}
	// Failed
	s.summary[StatusFailed]++
	if err == context.DeadlineExceeded || strings.HasSuffix(err.Error(), "would exceed context deadline") { // first is ctx.Err() and the second is a shapeio.Reder error
		s.logger.C <- LogEntry{
			Severity:    ERROR,
			Status:      StatusFailed,
			FilepathSrc: cp.From,
			FilepathDst: cp.To,
		}
		return nil
	}

	// Fatal error
	return err
}
<commit_msg>struct field normalize<commit_after>package copier

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/juju/errors"
)

type (
	// A Supervisor is batch copier handler.
	Supervisor struct {
		context.Context
		base          string
		logger        *Logger
		summary       map[string]int
		FilePaths     []string
		Destination   string
		ExecTimeout   time.Duration
		Progress      chan ProgressInfo
		Speed         float64
		Retries       int
		RetryInterval time.Duration
	}

	// ProgressInfo handle data for building a progress bar.
	ProgressInfo struct {
		Name        string
		Size        int64
		ProxyReader *ProxyReader
		Status      string
	}
)

// NewSupervisor returns a new Supervisor.
func NewSupervisor(paths []string, destination string) (*Supervisor, error) {
	base, err := GetBaseDirectory(paths)
	ctx := context.Background()
	return &Supervisor{
		Context: ctx,
		base:    base,
		logger:  NewLoggerWithContext(ctx),
		summary: map[string]int{
			StatusAlreadyExist: 0,
			StatusOverwritten:  0,
			StatusCopied:       0,
			StatusFailed:       0,
		},
		FilePaths:     paths,
		Destination:   destination,
		ExecTimeout:   10 * time.Minute,
		Progress:      make(chan ProgressInfo),
		Speed:         float64(512 * KBps),
		Retries:       5,
		RetryInterval: 2 * time.Second,
	}, err
}

// Logger returns the Supervisor logger.
func (s *Supervisor) Logger() *Logger {
	return s.logger
}

// Summary returns the summary of an execution.
func (s *Supervisor) Summary() map[string]int {
	return s.summary
}

// Execute runs the Supervisor copy batch.
func (s *Supervisor) Execute() error {
	fmt.Println("Start copy")
	fmt.Println("  - Speed:", int64(s.Speed), "Bps")
	fmt.Println("  - Timeout:", s.ExecTimeout)
	fmt.Println("  - Retries:", s.Retries)
	fmt.Println("  - Retry Interval:", s.RetryInterval)

	for _, from := range s.FilePaths {
		to := strings.Replace(from, s.base, s.Destination, 1)

		if err := s.execute(from, to, s.Retries, nil); err != nil {
			// Ignore returned error; go to next file.
			s.logger.C <- LogEntry{
				Severity:    ERROR,
				Status:      StatusFailed,
				FilepathSrc: from,
				FilepathDst: err.Error(),
			}
		}
	}
	return nil
}

func (s *Supervisor) execute(from, to string, retries int, err2 error) error {
	if retries == 0 {
		return err2 // keep last error
	}

	errc := make(chan error)
	var err error

	ctx, cancel := context.WithTimeout(s.Context, s.ExecTimeout)
	defer cancel()

	cp := NewExecWithContext(ctx, from, to, s.Speed)
	cp.Speed = s.Speed
	go func() {
		if err = cp.Execute(); err != nil {
			errc <- errors.Annotate(err, "Exec#Execute")
		}
		errc <- nil
	}()

	ready := false
	terminated := false
	for {
		select {
		case <-cp.Ready:
			s.Progress <- ProgressInfo{
				Name:        cp.Name(),
				Size:        cp.Size(),
				ProxyReader: cp.Reader(),
				Status:      cp.Status(),
			}
			ready = true
		case <-ctx.Done():
			if ctx.Err() != nil {
				time.Sleep(s.RetryInterval)
				err = s.execute(from, to, retries-1, ctx.Err())
			}

			terminated = true
		case err = <-errc:
			if err != nil {
				time.Sleep(s.RetryInterval)
				err = s.execute(from, to, retries-1, err)
			}

			terminated = true
		}

		if terminated {
			if !ready {
				s.Progress <- ProgressInfo{
					Name:        cp.Name(),
					Size:        cp.Size(),
					ProxyReader: cp.Reader(),
					Status:      StatusFailed,
				}
			}

			if err = s.logsOrReturnError(cp, err); err != nil {
				return err
			}
			break
		}
	}
	return nil
}

func (s *Supervisor) logsOrReturnError(cp *Exec, err error) error {
	// Succeed
	if err == nil {
		s.summary[cp.Status()]++
		s.logger.C <- LogEntry{
			Severity:    INFO,
			Status:      cp.Status(),
			FilepathSrc: cp.From,
			FilepathDst: cp.To,
		}
		return nil
	}
	// Failed
	s.summary[StatusFailed]++
	if err == context.DeadlineExceeded || strings.HasSuffix(err.Error(), "would exceed context deadline") { // first is ctx.Err() and the second is a shapeio.Reder error
		s.logger.C <- LogEntry{
			Severity:    ERROR,
			Status:      StatusFailed,
			FilepathSrc: cp.From,
			FilepathDst: cp.To,
		}
		return nil
	}

	// Fatal error
	return err
}
<|endoftext|>


<commit_before>package learn

import (
	cic "github.com/tsinghua-io/api-server/adapter/tsinghua.edu.cn/cic/learn"
	old "github.com/tsinghua-io/api-server/adapter/tsinghua.edu.cn/learn"
	"github.com/tsinghua-io/api-server/model"
	"github.com/tsinghua-io/api-server/util"
	"net/http"
	"strings"
)

type Adapter struct {
	*cic.Adapter
	Old *old.Adapter
}

func New(userId, password string) (ada *Adapter, status int, errMsg error) {
	ada = new(Adapter)

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		ada.Adapter, *status, *err = cic.New(userId, password)
	})
	sg.Go(func(status *int, err *error) {
		ada.Old, *status, *err = old.New(userId, password)
	})
	status, errMsg = sg.Wait()

	return
}

func (ada *Adapter) Announcements(courseId string) (announcements []*model.Announcement, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Announcements(courseId)
	} else {
		return ada.Old.Announcements(courseId)
	}
}

func (ada *Adapter) Assignments(courseId string) (assignments []*model.Assignment, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Assignments(courseId)
	} else {
		return ada.Old.Assignments(courseId)
	}
}

type fatName struct {
	name string
	seq  string
	sem  string
}

func newFatName(course *model.Course) fatName {
	return fatName{course.Name, course.CourseSequence, course.Semester}
}

func (ada *Adapter) nameIdMap() (m map[fatName]string, status int, errMsg error) {
	var courses []*model.Course
	if courses, status, errMsg = ada.Old.AllAttendedList(); errMsg == nil {
		m := make(map[fatName]string)
		for _, course := range courses {
			if strings.Contains(course.Id, "-") {
				m[newFatName(course)] = course.Id
			}
		}
	}
}

func replaceCourseIds(courses []*model.Course, m map[fatName]string) {
	for _, course := range courses {
		if oldId, ok := m[newFatName(course)]; ok {
			course.Id = oldId
		}
	}
}

func (ada *Adapter) Attended(semesterID string, english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.Attended(semesterID, english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) NowAttended(english bool) (thisCourses []*model.Course, nextCourses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.NowAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) PastAttended(english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.PastAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) AllAttended(english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.AllAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) Files(courseId string) (files []*model.File, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Files(courseId)
	} else {
		return ada.Old.Files(courseId)
	}
}

func (ada *Adapter) Profile() (profile *model.User, status int, errMsg error) {
	return ada.Adapter.Profile()
}
<commit_msg>Add HandlerFunc for mixed.<commit_after>package learn

import (
	cic "github.com/tsinghua-io/api-server/adapter/tsinghua.edu.cn/cic/learn"
	old "github.com/tsinghua-io/api-server/adapter/tsinghua.edu.cn/learn"
	"github.com/tsinghua-io/api-server/model"
	"github.com/tsinghua-io/api-server/util"
	"net/http"
	"strings"
)

type Adapter struct {
	*cic.Adapter
	Old *old.Adapter
}

func New(userId, password string) (ada *Adapter, status int, errMsg error) {
	ada = new(Adapter)

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		ada.Adapter, *status, *err = cic.New(userId, password)
	})
	sg.Go(func(status *int, err *error) {
		ada.Old, *status, *err = old.New(userId, password)
	})
	status, errMsg = sg.Wait()

	return
}

func (ada *Adapter) Announcements(courseId string) (announcements []*model.Announcement, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Announcements(courseId)
	} else {
		return ada.Old.Announcements(courseId)
	}
}

func (ada *Adapter) Assignments(courseId string) (assignments []*model.Assignment, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Assignments(courseId)
	} else {
		return ada.Old.Assignments(courseId)
	}
}

type fatName struct {
	name string
	seq  string
	sem  string
}

func newFatName(course *model.Course) fatName {
	return fatName{course.Name, course.CourseSequence, course.Semester}
}

func (ada *Adapter) nameIdMap() (m map[fatName]string, status int, errMsg error) {
	var courses []*model.Course
	if courses, status, errMsg = ada.Old.AllAttendedList(); errMsg == nil {
		m := make(map[fatName]string)
		for _, course := range courses {
			if strings.Contains(course.Id, "-") {
				m[newFatName(course)] = course.Id
			}
		}
	}
}

func replaceCourseIds(courses []*model.Course, m map[fatName]string) {
	for _, course := range courses {
		if oldId, ok := m[newFatName(course)]; ok {
			course.Id = oldId
		}
	}
}

func (ada *Adapter) Attended(semesterID string, english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.Attended(semesterID, english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) NowAttended(english bool) (thisCourses []*model.Course, nextCourses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.NowAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) PastAttended(english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.PastAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) AllAttended(english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.AllAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) Files(courseId string) (files []*model.File, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Files(courseId)
	} else {
		return ada.Old.Files(courseId)
	}
}

func (ada *Adapter) Profile() (profile *model.User, status int, errMsg error) {
	return ada.Adapter.Profile()
}

func HandlerFunc(f func(http.ResponseWriter, *http.Request, *Adapter)) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		userId, password, _ := req.BasicAuth()
		if ada, status, err := New(userId, password); err != nil {
			util.Error(rw, err.Error(), status)
		} else {
			f(rw, req, ada)
		}
	})
}
<|endoftext|>


<commit_before>package sync_test

import (
	"bytes"
	"io"
	"math/rand"
	"testing"

	"github.com/itchio/wharf/sync"
)

type RandReader struct {
	rand.Source
}

func (rr RandReader) Read(sink []byte) (int, error) {
	var tail, head int
	buf := make([]byte, 8)
	var r uint64
	for {
		head = min(tail+8, len(sink))
		if tail == head {
			return head, nil
		}

		r = (uint64)(rr.Int63())
		buf[0] = (byte)(r)
		buf[1] = (byte)(r >> 8)
		buf[2] = (byte)(r >> 16)
		buf[3] = (byte)(r >> 24)
		buf[4] = (byte)(r >> 32)
		buf[5] = (byte)(r >> 40)
		buf[6] = (byte)(r >> 48)
		buf[7] = (byte)(r >> 56)

		tail += copy(sink[tail:head], buf)
	}
}

type pair struct {
	Source, Target content
	Description    string
}
type content struct {
	Len   int
	Seed  int64
	Alter int
	Data  []byte
}

func (c *content) Fill() {
	c.Data = make([]byte, c.Len)
	src := rand.NewSource(c.Seed)
	RandReader{src}.Read(c.Data)

	if c.Alter > 0 {
		r := rand.New(src)
		for i := 0; i < c.Alter; i++ {
			at := r.Intn(len(c.Data))
			c.Data[at] += byte(r.Int())
		}
	}
}

func Test_GenData(t *testing.T) {
	// Use a seeded generator to get consistent results.
	// This allows testing the package without bundling many test files.

	var pairs = []pair{
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Target:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 5},
			Description: "Same length, slightly different content.",
		},
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 9824, Alter: 0},
			Target:      content{Len: 8*512*1024 + 89, Seed: 2345, Alter: 0},
			Description: "Same length, very different content.",
		},
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Target:      content{Len: 8*256*1024 + 19, Seed: 42, Alter: 0},
			Description: "Target shorter then source, same content.",
		},
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Target:      content{Len: 8*256*1024 + 19, Seed: 42, Alter: 5},
			Description: "Target shorter then source, slightly different content.",
		},
		pair{
			Source:      content{Len: 8*256*1024 + 19, Seed: 42, Alter: 0},
			Target:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Description: "Source shorter then target, same content.",
		},
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 5},
			Target:      content{Len: 8*256*1024 + 19, Seed: 42, Alter: 0},
			Description: "Source shorter then target, slightly different content.",
		},
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Target:      content{Len: 0, Seed: 42, Alter: 0},
			Description: "Target empty and source has content.",
		},
		pair{
			Source:      content{Len: 0, Seed: 42, Alter: 0},
			Target:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Description: "Source empty and target has content.",
		},
		pair{
			Source:      content{Len: 8 * 872, Seed: 9824, Alter: 0},
			Target:      content{Len: 8 * 235, Seed: 2345, Alter: 0},
			Description: "Source and target both smaller then a block size.",
		},
	}
	rs := sync.NewContext(16 * 1024)
	rsDelta := sync.NewContext(16 * 1024)
	for _, p := range pairs {
		(&p.Source).Fill()
		(&p.Target).Fill()

		sourceBuffer := bytes.NewReader(p.Source.Data)
		targetBuffer := bytes.NewReader(p.Target.Data)

		sig := make([]sync.BlockHash, 0, 10)
		err := rs.CreateSignature(0, targetBuffer, func(bl sync.BlockHash) error {
			sig = append(sig, bl)
			return nil
		})
		if err != nil {
			t.Errorf("Failed to create signature: %s", err)
		}
		lib := sync.NewBlockLibrary(sig)

		opsOut := make(chan sync.Operation)
		go func() {
			var blockCt, blockRangeCt, dataCt, bytes int
			defer close(opsOut)
			err := rsDelta.ComputeDiff(sourceBuffer, lib, func(op sync.Operation) error {
				switch op.Type {
				case sync.OpBlockRange:
					blockRangeCt++
				case sync.OpBlock:
					blockCt++
				case sync.OpData:
					// Copy data buffer so it may be reused in internal buffer.
					b := make([]byte, len(op.Data))
					copy(b, op.Data)
					op.Data = b
					dataCt++
					bytes += len(op.Data)
				}
				opsOut <- op
				return nil
			})
			t.Logf("Range Ops:%5d, Block Ops:%5d, Data Ops: %5d, Data Len: %5dKiB, For %s.", blockRangeCt, blockCt, dataCt, bytes/1024, p.Description)
			if err != nil {
				t.Errorf("Failed to create delta: %s", err)
			}
		}()

		result := new(bytes.Buffer)
		filePool := &SinglePool{reader: targetBuffer}

		targetBuffer.Seek(0, 0)
		err = rs.ApplyPatch(result, filePool, opsOut)
		if err != nil {
			t.Errorf("Failed to apply delta: %s", err)
		}

		if result.Len() != len(p.Source.Data) {
			t.Errorf("Result not same size as source: %s", p.Description)
		} else if bytes.Equal(result.Bytes(), p.Source.Data) == false {
			t.Errorf("Result is different from the source: %s", p.Description)
		}

		p.Source.Data = nil
		p.Target.Data = nil
	}
}

type SinglePool struct {
	reader io.ReadSeeker
}

func (sp *SinglePool) GetReader(fileIndex int64) (io.ReadSeeker, error) {
	return sp.reader, nil
}

func (sp *SinglePool) Close() error {
	return nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Fix gen_test<commit_after>package sync_test

import (
	"bytes"
	"io"
	"math/rand"
	"testing"

	"github.com/itchio/wharf/sync"
)

type RandReader struct {
	rand.Source
}

func (rr RandReader) Read(sink []byte) (int, error) {
	var tail, head int
	buf := make([]byte, 8)
	var r uint64
	for {
		head = min(tail+8, len(sink))
		if tail == head {
			return head, nil
		}

		r = (uint64)(rr.Int63())
		buf[0] = (byte)(r)
		buf[1] = (byte)(r >> 8)
		buf[2] = (byte)(r >> 16)
		buf[3] = (byte)(r >> 24)
		buf[4] = (byte)(r >> 32)
		buf[5] = (byte)(r >> 40)
		buf[6] = (byte)(r >> 48)
		buf[7] = (byte)(r >> 56)

		tail += copy(sink[tail:head], buf)
	}
}

type pair struct {
	Source, Target content
	Description    string
}
type content struct {
	Len   int
	Seed  int64
	Alter int
	Data  []byte
}

func (c *content) Fill() {
	c.Data = make([]byte, c.Len)
	src := rand.NewSource(c.Seed)
	RandReader{src}.Read(c.Data)

	if c.Alter > 0 {
		r := rand.New(src)
		for i := 0; i < c.Alter; i++ {
			at := r.Intn(len(c.Data))
			c.Data[at] += byte(r.Int())
		}
	}
}

func Test_GenData(t *testing.T) {
	// Use a seeded generator to get consistent results.
	// This allows testing the package without bundling many test files.

	var pairs = []pair{
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Target:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 5},
			Description: "Same length, slightly different content.",
		},
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 9824, Alter: 0},
			Target:      content{Len: 8*512*1024 + 89, Seed: 2345, Alter: 0},
			Description: "Same length, very different content.",
		},
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Target:      content{Len: 8*256*1024 + 19, Seed: 42, Alter: 0},
			Description: "Target shorter then source, same content.",
		},
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Target:      content{Len: 8*256*1024 + 19, Seed: 42, Alter: 5},
			Description: "Target shorter then source, slightly different content.",
		},
		pair{
			Source:      content{Len: 8*256*1024 + 19, Seed: 42, Alter: 0},
			Target:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Description: "Source shorter then target, same content.",
		},
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 5},
			Target:      content{Len: 8*256*1024 + 19, Seed: 42, Alter: 0},
			Description: "Source shorter then target, slightly different content.",
		},
		pair{
			Source:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Target:      content{Len: 0, Seed: 42, Alter: 0},
			Description: "Target empty and source has content.",
		},
		pair{
			Source:      content{Len: 0, Seed: 42, Alter: 0},
			Target:      content{Len: 8*512*1024 + 89, Seed: 42, Alter: 0},
			Description: "Source empty and target has content.",
		},
		pair{
			Source:      content{Len: 8 * 872, Seed: 9824, Alter: 0},
			Target:      content{Len: 8 * 235, Seed: 2345, Alter: 0},
			Description: "Source and target both smaller then a block size.",
		},
	}
	rs := sync.NewContext(16 * 1024)
	rsDelta := sync.NewContext(16 * 1024)
	for _, p := range pairs {
		(&p.Source).Fill()
		(&p.Target).Fill()

		sourceBuffer := bytes.NewReader(p.Source.Data)
		targetBuffer := bytes.NewReader(p.Target.Data)

		sig := make([]sync.BlockHash, 0, 10)
		err := rs.CreateSignature(0, targetBuffer, func(bl sync.BlockHash) error {
			sig = append(sig, bl)
			return nil
		})
		if err != nil {
			t.Errorf("Failed to create signature: %s", err)
		}
		lib := sync.NewBlockLibrary(sig)

		opsOut := make(chan sync.Operation)
		go func() {
			var blockRangeCt, dataCt, bytes int
			defer close(opsOut)
			err := rsDelta.ComputeDiff(sourceBuffer, lib, func(op sync.Operation) error {
				switch op.Type {
				case sync.OpBlockRange:
					blockRangeCt++
				case sync.OpData:
					// Copy data buffer so it may be reused in internal buffer.
					b := make([]byte, len(op.Data))
					copy(b, op.Data)
					op.Data = b
					dataCt++
					bytes += len(op.Data)
				}
				opsOut <- op
				return nil
			}, -1)
			t.Logf("Range Ops:%5d, Data Ops: %5d, Data Len: %5dKiB, For %s.", blockRangeCt, dataCt, bytes/1024, p.Description)
			if err != nil {
				t.Errorf("Failed to create delta: %s", err)
			}
		}()

		result := new(bytes.Buffer)
		filePool := &SinglePool{reader: targetBuffer}

		targetBuffer.Seek(0, 0)
		err = rs.ApplyPatch(result, filePool, opsOut)
		if err != nil {
			t.Errorf("Failed to apply delta: %s", err)
		}

		if result.Len() != len(p.Source.Data) {
			t.Errorf("Result not same size as source: %s", p.Description)
		} else if bytes.Equal(result.Bytes(), p.Source.Data) == false {
			t.Errorf("Result is different from the source: %s", p.Description)
		}

		p.Source.Data = nil
		p.Target.Data = nil
	}
}

type SinglePool struct {
	reader io.ReadSeeker
}

func (sp *SinglePool) GetReader(fileIndex int64) (io.ReadSeeker, error) {
	return sp.reader, nil
}

func (sp *SinglePool) Close() error {
	return nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package tcclient

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"reflect"
	"time"

	"github.com/taskcluster/httpbackoff"
	hawk "github.com/tent/hawk-go"
)

// CallSummary provides information about the underlying http request and
// response issued for a given API call.
type CallSummary struct {
	HttpRequest *http.Request
	// Keep a copy of request body in addition to the *http.Request, since
	// accessing the Body via the *http.Request object, you get a io.ReadCloser
	// - and after the request has been made, the body will have been read, and
	// the data lost... This way, it is still available after the api call
	// returns.
	HttpRequestBody string
	// The Go Type which is marshaled into json and used as the http request
	// body.
	HttpRequestObject interface{}
	HttpResponse      *http.Response
	// Keep a copy of response body in addition to the *http.Response, since
	// accessing the Body via the *http.Response object, you get a
	// io.ReadCloser - and after the response has been read once (to unmarshal
	// json into native go types) the data is lost... This way, it is still
	// available after the api call returns.
	HttpResponseBody string
	// Keep a record of how many http requests were attempted
	Attempts int
}

// utility function to create a URL object based on given data
func setURL(connectionData *ConnectionData, route string, query url.Values) (u *url.URL, err error) {
	u, err = url.Parse(connectionData.BaseURL + route)
	if err != nil {
		return nil, fmt.Errorf("Cannot parse url: '%v', is BaseURL (%v) set correctly?\n%v\n", connectionData.BaseURL+route, connectionData.BaseURL, err)
	}
	if query != nil {
		u.RawQuery = query.Encode()
	}
	return
}

// APICall is the generic REST API calling method which performs all REST API
// calls for this library.  Each auto-generated REST API method simply is a
// wrapper around this method, calling it with specific specific arguments.
func (connectionData *ConnectionData) APICall(payload interface{}, method, route string, result interface{}, query url.Values) (interface{}, *CallSummary, error) {
	callSummary := new(CallSummary)
	callSummary.HttpRequestObject = payload
	jsonPayload, err := json.Marshal(payload)
	if err != nil {
		return result, callSummary, err
	}
	callSummary.HttpRequestBody = string(jsonPayload)

	httpClient := &http.Client{}

	// function to perform http request - we call this using backoff library to
	// have exponential backoff in case of intermittent failures (e.g. network
	// blips or HTTP 5xx errors)
	httpCall := func() (*http.Response, error, error) {
		var ioReader io.Reader = nil
		if reflect.ValueOf(payload).IsValid() && !reflect.ValueOf(payload).IsNil() {
			ioReader = bytes.NewReader(jsonPayload)
		}
		u, err := setURL(connectionData, route, query)
		if err != nil {
			return nil, nil, fmt.Errorf("apiCall url cannot be parsed:\n%v\n", err)
		}
		httpRequest, err := http.NewRequest(method, u.String(), ioReader)
		if err != nil {
			return nil, nil, fmt.Errorf("Internal error: apiCall url cannot be parsed although thought to be valid: '%v', is the BaseURL (%v) set correctly?\n%v\n", u.String(), connectionData.BaseURL, err)
		}
		httpRequest.Header.Set("Content-Type", "application/json")
		callSummary.HttpRequest = httpRequest
		// Refresh Authorization header with each call...
		// Only authenticate if client library user wishes to.
		if connectionData.Authenticate {
			credentials := &hawk.Credentials{
				ID:   connectionData.Credentials.ClientId,
				Key:  connectionData.Credentials.AccessToken,
				Hash: sha256.New,
			}
			reqAuth := hawk.NewRequestAuth(httpRequest, credentials, 0)
			reqAuth.Ext, err = getExtHeader(connectionData.Credentials)
			if err != nil {
				return nil, nil, fmt.Errorf("Internal error: was not able to generate hawk ext header from provided credentials: %q\n%s", connectionData.Credentials, err)
			}
			httpRequest.Header.Set("Authorization", reqAuth.RequestHeader())
		}
		log.Printf("Making http request: %v", httpRequest)
		resp, err := httpClient.Do(httpRequest)
		return resp, err, nil
	}

	// Make HTTP API calls using an exponential backoff algorithm...
	callSummary.HttpResponse, callSummary.Attempts, err = httpbackoff.Retry(httpCall)

	if err != nil {
		return result, callSummary, err
	}

	// now read response into memory, so that we can return the body
	var body []byte
	body, err = ioutil.ReadAll(callSummary.HttpResponse.Body)

	if err != nil {
		return result, callSummary, err
	}

	callSummary.HttpResponseBody = string(body)

	// if result is passed in as nil, it means the API defines no response body
	// json
	if reflect.ValueOf(result).IsValid() && !reflect.ValueOf(result).IsNil() {
		err = json.Unmarshal([]byte(callSummary.HttpResponseBody), &result)
	}

	return result, callSummary, err
}

// SignedURL creates a signed URL using the given ConnectionData, where route
// is the url path relative to the BaseURL stored in the ConnectionData, query
// is the set of query string parameters, if any, and duration is the amount of
// time that the signed URL should remain valid for.
func (connectionData *ConnectionData) SignedURL(route string, query url.Values, duration time.Duration) (u *url.URL, err error) {
	u, err = setURL(connectionData, route, query)
	if err != nil {
		return
	}
	credentials := &hawk.Credentials{
		ID:   connectionData.Credentials.ClientId,
		Key:  connectionData.Credentials.AccessToken,
		Hash: sha256.New,
	}
	reqAuth, err := hawk.NewURLAuth(u.String(), credentials, duration)
	if err != nil {
		return
	}
	reqAuth.Ext, err = getExtHeader(connectionData.Credentials)
	if err != nil {
		return
	}
	bewitSignature := reqAuth.Bewit()
	if query == nil {
		query = url.Values{}
	}
	query.Set("bewit", bewitSignature)
	u.RawQuery = query.Encode()
	return
}

// getExtHeader generates the hawk ext header based on the authorizedScopes and
// the certificate used in the case of temporary credentials. The header is a
// base64 encoded json object with a "certificate" property set to the
// certificate of the temporary credentials and a "authorizedScopes" property
// set to the array of authorizedScopes, if provided.  If either "certificate"
// or "authorizedScopes" is not supplied, they will be omitted from the json
// result. If neither are provided, an empty string is returned, rather than a
// base64 encoded representation of "null" or "{}". Hawk interpets the empty
// string as meaning the ext header is not needed.
//
// See:
//   * http://docs.taskcluster.net/auth/authorized-scopes
//   * http://docs.taskcluster.net/auth/temporary-credentials
func getExtHeader(credentials *Credentials) (header string, err error) {
	ext := &ExtHeader{}
	if credentials.Certificate != "" {
		certObj := new(Certificate)
		err = json.Unmarshal([]byte(credentials.Certificate), certObj)
		if err != nil {
			return "", err
		}
		ext.Certificate = certObj
	}

	if credentials.AuthorizedScopes != nil {
		ext.AuthorizedScopes = &credentials.AuthorizedScopes
	}
	extJson, err := json.Marshal(ext)
	if err != nil {
		return "", err
	}
	if string(extJson) != "{}" {
		return base64.StdEncoding.EncodeToString(extJson), nil
	}
	return "", nil
}

// ExtHeader represents the authentication/authorization data that is encoded
// in the Ext HTTP header in outgoing Hawk HTTP requests.
type ExtHeader struct {
	Certificate *Certificate `json:"certificate,omitempty"`
	// use pointer to slice to distinguish between nil slice and empty slice
	AuthorizedScopes *[]string `json:"authorizedScopes,omitempty"`
}
<commit_msg>Doc correction<commit_after>package tcclient

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"reflect"
	"time"

	"github.com/taskcluster/httpbackoff"
	hawk "github.com/tent/hawk-go"
)

// CallSummary provides information about the underlying http request and
// response issued for a given API call.
type CallSummary struct {
	HttpRequest *http.Request
	// Keep a copy of request body in addition to the *http.Request, since
	// accessing the Body via the *http.Request object, you get a io.ReadCloser
	// - and after the request has been made, the body will have been read, and
	// the data lost... This way, it is still available after the api call
	// returns.
	HttpRequestBody string
	// The Go Type which is marshaled into json and used as the http request
	// body.
	HttpRequestObject interface{}
	HttpResponse      *http.Response
	// Keep a copy of response body in addition to the *http.Response, since
	// accessing the Body via the *http.Response object, you get a
	// io.ReadCloser - and after the response has been read once (to unmarshal
	// json into native go types) the data is lost... This way, it is still
	// available after the api call returns.
	HttpResponseBody string
	// Keep a record of how many http requests were attempted
	Attempts int
}

// utility function to create a URL object based on given data
func setURL(connectionData *ConnectionData, route string, query url.Values) (u *url.URL, err error) {
	u, err = url.Parse(connectionData.BaseURL + route)
	if err != nil {
		return nil, fmt.Errorf("Cannot parse url: '%v', is BaseURL (%v) set correctly?\n%v\n", connectionData.BaseURL+route, connectionData.BaseURL, err)
	}
	if query != nil {
		u.RawQuery = query.Encode()
	}
	return
}

// APICall is the generic REST API calling method which performs all REST API
// calls for this library.  Each auto-generated REST API method simply is a
// wrapper around this method, calling it with specific specific arguments.
func (connectionData *ConnectionData) APICall(payload interface{}, method, route string, result interface{}, query url.Values) (interface{}, *CallSummary, error) {
	callSummary := new(CallSummary)
	callSummary.HttpRequestObject = payload
	jsonPayload, err := json.Marshal(payload)
	if err != nil {
		return result, callSummary, err
	}
	callSummary.HttpRequestBody = string(jsonPayload)

	httpClient := &http.Client{}

	// function to perform http request - we call this using backoff library to
	// have exponential backoff in case of intermittent failures (e.g. network
	// blips or HTTP 5xx errors)
	httpCall := func() (*http.Response, error, error) {
		var ioReader io.Reader = nil
		if reflect.ValueOf(payload).IsValid() && !reflect.ValueOf(payload).IsNil() {
			ioReader = bytes.NewReader(jsonPayload)
		}
		u, err := setURL(connectionData, route, query)
		if err != nil {
			return nil, nil, fmt.Errorf("apiCall url cannot be parsed:\n%v\n", err)
		}
		httpRequest, err := http.NewRequest(method, u.String(), ioReader)
		if err != nil {
			return nil, nil, fmt.Errorf("Internal error: apiCall url cannot be parsed although thought to be valid: '%v', is the BaseURL (%v) set correctly?\n%v\n", u.String(), connectionData.BaseURL, err)
		}
		httpRequest.Header.Set("Content-Type", "application/json")
		callSummary.HttpRequest = httpRequest
		// Refresh Authorization header with each call...
		// Only authenticate if client library user wishes to.
		if connectionData.Authenticate {
			credentials := &hawk.Credentials{
				ID:   connectionData.Credentials.ClientId,
				Key:  connectionData.Credentials.AccessToken,
				Hash: sha256.New,
			}
			reqAuth := hawk.NewRequestAuth(httpRequest, credentials, 0)
			reqAuth.Ext, err = getExtHeader(connectionData.Credentials)
			if err != nil {
				return nil, nil, fmt.Errorf("Internal error: was not able to generate hawk ext header from provided credentials: %q\n%s", connectionData.Credentials, err)
			}
			httpRequest.Header.Set("Authorization", reqAuth.RequestHeader())
		}
		log.Printf("Making http request: %v", httpRequest)
		resp, err := httpClient.Do(httpRequest)
		return resp, err, nil
	}

	// Make HTTP API calls using an exponential backoff algorithm...
	callSummary.HttpResponse, callSummary.Attempts, err = httpbackoff.Retry(httpCall)

	if err != nil {
		return result, callSummary, err
	}

	// now read response into memory, so that we can return the body
	var body []byte
	body, err = ioutil.ReadAll(callSummary.HttpResponse.Body)

	if err != nil {
		return result, callSummary, err
	}

	callSummary.HttpResponseBody = string(body)

	// if result is passed in as nil, it means the API defines no response body
	// json
	if reflect.ValueOf(result).IsValid() && !reflect.ValueOf(result).IsNil() {
		err = json.Unmarshal([]byte(callSummary.HttpResponseBody), &result)
	}

	return result, callSummary, err
}

// SignedURL creates a signed URL using the given ConnectionData, where route
// is the url path relative to the BaseURL stored in the ConnectionData, query
// is the set of query string parameters, if any, and duration is the amount of
// time that the signed URL should remain valid for.
func (connectionData *ConnectionData) SignedURL(route string, query url.Values, duration time.Duration) (u *url.URL, err error) {
	u, err = setURL(connectionData, route, query)
	if err != nil {
		return
	}
	credentials := &hawk.Credentials{
		ID:   connectionData.Credentials.ClientId,
		Key:  connectionData.Credentials.AccessToken,
		Hash: sha256.New,
	}
	reqAuth, err := hawk.NewURLAuth(u.String(), credentials, duration)
	if err != nil {
		return
	}
	reqAuth.Ext, err = getExtHeader(connectionData.Credentials)
	if err != nil {
		return
	}
	bewitSignature := reqAuth.Bewit()
	if query == nil {
		query = url.Values{}
	}
	query.Set("bewit", bewitSignature)
	u.RawQuery = query.Encode()
	return
}

// getExtHeader generates the hawk ext header based on the authorizedScopes and
// the certificate used in the case of temporary credentials. The header is a
// base64 encoded json object with a "certificate" property set to the
// certificate of the temporary credentials and a "authorizedScopes" property
// set to the array of authorizedScopes, if provided.  If either "certificate"
// or "authorizedScopes" is not supplied, they will be omitted from the json
// result. If neither are provided, an empty string is returned, rather than a
// base64 encoded representation of "null" or "{}". Hawk interpets the empty
// string as meaning the ext header is not needed.
//
// See:
//   * http://docs.taskcluster.net/auth/authorized-scopes
//   * http://docs.taskcluster.net/auth/temporary-credentials
func getExtHeader(credentials *Credentials) (header string, err error) {
	ext := &ExtHeader{}
	if credentials.Certificate != "" {
		certObj := new(Certificate)
		err = json.Unmarshal([]byte(credentials.Certificate), certObj)
		if err != nil {
			return "", err
		}
		ext.Certificate = certObj
	}

	if credentials.AuthorizedScopes != nil {
		ext.AuthorizedScopes = &credentials.AuthorizedScopes
	}
	extJson, err := json.Marshal(ext)
	if err != nil {
		return "", err
	}
	if string(extJson) != "{}" {
		return base64.StdEncoding.EncodeToString(extJson), nil
	}
	return "", nil
}

// ExtHeader represents the authentication/authorization data that is encoded
// in the ext field inside the base64 decoded Authorization HTTP header in
// outgoing Hawk HTTP requests.
type ExtHeader struct {
	Certificate *Certificate `json:"certificate,omitempty"`
	// use pointer to slice to distinguish between nil slice and empty slice
	AuthorizedScopes *[]string `json:"authorizedScopes,omitempty"`
}
<|endoftext|>