<commit_before>package system // import "github.com/docker/docker/pkg/system"

import (
	"fmt"
	"syscall"
	"unsafe"

	"github.com/sirupsen/logrus"
	"golang.org/x/sys/windows"
)

const (
	OWNER_SECURITY_INFORMATION               = 0x00000001
	GROUP_SECURITY_INFORMATION               = 0x00000002
	DACL_SECURITY_INFORMATION                = 0x00000004
	SACL_SECURITY_INFORMATION                = 0x00000008
	LABEL_SECURITY_INFORMATION               = 0x00000010
	ATTRIBUTE_SECURITY_INFORMATION           = 0x00000020
	SCOPE_SECURITY_INFORMATION               = 0x00000040
	PROCESS_TRUST_LABEL_SECURITY_INFORMATION = 0x00000080
	ACCESS_FILTER_SECURITY_INFORMATION       = 0x00000100
	BACKUP_SECURITY_INFORMATION              = 0x00010000
	PROTECTED_DACL_SECURITY_INFORMATION      = 0x80000000
	PROTECTED_SACL_SECURITY_INFORMATION      = 0x40000000
	UNPROTECTED_DACL_SECURITY_INFORMATION    = 0x20000000
	UNPROTECTED_SACL_SECURITY_INFORMATION    = 0x10000000
)

const (
	SE_UNKNOWN_OBJECT_TYPE = iota
	SE_FILE_OBJECT
	SE_SERVICE
	SE_PRINTER
	SE_REGISTRY_KEY
	SE_LMSHARE
	SE_KERNEL_OBJECT
	SE_WINDOW_OBJECT
	SE_DS_OBJECT
	SE_DS_OBJECT_ALL
	SE_PROVIDER_DEFINED_OBJECT
	SE_WMIGUID_OBJECT
	SE_REGISTRY_WOW64_32KEY
)

const (
	SeTakeOwnershipPrivilege = "SeTakeOwnershipPrivilege"
)

const (
	ContainerAdministratorSidString = "S-1-5-93-2-1"
	ContainerUserSidString          = "S-1-5-93-2-2"
)

var (
	ntuserApiset                  = windows.NewLazyDLL("ext-ms-win-ntuser-window-l1-1-0")
	modadvapi32                   = windows.NewLazySystemDLL("advapi32.dll")
	procGetVersionExW             = modkernel32.NewProc("GetVersionExW")
	procSetNamedSecurityInfo      = modadvapi32.NewProc("SetNamedSecurityInfoW")
	procGetSecurityDescriptorDacl = modadvapi32.NewProc("GetSecurityDescriptorDacl")
)

// OSVersion is a wrapper for Windows version information
// https://msdn.microsoft.com/en-us/library/windows/desktop/ms724439(v=vs.85).aspx
type OSVersion struct {
	Version      uint32
	MajorVersion uint8
	MinorVersion uint8
	Build        uint16
}

// https://msdn.microsoft.com/en-us/library/windows/desktop/ms724833(v=vs.85).aspx
type osVersionInfoEx struct {
	OSVersionInfoSize uint32
	MajorVersion      uint32
	MinorVersion      uint32
	BuildNumber       uint32
	PlatformID        uint32
	CSDVersion        [128]uint16
	ServicePackMajor  uint16
	ServicePackMinor  uint16
	SuiteMask         uint16
	ProductType       byte
	Reserve           byte
}

// GetOSVersion gets the operating system version on Windows. Note that
// dockerd.exe must be manifested to get the correct version information.
func GetOSVersion() OSVersion {
	var err error
	osv := OSVersion{}
	osv.Version, err = windows.GetVersion()
	if err != nil {
		// GetVersion never fails.
		panic(err)
	}
	osv.MajorVersion = uint8(osv.Version & 0xFF)
	osv.MinorVersion = uint8(osv.Version >> 8 & 0xFF)
	osv.Build = uint16(osv.Version >> 16)
	return osv
}

func (osv OSVersion) ToString() string {
	return fmt.Sprintf("%d.%d.%d", osv.MajorVersion, osv.MinorVersion, osv.Build)
}

// IsWindowsClient returns true if the SKU is client
// @engine maintainers - this function should not be removed or modified as it
// is used to enforce licensing restrictions on Windows.
func IsWindowsClient() bool {
	osviex := &osVersionInfoEx{OSVersionInfoSize: 284}
	r1, _, err := procGetVersionExW.Call(uintptr(unsafe.Pointer(osviex)))
	if r1 == 0 {
		logrus.Warnf("GetVersionExW failed - assuming server SKU: %v", err)
		return false
	}
	const verNTWorkstation = 0x00000001
	return osviex.ProductType == verNTWorkstation
}

// Unmount is a platform-specific helper function to call
// the unmount syscall. Not supported on Windows
func Unmount(dest string) error {
	return nil
}

// CommandLineToArgv wraps the Windows syscall to turn a commandline into an argument array.
func CommandLineToArgv(commandLine string) ([]string, error) {
	var argc int32

	argsPtr, err := windows.UTF16PtrFromString(commandLine)
	if err != nil {
		return nil, err
	}

	argv, err := windows.CommandLineToArgv(argsPtr, &argc)
	if err != nil {
		return nil, err
	}
	defer windows.LocalFree(windows.Handle(uintptr(unsafe.Pointer(argv))))

	newArgs := make([]string, argc)
	for i, v := range (*argv)[:argc] {
		newArgs[i] = string(windows.UTF16ToString((*v)[:]))
	}

	return newArgs, nil
}

// HasWin32KSupport determines whether containers that depend on win32k can
// run on this machine. Win32k is the driver used to implement windowing.
func HasWin32KSupport() bool {
	// For now, check for ntuser API support on the host. In the future, a host
	// may support win32k in containers even if the host does not support ntuser
	// APIs.
	return ntuserApiset.Load() == nil
}

func SetNamedSecurityInfo(objectName *uint16, objectType uint32, securityInformation uint32, sidOwner *windows.SID, sidGroup *windows.SID, dacl *byte, sacl *byte) (result error) {
	r0, _, _ := syscall.Syscall9(procSetNamedSecurityInfo.Addr(), 7, uintptr(unsafe.Pointer(objectName)), uintptr(objectType), uintptr(securityInformation), uintptr(unsafe.Pointer(sidOwner)), uintptr(unsafe.Pointer(sidGroup)), uintptr(unsafe.Pointer(dacl)), uintptr(unsafe.Pointer(sacl)), 0, 0)
	if r0 != 0 {
		result = syscall.Errno(r0)
	}
	return
}

func GetSecurityDescriptorDacl(securityDescriptor *byte, daclPresent *uint32, dacl **byte, daclDefaulted *uint32) (result error) {
	r1, _, e1 := syscall.Syscall6(procGetSecurityDescriptorDacl.Addr(), 4, uintptr(unsafe.Pointer(securityDescriptor)), uintptr(unsafe.Pointer(daclPresent)), uintptr(unsafe.Pointer(dacl)), uintptr(unsafe.Pointer(daclDefaulted)), 0, 0)
	if r1 == 0 {
		if e1 != 0 {
			result = syscall.Errno(e1)
		} else {
			result = syscall.EINVAL
		}
	}
	return
}
<commit_msg>Deprecate pkg/system.GetOSVersion() in favor of hcsshim/osversion.Get()<commit_after>package system // import "github.com/docker/docker/pkg/system"

import (
	"fmt"
	"syscall"
	"unsafe"

	"github.com/Microsoft/hcsshim/osversion"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/windows"
)

const (
	OWNER_SECURITY_INFORMATION               = 0x00000001
	GROUP_SECURITY_INFORMATION               = 0x00000002
	DACL_SECURITY_INFORMATION                = 0x00000004
	SACL_SECURITY_INFORMATION                = 0x00000008
	LABEL_SECURITY_INFORMATION               = 0x00000010
	ATTRIBUTE_SECURITY_INFORMATION           = 0x00000020
	SCOPE_SECURITY_INFORMATION               = 0x00000040
	PROCESS_TRUST_LABEL_SECURITY_INFORMATION = 0x00000080
	ACCESS_FILTER_SECURITY_INFORMATION       = 0x00000100
	BACKUP_SECURITY_INFORMATION              = 0x00010000
	PROTECTED_DACL_SECURITY_INFORMATION      = 0x80000000
	PROTECTED_SACL_SECURITY_INFORMATION      = 0x40000000
	UNPROTECTED_DACL_SECURITY_INFORMATION    = 0x20000000
	UNPROTECTED_SACL_SECURITY_INFORMATION    = 0x10000000
)

const (
	SE_UNKNOWN_OBJECT_TYPE = iota
	SE_FILE_OBJECT
	SE_SERVICE
	SE_PRINTER
	SE_REGISTRY_KEY
	SE_LMSHARE
	SE_KERNEL_OBJECT
	SE_WINDOW_OBJECT
	SE_DS_OBJECT
	SE_DS_OBJECT_ALL
	SE_PROVIDER_DEFINED_OBJECT
	SE_WMIGUID_OBJECT
	SE_REGISTRY_WOW64_32KEY
)

const (
	SeTakeOwnershipPrivilege = "SeTakeOwnershipPrivilege"
)

const (
	ContainerAdministratorSidString = "S-1-5-93-2-1"
	ContainerUserSidString          = "S-1-5-93-2-2"
)

var (
	ntuserApiset                  = windows.NewLazyDLL("ext-ms-win-ntuser-window-l1-1-0")
	modadvapi32                   = windows.NewLazySystemDLL("advapi32.dll")
	procGetVersionExW             = modkernel32.NewProc("GetVersionExW")
	procSetNamedSecurityInfo      = modadvapi32.NewProc("SetNamedSecurityInfoW")
	procGetSecurityDescriptorDacl = modadvapi32.NewProc("GetSecurityDescriptorDacl")
)

// OSVersion is a wrapper for Windows version information
// https://msdn.microsoft.com/en-us/library/windows/desktop/ms724439(v=vs.85).aspx
type OSVersion osversion.OSVersion

// https://msdn.microsoft.com/en-us/library/windows/desktop/ms724833(v=vs.85).aspx
type osVersionInfoEx struct {
	OSVersionInfoSize uint32
	MajorVersion      uint32
	MinorVersion      uint32
	BuildNumber       uint32
	PlatformID        uint32
	CSDVersion        [128]uint16
	ServicePackMajor  uint16
	ServicePackMinor  uint16
	SuiteMask         uint16
	ProductType       byte
	Reserve           byte
}

// GetOSVersion gets the operating system version on Windows. Note that
// dockerd.exe must be manifested to get the correct version information.
// Deprecated: use github.com/Microsoft/hcsshim/osversion.Get() instead
func GetOSVersion() OSVersion {
	return OSVersion(osversion.Get())
}

func (osv OSVersion) ToString() string {
	return fmt.Sprintf("%d.%d.%d", osv.MajorVersion, osv.MinorVersion, osv.Build)
}

// IsWindowsClient returns true if the SKU is client
// @engine maintainers - this function should not be removed or modified as it
// is used to enforce licensing restrictions on Windows.
func IsWindowsClient() bool {
	osviex := &osVersionInfoEx{OSVersionInfoSize: 284}
	r1, _, err := procGetVersionExW.Call(uintptr(unsafe.Pointer(osviex)))
	if r1 == 0 {
		logrus.Warnf("GetVersionExW failed - assuming server SKU: %v", err)
		return false
	}
	const verNTWorkstation = 0x00000001
	return osviex.ProductType == verNTWorkstation
}

// Unmount is a platform-specific helper function to call
// the unmount syscall. Not supported on Windows
func Unmount(dest string) error {
	return nil
}

// CommandLineToArgv wraps the Windows syscall to turn a commandline into an argument array.
func CommandLineToArgv(commandLine string) ([]string, error) {
	var argc int32

	argsPtr, err := windows.UTF16PtrFromString(commandLine)
	if err != nil {
		return nil, err
	}

	argv, err := windows.CommandLineToArgv(argsPtr, &argc)
	if err != nil {
		return nil, err
	}
	defer windows.LocalFree(windows.Handle(uintptr(unsafe.Pointer(argv))))

	newArgs := make([]string, argc)
	for i, v := range (*argv)[:argc] {
		newArgs[i] = string(windows.UTF16ToString((*v)[:]))
	}

	return newArgs, nil
}

// HasWin32KSupport determines whether containers that depend on win32k can
// run on this machine. Win32k is the driver used to implement windowing.
func HasWin32KSupport() bool {
	// For now, check for ntuser API support on the host. In the future, a host
	// may support win32k in containers even if the host does not support ntuser
	// APIs.
	return ntuserApiset.Load() == nil
}

func SetNamedSecurityInfo(objectName *uint16, objectType uint32, securityInformation uint32, sidOwner *windows.SID, sidGroup *windows.SID, dacl *byte, sacl *byte) (result error) {
	r0, _, _ := syscall.Syscall9(procSetNamedSecurityInfo.Addr(), 7, uintptr(unsafe.Pointer(objectName)), uintptr(objectType), uintptr(securityInformation), uintptr(unsafe.Pointer(sidOwner)), uintptr(unsafe.Pointer(sidGroup)), uintptr(unsafe.Pointer(dacl)), uintptr(unsafe.Pointer(sacl)), 0, 0)
	if r0 != 0 {
		result = syscall.Errno(r0)
	}
	return
}

func GetSecurityDescriptorDacl(securityDescriptor *byte, daclPresent *uint32, dacl **byte, daclDefaulted *uint32) (result error) {
	r1, _, e1 := syscall.Syscall6(procGetSecurityDescriptorDacl.Addr(), 4, uintptr(unsafe.Pointer(securityDescriptor)), uintptr(unsafe.Pointer(daclPresent)), uintptr(unsafe.Pointer(dacl)), uintptr(unsafe.Pointer(daclDefaulted)), 0, 0)
	if r1 == 0 {
		if e1 != 0 {
			result = syscall.Errno(e1)
		} else {
			result = syscall.EINVAL
		}
	}
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package transformers

import (
	"sigs.k8s.io/kustomize/pkg/gvk"
	"sigs.k8s.io/kustomize/pkg/resmap"
	"sigs.k8s.io/kustomize/pkg/transformers/config"
)

type namespaceTransformer struct {
	namespace        string
	fieldSpecsToUse  []config.FieldSpec
	fieldSpecsToSkip []config.FieldSpec
}

var _ Transformer = &namespaceTransformer{}

// NewNamespaceTransformer construct a namespaceTransformer.
func NewNamespaceTransformer(ns string, cf []config.FieldSpec) Transformer {
	if len(ns) == 0 {
		return NewNoOpTransformer()
	}
	var skip []config.FieldSpec
	for _, g := range gvk.ClusterLevelGvks() {
		skip = append(skip, config.FieldSpec{Gvk: g})
	}
	return &namespaceTransformer{
		namespace:        ns,
		fieldSpecsToUse:  cf,
		fieldSpecsToSkip: skip,
	}
}

// Transform adds the namespace.
func (o *namespaceTransformer) Transform(m resmap.ResMap) error {
	mf := resmap.ResMap{}

	for id := range m {
		found := false
		for _, path := range o.fieldSpecsToSkip {
			if id.Gvk().IsSelected(&path.Gvk) {
				found = true
				break
			}
		}
		if !found {
			mf[id] = m[id]
			delete(m, id)
		}
	}

	for id := range mf {
		objMap := mf[id].Map()
		for _, path := range o.fieldSpecsToUse {
			if !id.Gvk().IsSelected(&path.Gvk) {
				continue
			}
			// make sure the object is non empty
			if len(objMap) > 0 {
				err := mutateField(
					objMap, path.PathSlice(), path.CreateIfNotPresent,
					func(_ interface{}) (interface{}, error) {
						return o.namespace, nil
					})
				if err != nil {
					return err
				}
			}
			newid := id.CopyWithNewNamespace(o.namespace)
			m[newid] = mf[id]
		}

	}
	o.updateClusterRoleBinding(m)
	return nil
}

func (o *namespaceTransformer) updateClusterRoleBinding(m resmap.ResMap) {
	saMap := map[string]bool{}
	for id := range m {
		if id.Gvk().Equals(gvk.Gvk{Version: "v1", Kind: "ServiceAccount"}) {
			saMap[id.Name()] = true
		}
	}

	for id := range m {
		if id.Gvk().Kind != "ClusterRoleBinding" && id.Gvk().Kind != "RoleBinding" {
			continue
		}
		objMap := m[id].Map()
		subjects := objMap["subjects"].([]interface{})
		for i := range subjects {
			subject := subjects[i].(map[string]interface{})
			kind, foundk := subject["kind"]
			name, foundn := subject["name"]
			if !foundk || !foundn || kind.(string) != "ServiceAccount" {
				continue
			}
			// a ServiceAccount named “default” exists in every active namespace
			if name.(string) == "default" || saMap[name.(string)] {
				subject := subjects[i].(map[string]interface{})
				mutateField(subject, []string{"namespace"}, true, func(_ interface{}) (interface{}, error) {
					return o.namespace, nil
				})
				subjects[i] = subject
			}
		}
		objMap["subjects"] = subjects
	}
}
<commit_msg>fix namespace transformer for cluster-scoped resources<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package transformers

import (
	"sigs.k8s.io/kustomize/pkg/gvk"
	"sigs.k8s.io/kustomize/pkg/resmap"
	"sigs.k8s.io/kustomize/pkg/transformers/config"
)

type namespaceTransformer struct {
	namespace        string
	fieldSpecsToUse  []config.FieldSpec
	fieldSpecsToSkip []config.FieldSpec
}

var _ Transformer = &namespaceTransformer{}

// NewNamespaceTransformer construct a namespaceTransformer.
func NewNamespaceTransformer(ns string, cf []config.FieldSpec) Transformer {
	if len(ns) == 0 {
		return NewNoOpTransformer()
	}
	var skip []config.FieldSpec
	for _, g := range gvk.ClusterLevelGvks() {
		skip = append(skip, config.FieldSpec{Gvk: g})
	}
	return &namespaceTransformer{
		namespace:        ns,
		fieldSpecsToUse:  cf,
		fieldSpecsToSkip: skip,
	}
}

// Transform adds the namespace.
func (o *namespaceTransformer) Transform(m resmap.ResMap) error {
	mf := o.filterResmap(m)

	for id := range mf {
		objMap := mf[id].Map()
		for _, path := range o.fieldSpecsToUse {
			switch path.Path {
			// Special casing .metadata.namespace since it is a common metadata field across all runtime.Object
			// We should add namespace if it's namespaced resource; otherwise, we should not.
			case "metadata/namespace":
				if id.Gvk().IsSelected(&path.Gvk) && !id.Gvk().IsClusterKind() {
					if len(objMap) > 0 {
						err := mutateField(
							objMap, path.PathSlice(), path.CreateIfNotPresent,
							func(_ interface{}) (interface{}, error) {
								return o.namespace, nil
							})
						if err != nil {
							return err
						}
					}
				}
			default:
				if !id.Gvk().IsSelected(&path.Gvk) {
					continue
				}
				// make sure the object is non empty
				if len(objMap) > 0 {
					err := mutateField(
						objMap, path.PathSlice(), path.CreateIfNotPresent,
						func(_ interface{}) (interface{}, error) {
							return o.namespace, nil
						})
					if err != nil {
						return err
					}
				}
			}

			if !id.Gvk().IsClusterKind() {
				newid := id.CopyWithNewNamespace(o.namespace)
				m[newid] = mf[id]
			} else {
				m[id] = mf[id]
			}
		}
	}
	o.updateClusterRoleBinding(m)
	return nil
}

func (o *namespaceTransformer) filterResmap(m resmap.ResMap) resmap.ResMap {
	mf := resmap.ResMap{}
	for id := range m {
		found := false
		for _, path := range o.fieldSpecsToSkip {
			if id.Gvk().IsSelected(&path.Gvk) {
				found = true
				mf[id] = m[id]
				delete(m, id)
			}
		}
		if !found {
			mf[id] = m[id]
			delete(m, id)
		}
	}
	return mf
}

func (o *namespaceTransformer) updateClusterRoleBinding(m resmap.ResMap) {
	saMap := map[string]bool{}
	for id := range m {
		if id.Gvk().Equals(gvk.Gvk{Version: "v1", Kind: "ServiceAccount"}) {
			saMap[id.Name()] = true
		}
	}

	for id := range m {
		if id.Gvk().Kind != "ClusterRoleBinding" && id.Gvk().Kind != "RoleBinding" {
			continue
		}
		objMap := m[id].Map()
		subjects := objMap["subjects"].([]interface{})
		for i := range subjects {
			subject := subjects[i].(map[string]interface{})
			kind, foundk := subject["kind"]
			name, foundn := subject["name"]
			if !foundk || !foundn || kind.(string) != "ServiceAccount" {
				continue
			}
			// a ServiceAccount named “default” exists in every active namespace
			if name.(string) == "default" || saMap[name.(string)] {
				subject := subjects[i].(map[string]interface{})
				mutateField(subject, []string{"namespace"}, true, func(_ interface{}) (interface{}, error) {
					return o.namespace, nil
				})
				subjects[i] = subject
			}
		}
		objMap["subjects"] = subjects
	}
}
<|endoftext|>


<commit_before>//go:build !providerless
// +build !providerless

/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awsebs

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"time"

	"k8s.io/klog/v2"
	"k8s.io/mount-utils"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/kubernetes/pkg/volume"
	volumeutil "k8s.io/kubernetes/pkg/volume/util"
	"k8s.io/legacy-cloud-providers/aws"
)

type awsElasticBlockStoreAttacher struct {
	host       volume.VolumeHost
	awsVolumes aws.Volumes
}

var _ volume.Attacher = &awsElasticBlockStoreAttacher{}

var _ volume.DeviceMounter = &awsElasticBlockStoreAttacher{}

var _ volume.AttachableVolumePlugin = &awsElasticBlockStorePlugin{}

var _ volume.DeviceMountableVolumePlugin = &awsElasticBlockStorePlugin{}

func (plugin *awsElasticBlockStorePlugin) NewAttacher() (volume.Attacher, error) {
	awsCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	return &awsElasticBlockStoreAttacher{
		host:       plugin.host,
		awsVolumes: awsCloud,
	}, nil
}

func (plugin *awsElasticBlockStorePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	return plugin.NewAttacher()
}

func (plugin *awsElasticBlockStorePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	return mounter.GetMountRefs(deviceMountPath)
}

func (attacher *awsElasticBlockStoreAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	volumeID := aws.KubernetesVolumeID(volumeSource.VolumeID)

	// awsCloud.AttachDisk checks if disk is already attached to node and
	// succeeds in that case, so no need to do that separately.
	devicePath, err := attacher.awsVolumes.AttachDisk(volumeID, nodeName)
	if err != nil {
		klog.Errorf("Error attaching volume %q to node %q: %+v", volumeID, nodeName, err)
		return "", err
	}

	return devicePath, nil
}

func (attacher *awsElasticBlockStoreAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {

	klog.Warningf("Attacher.VolumesAreAttached called for node %q - Please use BulkVerifyVolumes for AWS", nodeName)
	volumeNodeMap := map[types.NodeName][]*volume.Spec{
		nodeName: specs,
	}
	nodeVolumesResult := make(map[*volume.Spec]bool)
	nodesVerificationMap, err := attacher.BulkVerifyVolumes(volumeNodeMap)
	if err != nil {
		klog.Errorf("Attacher.VolumesAreAttached - error checking volumes for node %q with %v", nodeName, err)
		return nodeVolumesResult, err
	}

	if result, ok := nodesVerificationMap[nodeName]; ok {
		return result, nil
	}
	return nodeVolumesResult, nil
}

func (attacher *awsElasticBlockStoreAttacher) BulkVerifyVolumes(volumesByNode map[types.NodeName][]*volume.Spec) (map[types.NodeName]map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[types.NodeName]map[*volume.Spec]bool)
	diskNamesByNode := make(map[types.NodeName][]aws.KubernetesVolumeID)
	volumeSpecMap := make(map[aws.KubernetesVolumeID]*volume.Spec)

	for nodeName, volumeSpecs := range volumesByNode {
		for _, volumeSpec := range volumeSpecs {
			volumeSource, _, err := getVolumeSource(volumeSpec)

			if err != nil {
				klog.Errorf("Error getting volume (%q) source : %v", volumeSpec.Name(), err)
				continue
			}

			name := aws.KubernetesVolumeID(volumeSource.VolumeID)
			diskNamesByNode[nodeName] = append(diskNamesByNode[nodeName], name)

			nodeDisk, nodeDiskExists := volumesAttachedCheck[nodeName]

			if !nodeDiskExists {
				nodeDisk = make(map[*volume.Spec]bool)
			}
			nodeDisk[volumeSpec] = true
			volumeSpecMap[name] = volumeSpec
			volumesAttachedCheck[nodeName] = nodeDisk
		}
	}
	attachedResult, err := attacher.awsVolumes.DisksAreAttached(diskNamesByNode)

	if err != nil {
		klog.Errorf("Error checking if volumes are attached to nodes err = %v", err)
		return volumesAttachedCheck, err
	}

	for nodeName, nodeDisks := range attachedResult {
		for diskName, attached := range nodeDisks {
			if !attached {
				spec := volumeSpecMap[diskName]
				setNodeDisk(volumesAttachedCheck, spec, nodeName, false)
			}
		}
	}

	return volumesAttachedCheck, nil
}

func (attacher *awsElasticBlockStoreAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	volumeID := volumeSource.VolumeID
	partition := ""
	if volumeSource.Partition != 0 {
		partition = strconv.Itoa(int(volumeSource.Partition))
	}

	if devicePath == "" {
		return "", fmt.Errorf("waitForAttach failed for AWS Volume %q: devicePath is empty", volumeID)
	}

	ticker := time.NewTicker(checkSleepDuration)
	defer ticker.Stop()
	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for {
		select {
		case <-ticker.C:
			klog.V(5).Infof("Checking AWS Volume %q is attached at devicePath %q.", volumeID, devicePath)
			path, err := attacher.getDevicePath(volumeSource.VolumeID, partition, devicePath)
			if err != nil {
				// Log error, if any, and continue checking periodically. See issue #11321
				klog.Errorf("Error verifying AWS Volume (%q) is attached at devicePath %q: %v", volumeID, devicePath, err)
			} else if path != "" {
				// A device path has successfully been created for the PD
				klog.Infof("Successfully found attached AWS Volume %q at path %q.", volumeID, path)
				return path, nil
			}
		case <-timer.C:
			return "", fmt.Errorf("could not find attached AWS Volume %q. Timeout waiting for mount paths to be created", volumeID)
		}
	}
}

func (attacher *awsElasticBlockStoreAttacher) GetDeviceMountPath(
	spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return makeGlobalPDPath(attacher.host, aws.KubernetesVolumeID(volumeSource.VolumeID)), nil
}

// FIXME: this method can be further pruned.
func (attacher *awsElasticBlockStoreAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, _ volume.DeviceMounterArgs) error {
	mounter := attacher.host.GetMounter(awsElasticBlockStorePluginName)
	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if os.IsNotExist(err) {
			dir := deviceMountPath
			if runtime.GOOS == "windows" {
				// On Windows, FormatAndMount will mklink (create a symbolic link) at deviceMountPath later, so don't create a
				// directory at deviceMountPath now. Otherwise mklink will error: "Cannot create a file when that file already exists".
				// Instead, create the parent of deviceMountPath. For example when deviceMountPath is:
				// C:\var\lib\kubelet\plugins\kubernetes.io\aws-ebs\mounts\aws\us-west-2b\vol-xxx
				// create us-west-2b. FormatAndMount will make vol-xxx a symlink to the drive (e.g. D:\)
				dir = filepath.Dir(deviceMountPath)
			}
			if err := os.MkdirAll(dir, 0750); err != nil {
				return fmt.Errorf("making dir %s failed with %s", dir, err)
			}
			notMnt = true
		} else {
			return err
		}
	}

	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return err
	}

	options := []string{}
	if readOnly {
		options = append(options, "ro")
	}
	if notMnt {
		diskMounter := volumeutil.NewSafeFormatAndMountFromHost(awsElasticBlockStorePluginName, attacher.host)
		mountOptions := volumeutil.MountOptionFromSpec(spec, options...)
		err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions)
		if err != nil {
			os.Remove(deviceMountPath)
			return err
		}
	}
	return nil
}

type awsElasticBlockStoreDetacher struct {
	mounter    mount.Interface
	awsVolumes aws.Volumes
}

var _ volume.Detacher = &awsElasticBlockStoreDetacher{}

var _ volume.DeviceUnmounter = &awsElasticBlockStoreDetacher{}

func (plugin *awsElasticBlockStorePlugin) NewDetacher() (volume.Detacher, error) {
	awsCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	return &awsElasticBlockStoreDetacher{
		mounter:    plugin.host.GetMounter(plugin.GetPluginName()),
		awsVolumes: awsCloud,
	}, nil
}

func (plugin *awsElasticBlockStorePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return plugin.NewDetacher()
}

func (detacher *awsElasticBlockStoreDetacher) Detach(volumeName string, nodeName types.NodeName) error {
	volumeID := aws.KubernetesVolumeID(path.Base(volumeName))

	if _, err := detacher.awsVolumes.DetachDisk(volumeID, nodeName); err != nil {
		klog.Errorf("Error detaching volumeID %q: %v", volumeID, err)
		return err
	}
	return nil
}

func (detacher *awsElasticBlockStoreDetacher) UnmountDevice(deviceMountPath string) error {
	return mount.CleanupMountPoint(deviceMountPath, detacher.mounter, false)
}

func (plugin *awsElasticBlockStorePlugin) CanAttach(spec *volume.Spec) (bool, error) {
	return true, nil
}

func (plugin *awsElasticBlockStorePlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}

func setNodeDisk(
	nodeDiskMap map[types.NodeName]map[*volume.Spec]bool,
	volumeSpec *volume.Spec,
	nodeName types.NodeName,
	check bool) {

	volumeMap := nodeDiskMap[nodeName]
	if volumeMap == nil {
		volumeMap = make(map[*volume.Spec]bool)
		nodeDiskMap[nodeName] = volumeMap
	}
	volumeMap[volumeSpec] = check
}
<commit_msg>optimize ifelse<commit_after>//go:build !providerless
// +build !providerless

/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awsebs

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"time"

	"k8s.io/klog/v2"
	"k8s.io/mount-utils"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/kubernetes/pkg/volume"
	volumeutil "k8s.io/kubernetes/pkg/volume/util"
	"k8s.io/legacy-cloud-providers/aws"
)

type awsElasticBlockStoreAttacher struct {
	host       volume.VolumeHost
	awsVolumes aws.Volumes
}

var _ volume.Attacher = &awsElasticBlockStoreAttacher{}

var _ volume.DeviceMounter = &awsElasticBlockStoreAttacher{}

var _ volume.AttachableVolumePlugin = &awsElasticBlockStorePlugin{}

var _ volume.DeviceMountableVolumePlugin = &awsElasticBlockStorePlugin{}

func (plugin *awsElasticBlockStorePlugin) NewAttacher() (volume.Attacher, error) {
	awsCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	return &awsElasticBlockStoreAttacher{
		host:       plugin.host,
		awsVolumes: awsCloud,
	}, nil
}

func (plugin *awsElasticBlockStorePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
	return plugin.NewAttacher()
}

func (plugin *awsElasticBlockStorePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
	mounter := plugin.host.GetMounter(plugin.GetPluginName())
	return mounter.GetMountRefs(deviceMountPath)
}

func (attacher *awsElasticBlockStoreAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	volumeID := aws.KubernetesVolumeID(volumeSource.VolumeID)

	// awsCloud.AttachDisk checks if disk is already attached to node and
	// succeeds in that case, so no need to do that separately.
	devicePath, err := attacher.awsVolumes.AttachDisk(volumeID, nodeName)
	if err != nil {
		klog.Errorf("Error attaching volume %q to node %q: %+v", volumeID, nodeName, err)
		return "", err
	}

	return devicePath, nil
}

func (attacher *awsElasticBlockStoreAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) {

	klog.Warningf("Attacher.VolumesAreAttached called for node %q - Please use BulkVerifyVolumes for AWS", nodeName)
	volumeNodeMap := map[types.NodeName][]*volume.Spec{
		nodeName: specs,
	}
	nodeVolumesResult := make(map[*volume.Spec]bool)
	nodesVerificationMap, err := attacher.BulkVerifyVolumes(volumeNodeMap)
	if err != nil {
		klog.Errorf("Attacher.VolumesAreAttached - error checking volumes for node %q with %v", nodeName, err)
		return nodeVolumesResult, err
	}

	if result, ok := nodesVerificationMap[nodeName]; ok {
		return result, nil
	}
	return nodeVolumesResult, nil
}

func (attacher *awsElasticBlockStoreAttacher) BulkVerifyVolumes(volumesByNode map[types.NodeName][]*volume.Spec) (map[types.NodeName]map[*volume.Spec]bool, error) {
	volumesAttachedCheck := make(map[types.NodeName]map[*volume.Spec]bool)
	diskNamesByNode := make(map[types.NodeName][]aws.KubernetesVolumeID)
	volumeSpecMap := make(map[aws.KubernetesVolumeID]*volume.Spec)

	for nodeName, volumeSpecs := range volumesByNode {
		for _, volumeSpec := range volumeSpecs {
			volumeSource, _, err := getVolumeSource(volumeSpec)

			if err != nil {
				klog.Errorf("Error getting volume (%q) source : %v", volumeSpec.Name(), err)
				continue
			}

			name := aws.KubernetesVolumeID(volumeSource.VolumeID)
			diskNamesByNode[nodeName] = append(diskNamesByNode[nodeName], name)

			nodeDisk, nodeDiskExists := volumesAttachedCheck[nodeName]

			if !nodeDiskExists {
				nodeDisk = make(map[*volume.Spec]bool)
			}
			nodeDisk[volumeSpec] = true
			volumeSpecMap[name] = volumeSpec
			volumesAttachedCheck[nodeName] = nodeDisk
		}
	}
	attachedResult, err := attacher.awsVolumes.DisksAreAttached(diskNamesByNode)

	if err != nil {
		klog.Errorf("Error checking if volumes are attached to nodes err = %v", err)
		return volumesAttachedCheck, err
	}

	for nodeName, nodeDisks := range attachedResult {
		for diskName, attached := range nodeDisks {
			if !attached {
				spec := volumeSpecMap[diskName]
				setNodeDisk(volumesAttachedCheck, spec, nodeName, false)
			}
		}
	}

	return volumesAttachedCheck, nil
}

func (attacher *awsElasticBlockStoreAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	volumeID := volumeSource.VolumeID
	partition := ""
	if volumeSource.Partition != 0 {
		partition = strconv.Itoa(int(volumeSource.Partition))
	}

	if devicePath == "" {
		return "", fmt.Errorf("waitForAttach failed for AWS Volume %q: devicePath is empty", volumeID)
	}

	ticker := time.NewTicker(checkSleepDuration)
	defer ticker.Stop()
	timer := time.NewTimer(timeout)
	defer timer.Stop()

	for {
		select {
		case <-ticker.C:
			klog.V(5).Infof("Checking AWS Volume %q is attached at devicePath %q.", volumeID, devicePath)
			path, err := attacher.getDevicePath(volumeSource.VolumeID, partition, devicePath)
			if err != nil {
				// Log error, if any, and continue checking periodically. See issue #11321
				klog.Errorf("Error verifying AWS Volume (%q) is attached at devicePath %q: %v", volumeID, devicePath, err)
			} else if path != "" {
				// A device path has successfully been created for the PD
				klog.Infof("Successfully found attached AWS Volume %q at path %q.", volumeID, path)
				return path, nil
			}
		case <-timer.C:
			return "", fmt.Errorf("could not find attached AWS Volume %q. Timeout waiting for mount paths to be created", volumeID)
		}
	}
}

func (attacher *awsElasticBlockStoreAttacher) GetDeviceMountPath(
	spec *volume.Spec) (string, error) {
	volumeSource, _, err := getVolumeSource(spec)
	if err != nil {
		return "", err
	}

	return makeGlobalPDPath(attacher.host, aws.KubernetesVolumeID(volumeSource.VolumeID)), nil
}

// FIXME: this method can be further pruned.
func (attacher *awsElasticBlockStoreAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, _ volume.DeviceMounterArgs) error {
	mounter := attacher.host.GetMounter(awsElasticBlockStorePluginName)
	notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		dir := deviceMountPath
		if runtime.GOOS == "windows" {
			// On Windows, FormatAndMount will mklink (create a symbolic link) at deviceMountPath later, so don't create a
			// directory at deviceMountPath now. Otherwise mklink will error: "Cannot create a file when that file already exists".
			// Instead, create the parent of deviceMountPath. For example when deviceMountPath is:
			// C:\var\lib\kubelet\plugins\kubernetes.io\aws-ebs\mounts\aws\us-west-2b\vol-xxx
			// create us-west-2b. FormatAndMount will make vol-xxx a symlink to the drive (e.g. D:\)
			dir = filepath.Dir(deviceMountPath)
		}
		if err := os.MkdirAll(dir, 0750); err != nil {
			return fmt.Errorf("making dir %s failed with %s", dir, err)
		}
		notMnt = true
	}

	volumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return err
	}

	options := []string{}
	if readOnly {
		options = append(options, "ro")
	}
	if notMnt {
		diskMounter := volumeutil.NewSafeFormatAndMountFromHost(awsElasticBlockStorePluginName, attacher.host)
		mountOptions := volumeutil.MountOptionFromSpec(spec, options...)
		err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions)
		if err != nil {
			os.Remove(deviceMountPath)
			return err
		}
	}
	return nil
}

type awsElasticBlockStoreDetacher struct {
	mounter    mount.Interface
	awsVolumes aws.Volumes
}

var _ volume.Detacher = &awsElasticBlockStoreDetacher{}

var _ volume.DeviceUnmounter = &awsElasticBlockStoreDetacher{}

func (plugin *awsElasticBlockStorePlugin) NewDetacher() (volume.Detacher, error) {
	awsCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
	if err != nil {
		return nil, err
	}

	return &awsElasticBlockStoreDetacher{
		mounter:    plugin.host.GetMounter(plugin.GetPluginName()),
		awsVolumes: awsCloud,
	}, nil
}

func (plugin *awsElasticBlockStorePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
	return plugin.NewDetacher()
}

func (detacher *awsElasticBlockStoreDetacher) Detach(volumeName string, nodeName types.NodeName) error {
	volumeID := aws.KubernetesVolumeID(path.Base(volumeName))

	if _, err := detacher.awsVolumes.DetachDisk(volumeID, nodeName); err != nil {
		klog.Errorf("Error detaching volumeID %q: %v", volumeID, err)
		return err
	}
	return nil
}

func (detacher *awsElasticBlockStoreDetacher) UnmountDevice(deviceMountPath string) error {
	return mount.CleanupMountPoint(deviceMountPath, detacher.mounter, false)
}

func (plugin *awsElasticBlockStorePlugin) CanAttach(spec *volume.Spec) (bool, error) {
	return true, nil
}

func (plugin *awsElasticBlockStorePlugin) CanDeviceMount(spec *volume.Spec) (bool, error) {
	return true, nil
}

func setNodeDisk(
	nodeDiskMap map[types.NodeName]map[*volume.Spec]bool,
	volumeSpec *volume.Spec,
	nodeName types.NodeName,
	check bool) {

	volumeMap := nodeDiskMap[nodeName]
	if volumeMap == nil {
		volumeMap = make(map[*volume.Spec]bool)
		nodeDiskMap[nodeName] = volumeMap
	}
	volumeMap[volumeSpec] = check
}
<|endoftext|>


<commit_before>// errchk $G -e $D/$F.go

// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type xyz struct {
    ch chan
} // ERROR "unexpected } in channel type"

func Foo(y chan) { // ERROR "unexpected \) in channel type"
}

func Bar(x chan, y int) { // ERROR "unexpected comma in channel type"
}
<commit_msg>test: match gccgo error messages for syntax/chan.go.<commit_after>// errchk $G -e $D/$F.go

// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type xyz struct {
    ch chan
} // ERROR "unexpected .*}.* in channel type"

func Foo(y chan) { // ERROR "unexpected .*\).* in channel type"
}

func Bar(x chan, y int) { // ERROR "unexpected comma in channel type"
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru-admin authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strconv"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/quota"
)

type viewUserQuota struct{}

func (viewUserQuota) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "view-user-quota",
		MinArgs: 1,
		Usage:   "view-user-quota <user-email>",
		Desc:    "Displays the current usage and limit of the user",
	}
}

func (viewUserQuota) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "User: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Apps usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type changeUserQuota struct{}

func (changeUserQuota) Info() *cmd.Info {
	desc := `Changes the limit of apps that a user can create

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "change-user-quota",
		MinArgs: 2,
		Usage:   "change-user-quota <user-email> <new-limit>",
		Desc:    desc,
	}
}

func (changeUserQuota) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	body := bytes.NewBufferString("limit=" + limit)
	request, _ := http.NewRequest("POST", url, body)
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

type viewAppQuota struct{}

func (viewAppQuota) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "view-app-quota",
		MinArgs: 1,
		Usage:   "view-app-quota <app-name>",
		Desc:    "Displays the current usage and limit of the given app",
	}
}

func (viewAppQuota) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/apps/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "App: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Units usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type changeAppQuota struct{}

func (changeAppQuota) Info() *cmd.Info {
	desc := `Changes the limit of units that an app can have

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "change-app-quota",
		MinArgs: 2,
		Usage:   "change-app-quota <app-name> <new-limit>",
		Desc:    desc,
	}
}

func (changeAppQuota) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/apps/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	body := bytes.NewBufferString("limit=" + limit)
	request, _ := http.NewRequest("POST", url, body)
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

func parseLimit(value string) (string, error) {
	if value == "unlimited" {
		return "-1", nil
	}
	n, err := strconv.Atoi(value)
	if err != nil {
		return "", errors.New(`invalid limit. It must be either an integer or "unlimited"`)
	}
	return strconv.Itoa(n), nil
}
<commit_msg>The copyright year of the quota file was updated. Related to #1318<commit_after>// Copyright 2015 tsuru-admin authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strconv"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/quota"
)

type viewUserQuota struct{}

func (viewUserQuota) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "view-user-quota",
		MinArgs: 1,
		Usage:   "view-user-quota <user-email>",
		Desc:    "Displays the current usage and limit of the user",
	}
}

func (viewUserQuota) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "User: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Apps usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type changeUserQuota struct{}

func (changeUserQuota) Info() *cmd.Info {
	desc := `Changes the limit of apps that a user can create

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "change-user-quota",
		MinArgs: 2,
		Usage:   "change-user-quota <user-email> <new-limit>",
		Desc:    desc,
	}
}

func (changeUserQuota) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	body := bytes.NewBufferString("limit=" + limit)
	request, _ := http.NewRequest("POST", url, body)
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

type viewAppQuota struct{}

func (viewAppQuota) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "view-app-quota",
		MinArgs: 1,
		Usage:   "view-app-quota <app-name>",
		Desc:    "Displays the current usage and limit of the given app",
	}
}

func (viewAppQuota) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/apps/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "App: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Units usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type changeAppQuota struct{}

func (changeAppQuota) Info() *cmd.Info {
	desc := `Changes the limit of units that an app can have

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "change-app-quota",
		MinArgs: 2,
		Usage:   "change-app-quota <app-name> <new-limit>",
		Desc:    desc,
	}
}

func (changeAppQuota) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/apps/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	body := bytes.NewBufferString("limit=" + limit)
	request, _ := http.NewRequest("POST", url, body)
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

func parseLimit(value string) (string, error) {
	if value == "unlimited" {
		return "-1", nil
	}
	n, err := strconv.Atoi(value)
	if err != nil {
		return "", errors.New(`invalid limit. It must be either an integer or "unlimited"`)
	}
	return strconv.Itoa(n), nil
}
<|endoftext|>


<commit_before>package bayes

/*Takes a text sample (stemmed state doesn't matter and is only used for
 *improving accuracy) and either trains the system based on it, or classifies
 *it based on previously trained data.*/

import (
 "sync"
 "database/sql"
 "log"
 _ "github.com/mattn/go-sqlite3"
)

//Tolerance := 0.05 //Tolerance -- i.e. if the result is 0.5 +- tolerance value, return "undetermined" category

type index struct { //There should be at least two of these -- one each for "good" and "bad" data.
	sync.RWMutex
	m map[string]float64
}

func Parse(text []string, cat1Index string, cat2Index string, tolerance float64) int {
	myCat1Index := loadIndex(cat1Index)
	myCat2Index := loadIndex(cat2Index)
	myCat1Index.RLock()
	myCat2Index.RLock()
	//iterate over goodindex and badindex, and sum the values.
	cat1total := 0.00;
	cat2total := 0.00;
	for _, value := range myCat1Index.m {
		cat1total += value
	}
	for _, value := range myCat2Index.m {
		cat2total += value
	}
	i := 0.00
	invi := 0.00
	for _, token := range text {
		cat1token := myCat1Index.m[token]
		cat2token := myCat2Index.m[token]
		prob := calcProbability(cat1token, cat1total, cat2token, cat2total)
		if i != 0.00 {
			i = i * prob
		} else {
			i = prob
		}
		if invi != 0.00 {
			invi = invi * (1.00 - prob)
		} else {
			invi = 1.00 - prob
		}
	}
	myCat1Index.RUnlock()
	myCat2Index.RUnlock()
	totprob := i / (i + invi)
	if totprob > (0.5 + tolerance) {
		return 1
	} else if totprob < (0.5 - tolerance) {
		return -1
	}
	return 0
}
func calcProbability(cat1count float64, cat1total float64, cat2count float64, cat2total float64) float64 {
	bw := cat1count / cat1total
	gw := cat2count / cat2total
	pw := ((bw) / ((bw) + (gw)))
	s := 1.00
	x := 0.5
	n := cat1count + cat2count
	fw := ((s * x) + (n * pw)) / (s + n)
	return fw

}
func Train(trainedIndex string, text []string) {
	myIndex := loadIndex(trainedIndex)
	myIndex.Lock();
	for _, token := range text {
		myIndex.m[token]++
	}
	saveIndex(trainedIndex, myIndex)
	myIndex.Unlock()
}
func loadIndex(indexName string) index {

	myIndex := index{}
	database, err := sql.Open("sqlite3", "./rss2.db")
	_, err = database.Exec(
 		"CREATE TABLE IF NOT EXISTS Bayes ( id varchar(255), token varchar(255), count REAL, PRIMARY KEY (id, token))")
	if err != nil {
 		log.Fatal(err)
	}
	sth, err := database.Query("SELECT token, count FROM Bayes WHERE id=?",indexName)
	myIndex.m = make(map[string]float64)
	for sth.Next(){
		var token sql.NullString
		var count sql.NullFloat64
		if err := sth.Scan(&token, &count); err != nil {
			log.Fatal(err)
		}
		myCount := count.Float64
		myToken := token.String
		myIndex.m[myToken] = myCount
	}
	database.Close()
	return myIndex
	//load index from database
}
func saveIndex(indexName string, myIndex index) {
	//save index to database
	database, err := sql.Open("sqlite3", "./rss2.db")
	if err != nil {
		log.Fatal(err)
	}
	for token, count := range myIndex.m {
		database.Exec("INSERT OR REPLACE INTO Bayes(id, token, count) VALUES(?,?,?)",indexName, token, count)
	}
	database.Close()
}
<commit_msg>Documentation for bayes<commit_after>//Package bayes implements a naive Bayes classifier.  Trained models are
//stored in a SQLite database.
package bayes

/*Takes a text sample (stemmed state doesn't matter and is only used for
 *improving accuracy) and either trains the system based on it, or classifies
 *it based on previously trained data.*/

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"sync"
)

//Tolerance := 0.05 //Tolerance -- i.e. if the result is 0.5 +- tolerance value, return "undetermined" category

type index struct {
	sync.RWMutex
	m map[string]float64
}

//Classify takes a text sample and classifies it as one of two categories.
func Classify(text []string, cat1Index string, cat2Index string, tolerance float64) int {
	myCat1Index := loadIndex(cat1Index)
	myCat2Index := loadIndex(cat2Index)
	myCat1Index.RLock()
	myCat2Index.RLock()
	//iterate over goodindex and badindex, and sum the values.
	cat1total := 0.00
	cat2total := 0.00
	for _, value := range myCat1Index.m {
		cat1total += value
	}
	for _, value := range myCat2Index.m {
		cat2total += value
	}
	i := 0.00
	invi := 0.00
	for _, token := range text {
		cat1token := myCat1Index.m[token]
		cat2token := myCat2Index.m[token]
		prob := calcProbability(cat1token, cat1total, cat2token, cat2total)
		if i != 0.00 {
			i = i * prob
		} else {
			i = prob
		}
		if invi != 0.00 {
			invi = invi * (1.00 - prob)
		} else {
			invi = 1.00 - prob
		}
	}
	myCat1Index.RUnlock()
	myCat2Index.RUnlock()
	totprob := i / (i + invi)
	if totprob > (0.5 + tolerance) {
		return 1
	} else if totprob < (0.5 - tolerance) {
		return -1
	}
	return 0
}

//calcProbability calculates the probability any given word is part of either category
func calcProbability(cat1count float64, cat1total float64, cat2count float64, cat2total float64) float64 {
	bw := cat1count / cat1total
	gw := cat2count / cat2total
	pw := ((bw) / ((bw) + (gw)))
	s := 1.00
	x := 0.5
	n := cat1count + cat2count
	fw := ((s * x) + (n * pw)) / (s + n)
	return fw

}

//Train trains a category with a text sample.
func Train(trainedIndex string, text []string) {
	myIndex := loadIndex(trainedIndex)
	myIndex.Lock()
	for _, token := range text {
		myIndex.m[token]++
	}
	saveIndex(trainedIndex, myIndex)
	myIndex.Unlock()
}

//loadIndex loads an index from a database.
func loadIndex(indexName string) index {

	myIndex := index{}
	database, err := sql.Open("sqlite3", "./bayes.db")
	_, err = database.Exec(
		"CREATE TABLE IF NOT EXISTS Bayes ( id varchar(255), token varchar(255), count REAL, PRIMARY KEY (id, token))")
	if err != nil {
		log.Fatal(err)
	}
	sth, err := database.Query("SELECT token, count FROM Bayes WHERE id=?", indexName)
	myIndex.m = make(map[string]float64)
	for sth.Next() {
		var token sql.NullString
		var count sql.NullFloat64
		if err := sth.Scan(&token, &count); err != nil {
			log.Fatal(err)
		}
		myCount := count.Float64
		myToken := token.String
		myIndex.m[myToken] = myCount
	}
	database.Close()
	return myIndex
	//load index from database
}

//saveIndex saves the index to the database.
func saveIndex(indexName string, myIndex index) {
	//save index to database
	database, err := sql.Open("sqlite3", "./rss2.db")
	if err != nil {
		log.Fatal(err)
	}
	for token, count := range myIndex.m {
		database.Exec("INSERT OR REPLACE INTO Bayes(id, token, count) VALUES(?,?,?)", indexName, token, count)
	}
	database.Close()
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package eval

import (
	"bytes"
	"context"
	"flag"
	"sync"
	"time"

	"golang.org/x/sync/errgroup"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/text"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/rts/presubmit/eval/history"
	evalpb "go.chromium.org/luci/rts/presubmit/eval/proto"
)

// defaults
const (
	defaultConcurrency = 100

	defaultProgressReportInterval = 5 * time.Second
)

// Eval estimates safety and efficiency of a given selection strategy.
type Eval struct {
	// The selection strategy to evaluate.
	Strategy Strategy

	// MaxDistance is the maximum distance to run a test.
	// If a test's distance is <= MaxDistance, then it is executed, regardless of
	// MaxRank.
	//
	// TODO(nodir): automatically compute the optimal MaxDistance based on
	// MinChangeRecall and MinTestRecall.
	MaxDistance float64

	// MaxRank is a test rank threshold: if a test's rank <= MaxRank, then it is
	// executed, regardless of MaxDistance.
	// Note that multiple tests may have the same rank.
	//
	// TODO(nodir): automatically compute the optimal MaxRank based on
	// MinChangeRecall and MinTestRecall.
	MaxRank int

	// The number of goroutines to spawn for each metric.
	// If <=0, defaults to 100.
	Concurrency int

	// History are the historical records to use for evaluation.
	History *history.Player

	// How often to report progress. Defaults to 5s.
	ProgressReportInterval time.Duration

	// If true, log lost rejections.
	// See also ChangeRecall.LostRejections.
	LogLostRejections bool
}

// RegisterFlags registers flags for the Eval fields.
func (e *Eval) RegisterFlags(fs *flag.FlagSet) error {
	// The default value of 0.5 makes sense for those selection strategies that
	// use distance between 0.0 and 1.0.
	fs.Float64Var(&e.MaxDistance, "max-distance", 0.5, text.Doc(`
		Max distance from tests to the changed files.
		If a test is closer than or equal -max-distance, then it is selected regardless of -max-rank.
	`))
	fs.IntVar(&e.MaxRank, "max-rank", 0, text.Doc(`
		Max rank for tests to run.
		If a test has a rank less or equal -max-rank, then it is selected regardless of -max-distance.
		Ignored if 0
	`))
	fs.IntVar(&e.Concurrency, "j", defaultConcurrency, "Number of job to run parallel")
	fs.Var(&historyFileInputFlag{ptr: &e.History}, "history", "Path to the history file")
	fs.DurationVar(&e.ProgressReportInterval, "progress-report-interval", defaultProgressReportInterval, "How often to report progress")
	fs.BoolVar(&e.LogLostRejections, "log-lost-rejections", false, "Log every lost rejection, to diagnose the selection strategy")
	return nil
}

// ValidateFlags validates values of flags registered using RegisterFlags.
func (e *Eval) ValidateFlags() error {
	if e.History == nil {
		return errors.New("-history is required")
	}
	return nil
}

// Run evaluates the candidate strategy.
func (e *Eval) Run(ctx context.Context) (*Result, error) {
	run := evalRun{
		// make a copy of settings
		Eval: *e,
	}
	if err := run.run(ctx); err != nil {
		return nil, err
	}
	return &run.res, nil
}

type evalRun struct {
	Eval

	// internal mutable state
	res                      Result
	buf                      bytes.Buffer
	mostRecentProgressReport time.Time
	mu                       sync.Mutex
}

func (r *evalRun) run(ctx context.Context) error {
	// Init internal state.
	r.res = Result{}
	r.mostRecentProgressReport = time.Time{}

	eg, ctx := errgroup.WithContext(ctx)
	defer eg.Wait()

	// Analyze safety.
	eg.Go(func() error {
		err := r.evaluateSafety(ctx)
		return errors.Annotate(err, "failed to evaluate safety").Err()
	})

	// Analyze efficiency.
	eg.Go(func() error {
		err := r.evaluateEfficiency(ctx)
		return errors.Annotate(err, "failed to evaluate efficiency").Err()
	})

	// Play back the history.
	eg.Go(func() error {
		err := r.History.Playback(ctx)
		r.res.TotalRecords = r.History.TotalRecords()
		return errors.Annotate(err, "failed to playback history").Err()
	})

	return eg.Wait()
}

// evaluateSafety reads rejections from r.rejectionC,
// updates r.res.Safety and calls r.maybeReportProgress.
func (r *evalRun) evaluateSafety(ctx context.Context) error {
	eg, ctx := errgroup.WithContext(ctx)

	for i := 0; i < r.Concurrency; i++ {
		eg.Go(func() error {
			return r.analyzeRejections(ctx)
		})
	}
	return eg.Wait()
}

func (r *evalRun) analyzeRejections(ctx context.Context) error {
	cr := &r.res.Safety.ChangeRecall
	tr := &r.res.Safety.TestRecall

	buf := &bytes.Buffer{}
	p := &rejectionPrinter{printer: newPrinter(buf)}
	return analyzeRejections(ctx, r.History.RejectionC, r.Strategy, func(ctx context.Context, rej analyzedRejection) error {
		// The rejection is preserved if at least one test ran.
		// Conclude that based on the closest test.
		preservedRejection := r.shouldRun(rej.Closest)

		if !preservedRejection && r.LogLostRejections {
			buf.Reset()
			p.rejection(rej.Rejection)
			logging.Infof(ctx, "%s", buf.Bytes())
		}

		r.mu.Lock()
		defer r.mu.Unlock()
		cr.TotalRejections++
		tr.TotalFailures += len(rej.FailedTestVariants)
		if !preservedRejection {
			cr.LostRejections = append(cr.LostRejections, rej.Rejection)
		}
		for i, af := range rej.Affectedness {
			if !r.shouldRun(af) {
				tr.LostFailures = append(tr.LostFailures, LostTestFailure{
					Rejection:   rej.Rejection,
					TestVariant: rej.FailedTestVariants[i],
				})
			}
		}

		r.maybeReportProgress(ctx)
		return nil
	})
}

// evaluateEfficiency reads test durations from r.durationC,
// updates r.res.Efficiency and calls r.maybeReportProgress.
func (r *evalRun) evaluateEfficiency(ctx context.Context) error {
	eg, ctx := errgroup.WithContext(ctx)

	// Run the selection strategy in r.Concurrency goroutines.
	for i := 0; i < r.Concurrency; i++ {
		eg.Go(func() error {
			in := Input{TestVariants: make([]*evalpb.TestVariant, 1)}
			out := &Output{TestVariantAffectedness: make(AffectednessSlice, 1)}
			for td := range r.History.DurationC {
				// Invoke the strategy.
				in.ChangedFiles = in.ChangedFiles[:0]
				in.ensureChangedFilesInclude(td.Patchsets...)
				in.TestVariants[0] = td.TestVariant
				out.TestVariantAffectedness[0] = Affectedness{}
				if err := r.Strategy(ctx, in, out); err != nil {
					return err
				}

				// Record results.
				dur := td.Duration.AsDuration()
				r.mu.Lock()
				r.res.Efficiency.TestResults++
				r.res.Efficiency.SampleDuration += dur
				if r.shouldRun(out.TestVariantAffectedness[0]) {
					r.res.Efficiency.ForecastDuration += dur
				}
				r.maybeReportProgress(ctx)
				r.mu.Unlock()
			}
			return ctx.Err()
		})
	}

	return eg.Wait()
}

func (r *evalRun) maybeReportProgress(ctx context.Context) {
	interval := r.ProgressReportInterval
	if interval <= 0 {
		interval = defaultProgressReportInterval
	}

	now := clock.Now(ctx)
	switch {
	case r.mostRecentProgressReport.IsZero():
		r.mostRecentProgressReport = now

	case now.After(r.mostRecentProgressReport.Add(interval)):
		r.buf.Reset()
		r.res.oneLine(&r.buf)
		logging.Infof(ctx, "%s", r.buf.Bytes())
		r.mostRecentProgressReport = now
	}
}

func (e *Eval) shouldRun(a Affectedness) bool {
	return a.Distance <= e.MaxDistance || (a.Rank != 0 && a.Rank <= e.MaxRank)
}
<commit_msg>[rts] Add parallelize() helper<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package eval

import (
	"bytes"
	"context"
	"flag"
	"sync"
	"time"

	"golang.org/x/sync/errgroup"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/text"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/rts/presubmit/eval/history"
	evalpb "go.chromium.org/luci/rts/presubmit/eval/proto"
)

// defaults
const (
	defaultConcurrency = 100

	defaultProgressReportInterval = 5 * time.Second
)

// Eval estimates safety and efficiency of a given selection strategy.
type Eval struct {
	// The selection strategy to evaluate.
	Strategy Strategy

	// MaxDistance is the maximum distance to run a test.
	// If a test's distance is <= MaxDistance, then it is executed, regardless of
	// MaxRank.
	//
	// TODO(nodir): automatically compute the optimal MaxDistance based on
	// MinChangeRecall and MinTestRecall.
	MaxDistance float64

	// MaxRank is a test rank threshold: if a test's rank <= MaxRank, then it is
	// executed, regardless of MaxDistance.
	// Note that multiple tests may have the same rank.
	//
	// TODO(nodir): automatically compute the optimal MaxRank based on
	// MinChangeRecall and MinTestRecall.
	MaxRank int

	// The number of goroutines to spawn for each metric.
	// If <=0, defaults to 100.
	Concurrency int

	// History are the historical records to use for evaluation.
	History *history.Player

	// How often to report progress. Defaults to 5s.
	ProgressReportInterval time.Duration

	// If true, log lost rejections.
	// See also ChangeRecall.LostRejections.
	LogLostRejections bool
}

// RegisterFlags registers flags for the Eval fields.
func (e *Eval) RegisterFlags(fs *flag.FlagSet) error {
	// The default value of 0.5 makes sense for those selection strategies that
	// use distance between 0.0 and 1.0.
	fs.Float64Var(&e.MaxDistance, "max-distance", 0.5, text.Doc(`
		Max distance from tests to the changed files.
		If a test is closer than or equal -max-distance, then it is selected regardless of -max-rank.
	`))
	fs.IntVar(&e.MaxRank, "max-rank", 0, text.Doc(`
		Max rank for tests to run.
		If a test has a rank less or equal -max-rank, then it is selected regardless of -max-distance.
		Ignored if 0
	`))
	fs.IntVar(&e.Concurrency, "j", defaultConcurrency, "Number of job to run parallel")
	fs.Var(&historyFileInputFlag{ptr: &e.History}, "history", "Path to the history file")
	fs.DurationVar(&e.ProgressReportInterval, "progress-report-interval", defaultProgressReportInterval, "How often to report progress")
	fs.BoolVar(&e.LogLostRejections, "log-lost-rejections", false, "Log every lost rejection, to diagnose the selection strategy")
	return nil
}

// ValidateFlags validates values of flags registered using RegisterFlags.
func (e *Eval) ValidateFlags() error {
	if e.History == nil {
		return errors.New("-history is required")
	}
	return nil
}

// Run evaluates the candidate strategy.
func (e *Eval) Run(ctx context.Context) (*Result, error) {
	run := evalRun{
		// make a copy of settings
		Eval: *e,
	}
	if err := run.run(ctx); err != nil {
		return nil, err
	}
	return &run.res, nil
}

type evalRun struct {
	Eval

	// internal mutable state
	res                      Result
	buf                      bytes.Buffer
	mostRecentProgressReport time.Time
	mu                       sync.Mutex
}

func (r *evalRun) run(ctx context.Context) error {
	// Init internal state.
	r.res = Result{}
	r.mostRecentProgressReport = time.Time{}

	eg, ctx := errgroup.WithContext(ctx)
	defer eg.Wait()

	// Analyze safety.
	eg.Go(func() error {
		err := r.evaluateSafety(ctx)
		return errors.Annotate(err, "failed to evaluate safety").Err()
	})

	// Analyze efficiency.
	eg.Go(func() error {
		err := r.evaluateEfficiency(ctx)
		return errors.Annotate(err, "failed to evaluate efficiency").Err()
	})

	// Play back the history.
	eg.Go(func() error {
		err := r.History.Playback(ctx)
		r.res.TotalRecords = r.History.TotalRecords()
		return errors.Annotate(err, "failed to playback history").Err()
	})

	return eg.Wait()
}

// evaluateSafety reads rejections from r.rejectionC,
// updates r.res.Safety and calls r.maybeReportProgress.
func (r *evalRun) evaluateSafety(ctx context.Context) error {
	return r.parallelize(ctx, func(ctx context.Context) error {
		cr := &r.res.Safety.ChangeRecall
		tr := &r.res.Safety.TestRecall

		buf := &bytes.Buffer{}
		p := &rejectionPrinter{printer: newPrinter(buf)}
		return analyzeRejections(ctx, r.History.RejectionC, r.Strategy, func(ctx context.Context, rej analyzedRejection) error {
			// The rejection is preserved if at least one test ran.
			// Conclude that based on the closest test.
			preservedRejection := r.shouldRun(rej.Closest)

			if !preservedRejection && r.LogLostRejections {
				buf.Reset()
				p.rejection(rej.Rejection)
				logging.Infof(ctx, "%s", buf.Bytes())
			}

			r.mu.Lock()
			defer r.mu.Unlock()
			cr.TotalRejections++
			tr.TotalFailures += len(rej.FailedTestVariants)
			if !preservedRejection {
				cr.LostRejections = append(cr.LostRejections, rej.Rejection)
			}
			for i, af := range rej.Affectedness {
				if !r.shouldRun(af) {
					tr.LostFailures = append(tr.LostFailures, LostTestFailure{
						Rejection:   rej.Rejection,
						TestVariant: rej.FailedTestVariants[i],
					})
				}
			}

			r.maybeReportProgress(ctx)
			return nil
		})
	})
}

// evaluateEfficiency reads test durations from r.durationC,
// updates r.res.Efficiency and calls r.maybeReportProgress.
func (r *evalRun) evaluateEfficiency(ctx context.Context) error {
	return r.parallelize(ctx, func(ctx context.Context) error {
		in := Input{TestVariants: make([]*evalpb.TestVariant, 1)}
		out := &Output{TestVariantAffectedness: make(AffectednessSlice, 1)}
		for td := range r.History.DurationC {
			// Invoke the strategy.
			in.ChangedFiles = in.ChangedFiles[:0]
			in.ensureChangedFilesInclude(td.Patchsets...)
			in.TestVariants[0] = td.TestVariant
			out.TestVariantAffectedness[0] = Affectedness{}
			if err := r.Strategy(ctx, in, out); err != nil {
				return err
			}

			// Record results.
			dur := td.Duration.AsDuration()
			r.mu.Lock()
			r.res.Efficiency.TestResults++
			r.res.Efficiency.SampleDuration += dur
			if r.shouldRun(out.TestVariantAffectedness[0]) {
				r.res.Efficiency.ForecastDuration += dur
			}
			r.maybeReportProgress(ctx)
			r.mu.Unlock()
		}
		return ctx.Err()
	})
}

func (r *evalRun) maybeReportProgress(ctx context.Context) {
	interval := r.ProgressReportInterval
	if interval <= 0 {
		interval = defaultProgressReportInterval
	}

	now := clock.Now(ctx)
	switch {
	case r.mostRecentProgressReport.IsZero():
		r.mostRecentProgressReport = now

	case now.After(r.mostRecentProgressReport.Add(interval)):
		r.buf.Reset()
		r.res.oneLine(&r.buf)
		logging.Infof(ctx, "%s", r.buf.Bytes())
		r.mostRecentProgressReport = now
	}
}

func (e *Eval) shouldRun(a Affectedness) bool {
	return a.Distance <= e.MaxDistance || (a.Rank != 0 && a.Rank <= e.MaxRank)
}

func (e *Eval) parallelize(ctx context.Context, f func(ctx context.Context) error) error {
	eg, ctx := errgroup.WithContext(ctx)
	concurrency := e.Concurrency
	if concurrency <= 0 {
		concurrency = defaultConcurrency
	}
	for i := 0; i < concurrency; i++ {
		eg.Go(func() error {
			return f(ctx)
		})
	}
	return eg.Wait()
}
<|endoftext|>


<commit_before><commit_msg>sync latest code<commit_after><|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package runtime

import (
	"github.com/islisp-dev/iris/runtime/env"
	"github.com/islisp-dev/iris/runtime/ilos"
	"github.com/islisp-dev/iris/runtime/ilos/class"
	"github.com/islisp-dev/iris/runtime/ilos/instance"
)

/*
ISLISP defines three ways in which to perform non-e exits:
  Destination   Kind     Established by  Invoked by  Operation Performed
  block name    block    return-from     lexical     exit
  tagbody tag   tagbody  go              lexical     transfer of control
  catch tag     atch     throw           dynamic     exit

A non-e exit, is an operation that forces transfer of control and possibly
data from an invoking special form to a previously established point in a program,
called the destination of the exit.

A lexical exit is a non-e exit from a return-from form to a block form
which contains it both lexically and dynamically, forcing the block to return
an object specified in the return-from form.

A dynamic exit is a non-e exit from a throw form to a catch form
which contains it dynamically (but not necessarily lexically),
forcing the catch to return an object specified in the throw form.

A lexical transfer of control is a non-e exit from a go form to a tagged point
in a tagbody form which contains it both lexically and dynamically.

When a non-e exit is initiated, any potential destination that was established
more recently than the destination to which control is being transferred
is immediately considered invalid.
*/

func Block(e env.Environment, tag ilos.Instance, body ...ilos.Instance) (ilos.Instance, ilos.Instance) {
	uid := instance.NewInteger(uniqueInt())
	if ilos.InstanceOf(class.Number, tag) || ilos.InstanceOf(class.Character, tag) {
		return SignalCondition(e, instance.NewDomainError(e, tag, class.Object), Nil)
	}
	if !e.BlockTag.Define(tag, uid) {
		return SignalCondition(e, instance.NewImmutableBinding(e), Nil)
	}
	var fail ilos.Instance
	sucess := Nil
	for _, cadr := range body {
		sucess, fail = Eval(e, cadr)
		if fail != nil {
			if ilos.InstanceOf(class.BlockTag, fail) {
				tag1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.TAG"), class.Escape) // Checked at the head of// This condition
				uid1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.UID"), class.Escape)
				if tag == tag1 && uid == uid1 {
					obj, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.OBJECT"), class.BlockTag) // Checked at the head of// This condition
					e.BlockTag.Delete(tag)
					return obj, nil
				}
			}
			e.BlockTag.Delete(tag)
			return nil, fail
		}
	}
	e.BlockTag.Delete(tag)
	return sucess, nil
}

func ReturnFrom(e env.Environment, tag, object ilos.Instance) (ilos.Instance, ilos.Instance) {
	if ilos.InstanceOf(class.Number, tag) || ilos.InstanceOf(class.Character, tag) {
		return SignalCondition(e, instance.NewDomainError(e, tag, class.Object), Nil)
	}
	object, err := Eval(e, object)
	if err != nil {
		return nil, err
	}
	uid, ok := e.BlockTag.Get(tag)
	if !ok {
		return SignalCondition(e, instance.NewControlError(e), Nil)
	}
	return nil, instance.NewBlockTag(tag, uid, object)
}

func Catch(e env.Environment, tag ilos.Instance, body ...ilos.Instance) (ilos.Instance, ilos.Instance) {
	var err ilos.Instance
	tag, err = Eval(e, tag)
	uid := instance.NewInteger(uniqueInt())
	if err != nil {
		return nil, err
	}
	if ilos.InstanceOf(class.Number, tag) || ilos.InstanceOf(class.Character, tag) {
		return SignalCondition(e, instance.NewDomainError(e, tag, class.Object), Nil)
	}
	if !e.CatchTag.Define(tag, uid) {
		return SignalCondition(e, instance.NewImmutableBinding(e), Nil)
	}
	var fail ilos.Instance
	sucess := Nil
	for _, cadr := range body {
		sucess, fail = Eval(e, cadr)
		if fail != nil {
			if ilos.InstanceOf(class.CatchTag, fail) {
				tag1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.TAG"), class.Escape) // Checked at the head of// This condition
				uid1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.UID"), class.Escape) // Checked at the head of// This condition
				if tag == tag1 && uid == uid1 {
					obj, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.OBJECT"), class.CatchTag) // Checked at the head of// This condition
					e.CatchTag.Delete(tag)
					return obj, nil
				}
			}
			e.CatchTag.Delete(tag)
			return nil, fail
		}
	}
	e.CatchTag.Delete(tag)
	return sucess, nil
}

func Throw(e env.Environment, tag, object ilos.Instance) (ilos.Instance, ilos.Instance) {
	var err ilos.Instance
	tag, err = Eval(e, tag)
	if err != nil {
		return nil, err
	}
	if ilos.InstanceOf(class.Number, tag) || ilos.InstanceOf(class.Character, tag) {
		return SignalCondition(e, instance.NewDomainError(e, tag, class.Object), Nil)
	}
	object, err = Eval(e, object)
	if err != nil {
		return nil, err
	}
	uid, ok := e.CatchTag.Get(tag)
	if !ok {
		return SignalCondition(e, instance.NewControlError(e), Nil)

	}
	return nil, instance.NewCatchTag(tag, uid, object)
}

func Tagbody(e env.Environment, body ...ilos.Instance) (ilos.Instance, ilos.Instance) {
	uid := instance.NewInteger(uniqueInt())
	for _, cadr := range body {
		if !ilos.InstanceOf(class.Cons, cadr) {
			if !e.TagbodyTag.Define(cadr, uid) { // ref cddr
				return SignalCondition(e, instance.NewImmutableBinding(e), Nil)
			}
		}
	}
	for idx, cadr := range body {
		if ilos.InstanceOf(class.Cons, cadr) {
			_, fail := Eval(e, cadr)
			if fail != nil {
			TAG:
				if ilos.InstanceOf(class.TagbodyTag, fail) {
					tag1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.TAG"), class.Escape) // Checked at the top of// This loop
					uid1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.UID"), class.Escape) // Checked at the top of// This loop
					found := false
					for _, tag := range body {
						if tag == tag1 && uid == uid1 {
							found = true
							break
						}
					}
					if found {
						for _, form := range body[idx+1:] {
							if ilos.InstanceOf(class.Cons, form) {
								_, fail = Eval(e, form)
								if fail != nil {
									goto TAG
								}
							}
						}
						break
					}

				}
				return nil, fail
			}
		}
	}
	return Nil, nil
}

func Go(e env.Environment, tag ilos.Instance) (ilos.Instance, ilos.Instance) {
	uid, ok := e.TagbodyTag.Get(tag)
	if !ok {
		return SignalCondition(e, instance.NewControlError(e), Nil)
	}
	return nil, instance.NewTagbodyTag(tag, uid)
}

// UnwindProtect first evaluates form. Evaluation of the cleanup-forms always
// occurs, regardless of whether the exit is normal or non-e. If the form exits
// normally yielding a value R, then if all of the cleanup-forms exit normally
// the value R is returned by the unwind-protect form. If a non-e exit from form
// occurs, then the cleanup-forms are executed as part of that exit, and then if
// all of the cleanup-forms exit normally the original non-e exit continues. The
// cleanup-forms are evaluated from left to right, discarding the resulting
// values. If execution of the cleanup-forms finishes normally, exit from the
// unwind-protect form proceeds as described above. It is permissible for a
// cleanup-form to contain a non-e exit from the unwind-protect form, subject to
// the following constraint: An error shall be signaled if during execution of
// the cleanup-forms of an unwind-protect form, a non-e exit is executed to a
// destination which has been marked as invalid due to some other non-e exit
// that is already in progress (error-id. control-error). Note: Because ISLISP
// does not specify an interactive debugger, it is unspecified whether or how
// error recovery can occur interactively if programmatic handling fails. The
// intent is that if the ISLISP processor does not terminate abnormally, normal
// mechanisms for non-e exit (return-from, throw, or go) would be used as
// necessary and would respect these cleanup-forms.
func UnwindProtect(e env.Environment, form ilos.Instance, cleanupForms ...ilos.Instance) (ilos.Instance, ilos.Instance) {
	ret1, err1 := Eval(e, form)
	ret2, err2 := Progn(e, cleanupForms...)
	if ilos.InstanceOf(class.Escape, err2) {
		return SignalCondition(e, instance.NewControlError(e), Nil)
	}
	if err2 != nil {
		return ret2, err2
	}
	return ret1, err1
}
<commit_msg>fix unwind-protect<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package runtime

import (
	"github.com/islisp-dev/iris/runtime/env"
	"github.com/islisp-dev/iris/runtime/ilos"
	"github.com/islisp-dev/iris/runtime/ilos/class"
	"github.com/islisp-dev/iris/runtime/ilos/instance"
)

/*
ISLISP defines three ways in which to perform non-e exits:
  Destination   Kind     Established by  Invoked by  Operation Performed
  block name    block    return-from     lexical     exit
  tagbody tag   tagbody  go              lexical     transfer of control
  catch tag     atch     throw           dynamic     exit

A non-e exit, is an operation that forces transfer of control and possibly
data from an invoking special form to a previously established point in a program,
called the destination of the exit.

A lexical exit is a non-e exit from a return-from form to a block form
which contains it both lexically and dynamically, forcing the block to return
an object specified in the return-from form.

A dynamic exit is a non-e exit from a throw form to a catch form
which contains it dynamically (but not necessarily lexically),
forcing the catch to return an object specified in the throw form.

A lexical transfer of control is a non-e exit from a go form to a tagged point
in a tagbody form which contains it both lexically and dynamically.

When a non-e exit is initiated, any potential destination that was established
more recently than the destination to which control is being transferred
is immediately considered invalid.
*/

func Block(e env.Environment, tag ilos.Instance, body ...ilos.Instance) (ilos.Instance, ilos.Instance) {
	uid := instance.NewInteger(uniqueInt())
	if ilos.InstanceOf(class.Number, tag) || ilos.InstanceOf(class.Character, tag) {
		return SignalCondition(e, instance.NewDomainError(e, tag, class.Object), Nil)
	}
	if !e.BlockTag.Define(tag, uid) {
		return SignalCondition(e, instance.NewImmutableBinding(e), Nil)
	}
	var fail ilos.Instance
	sucess := Nil
	for _, cadr := range body {
		sucess, fail = Eval(e, cadr)
		if fail != nil {
			if ilos.InstanceOf(class.BlockTag, fail) {
				tag1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.TAG"), class.Escape) // Checked at the head of// This condition
				uid1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.UID"), class.Escape)
				if tag == tag1 && uid == uid1 {
					obj, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.OBJECT"), class.BlockTag) // Checked at the head of// This condition
					e.BlockTag.Delete(tag)
					return obj, nil
				}
			}
			e.BlockTag.Delete(tag)
			return nil, fail
		}
	}
	e.BlockTag.Delete(tag)
	return sucess, nil
}

func ReturnFrom(e env.Environment, tag, object ilos.Instance) (ilos.Instance, ilos.Instance) {
	if ilos.InstanceOf(class.Number, tag) || ilos.InstanceOf(class.Character, tag) {
		return SignalCondition(e, instance.NewDomainError(e, tag, class.Object), Nil)
	}
	object, err := Eval(e, object)
	if err != nil {
		return nil, err
	}
	uid, ok := e.BlockTag.Get(tag)
	if !ok {
		return SignalCondition(e, instance.NewControlError(e), Nil)
	}
	return nil, instance.NewBlockTag(tag, uid, object)
}

func Catch(e env.Environment, tag ilos.Instance, body ...ilos.Instance) (ilos.Instance, ilos.Instance) {
	var err ilos.Instance
	tag, err = Eval(e, tag)
	uid := instance.NewInteger(uniqueInt())
	if err != nil {
		return nil, err
	}
	if ilos.InstanceOf(class.Number, tag) || ilos.InstanceOf(class.Character, tag) {
		return SignalCondition(e, instance.NewDomainError(e, tag, class.Object), Nil)
	}
	if !e.CatchTag.Define(tag, uid) {
		return SignalCondition(e, instance.NewImmutableBinding(e), Nil)
	}
	var fail ilos.Instance
	sucess := Nil
	for _, cadr := range body {
		sucess, fail = Eval(e, cadr)
		if fail != nil {
			if ilos.InstanceOf(class.CatchTag, fail) {
				tag1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.TAG"), class.Escape) // Checked at the head of// This condition
				uid1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.UID"), class.Escape) // Checked at the head of// This condition
				if tag == tag1 && uid == uid1 {
					obj, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.OBJECT"), class.CatchTag) // Checked at the head of// This condition
					e.CatchTag.Delete(tag)
					return obj, nil
				}
			}
			e.CatchTag.Delete(tag)
			return nil, fail
		}
	}
	e.CatchTag.Delete(tag)
	return sucess, nil
}

func Throw(e env.Environment, tag, object ilos.Instance) (ilos.Instance, ilos.Instance) {
	var err ilos.Instance
	tag, err = Eval(e, tag)
	if err != nil {
		return nil, err
	}
	if ilos.InstanceOf(class.Number, tag) || ilos.InstanceOf(class.Character, tag) {
		return SignalCondition(e, instance.NewDomainError(e, tag, class.Object), Nil)
	}
	object, err = Eval(e, object)
	if err != nil {
		return nil, err
	}
	uid, ok := e.CatchTag.Get(tag)
	if !ok {
		return SignalCondition(e, instance.NewControlError(e), Nil)

	}
	return nil, instance.NewCatchTag(tag, uid, object)
}

func Tagbody(e env.Environment, body ...ilos.Instance) (ilos.Instance, ilos.Instance) {
	uid := instance.NewInteger(uniqueInt())
	for _, cadr := range body {
		if !ilos.InstanceOf(class.Cons, cadr) {
			if !e.TagbodyTag.Define(cadr, uid) { // ref cddr
				return SignalCondition(e, instance.NewImmutableBinding(e), Nil)
			}
		}
	}
	for idx, cadr := range body {
		if ilos.InstanceOf(class.Cons, cadr) {
			_, fail := Eval(e, cadr)
			if fail != nil {
			TAG:
				if ilos.InstanceOf(class.TagbodyTag, fail) {
					tag1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.TAG"), class.Escape) // Checked at the top of// This loop
					uid1, _ := fail.(instance.Instance).GetSlotValue(instance.NewSymbol("IRIS.UID"), class.Escape) // Checked at the top of// This loop
					found := false
					for _, tag := range body {
						if tag == tag1 && uid == uid1 {
							found = true
							break
						}
					}
					if found {
						for _, form := range body[idx+1:] {
							if ilos.InstanceOf(class.Cons, form) {
								_, fail = Eval(e, form)
								if fail != nil {
									goto TAG
								}
							}
						}
						break
					}

				}
				return nil, fail
			}
		}
	}
	return Nil, nil
}

func Go(e env.Environment, tag ilos.Instance) (ilos.Instance, ilos.Instance) {
	uid, ok := e.TagbodyTag.Get(tag)
	if !ok {
		return SignalCondition(e, instance.NewControlError(e), Nil)
	}
	return nil, instance.NewTagbodyTag(tag, uid)
}

// UnwindProtect first evaluates form. Evaluation of the cleanup-forms always
// occurs, regardless of whether the exit is normal or non-e. If the form exits
// normally yielding a value R, then if all of the cleanup-forms exit normally
// the value R is returned by the unwind-protect form. If a non-e exit from form
// occurs, then the cleanup-forms are executed as part of that exit, and then if
// all of the cleanup-forms exit normally the original non-e exit continues. The
// cleanup-forms are evaluated from left to right, discarding the resulting
// values. If execution of the cleanup-forms finishes normally, exit from the
// unwind-protect form proceeds as described above. It is permissible for a
// cleanup-form to contain a non-e exit from the unwind-protect form, subject to
// the following constraint: An error shall be signaled if during execution of
// the cleanup-forms of an unwind-protect form, a non-e exit is executed to a
// destination which has been marked as invalid due to some other non-e exit
// that is already in progress (error-id. control-error). Note: Because ISLISP
// does not specify an interactive debugger, it is unspecified whether or how
// error recovery can occur interactively if programmatic handling fails. The
// intent is that if the ISLISP processor does not terminate abnormally, normal
// mechanisms for non-e exit (return-from, throw, or go) would be used as
// necessary and would respect these cleanup-forms.
func UnwindProtect(e env.Environment, form ilos.Instance, cleanupForms ...ilos.Instance) (ilos.Instance, ilos.Instance) {
	ret1, err1 := Eval(e, form)
	ret2, err2 := Progn(e, cleanupForms...)
	if err2 != nil {
		if ilos.InstanceOf(class.Escape, err2) {
			return SignalCondition(e, instance.NewControlError(e), Nil)
		}
		return ret2, err2
	}
	return ret1, err1
}
<|endoftext|>


<commit_before>/*
   Copyright 2013 Juliano Martinez <juliano@martinez.io>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

      Based on http://github.com/nf/webfront

   @author: Juliano Martinez
*/

package http_server

import (
	"fmt"
	"github.com/fiorix/go-redis/redis"
	hpr_utils "github.com/ncode/hot-potato-router/utils"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"sync"
	"time"
)

var (
	cfg = hpr_utils.NewConfig()
	rc  = redis.New(cfg.Options["redis"]["server_list"])
)

func xff(req *http.Request) string {
	remote_addr := strings.Split(req.RemoteAddr, ":")
	if len(remote_addr) == 0 {
		return ""
	}
	return remote_addr[0]
}

type Server struct {
	mu      sync.RWMutex
	last    time.Time
	proxy   map[string][]Proxy
	backend map[string]int
}

type Proxy struct {
	Connections int64
	Backend     string
	handler     http.Handler
}

func Listen(fd int, addr string) net.Listener {
	var l net.Listener
	var err error
	if fd >= 3 {
		l, err = net.FileListener(os.NewFile(uintptr(fd), "http"))
	} else {
		l, err = net.Listen("tcp", addr)
	}
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func NewServer(probe time.Duration) (*Server, error) {
	s := new(Server)
	s.proxy = make(map[string][]Proxy)
	s.backend = make(map[string]int)
	go s.probe_backends(probe)
	return s, nil
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h := s.handler(r); h != nil {
		client := xff(r)
		hpr_utils.Log(fmt.Sprintf("Request from: %s Url: %s", client, r.Host))
		r.Header.Add("X-Forwarded-For", client)
		r.Header.Add("X-Real-IP", client)
		h.ServeHTTP(w, r)
		return
	}
	http.Error(w, "Not found.", http.StatusNotFound)
}

func (s *Server) handler(req *http.Request) http.Handler {
	h := req.Host
	if i := strings.Index(h, ":"); i >= 0 {
		h = h[:i]
	}

	_, ok := s.proxy[h]
	if !ok {
		f, _ := rc.ZRange(fmt.Sprint("hpr-backends::%s", h), 0, -1, true)
		if len(f) == 0 {
			return nil
		}
		s.mu.Lock()
		for _, be := range f {

			log.Println(be)
			s.proxy[h] = append(s.proxy[h], Proxy{0, be, makeHandler(be)})
		}
		s.mu.Unlock()
	}
	return s.Next(h)
}

/* TODO: Implement more balance algorithms */
func (s *Server) Next(h string) http.Handler {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.backend[h]++
	total := len(s.proxy[h])
	if s.backend[h] == total {
		s.backend[h] = 0
	}
	hpr_utils.Log(fmt.Sprintf("Using backend: %s Url: %s", s.proxy[h][s.backend[h]].Backend, h))
	return s.proxy[h][s.backend[h]].handler
}

func (s *Server) probe_backends(probe time.Duration) {
	for {
		time.Sleep(probe)
		// s.mu.Lock()
		for key, value := range s.proxy {
			hpr_utils.Log(fmt.Sprintf("Key: %s Value: %s", key, value))
		}
		// s.mu.Unlock()
	}
}

func makeHandler(f string) http.Handler {
	if f != "" {
		return &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.URL.Scheme = "http"
				req.URL.Host = f
			},
		}
	}
	return nil
}
<commit_msg>some tests<commit_after>/*
   Copyright 2013 Juliano Martinez <juliano@martinez.io>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

      Based on http://github.com/nf/webfront

   @author: Juliano Martinez
*/

package http_server

import (
	"fmt"
	"github.com/fiorix/go-redis/redis"
	hpr_utils "github.com/ncode/hot-potato-router/utils"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"sync"
	"time"
)

var (
	cfg = hpr_utils.NewConfig()
	rc  = redis.New(cfg.Options["redis"]["server_list"])
)

func xff(req *http.Request) string {
	remote_addr := strings.Split(req.RemoteAddr, ":")
	if len(remote_addr) == 0 {
		return ""
	}
	return remote_addr[0]
}

type Server struct {
	mu      sync.RWMutex
	last    time.Time
	proxy   map[string][]Proxy
	backend map[string]int
}

type Proxy struct {
	Connections int64
	Backend     string
	handler     http.Handler
}

func Listen(fd int, addr string) net.Listener {
	var l net.Listener
	var err error
	if fd >= 3 {
		l, err = net.FileListener(os.NewFile(uintptr(fd), "http"))
	} else {
		l, err = net.Listen("tcp", addr)
	}
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func NewServer(probe time.Duration) (*Server, error) {
	s := new(Server)
	s.proxy = make(map[string][]Proxy)
	s.backend = make(map[string]int)
	go s.probe_backends(probe)
	return s, nil
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h := s.handler(r); h != nil {
		client := xff(r)
		hpr_utils.Log(fmt.Sprintf("Request from: %s Url: %s", client, r.Host))
		r.Header.Add("X-Forwarded-For", client)
		r.Header.Add("X-Real-IP", client)
		h.ServeHTTP(w, r)
		return
	}
	http.Error(w, "Not found.", http.StatusNotFound)
}

func (s *Server) handler(req *http.Request) http.Handler {
	h := req.Host
	if i := strings.Index(h, ":"); i >= 0 {
		h = h[:i]
	}

	_, ok := s.proxy[h]
	if !ok {
		f, _ := rc.ZRange(fmt.Sprint("hpr-backends::%s", h), 0, -1, true)
		log.Println(f[0])
		/* if len(f) == 0 {
			return nil
		}*/
		s.mu.Lock()
		for _, be := range f {

			log.Println(be)
			s.proxy[h] = append(s.proxy[h], Proxy{0, be, makeHandler(be)})
		}
		s.mu.Unlock()
	}
	return s.Next(h)
}

/* TODO: Implement more balance algorithms */
func (s *Server) Next(h string) http.Handler {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.backend[h]++
	total := len(s.proxy[h])
	if s.backend[h] == total {
		s.backend[h] = 0
	}
	hpr_utils.Log(fmt.Sprintf("Using backend: %s Url: %s", s.proxy[h][s.backend[h]].Backend, h))
	return s.proxy[h][s.backend[h]].handler
}

func (s *Server) probe_backends(probe time.Duration) {
	for {
		time.Sleep(probe)
		// s.mu.Lock()
		for key, value := range s.proxy {
			hpr_utils.Log(fmt.Sprintf("Key: %s Value: %s", key, value))
		}
		// s.mu.Unlock()
	}
}

func makeHandler(f string) http.Handler {
	if f != "" {
		return &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.URL.Scheme = "http"
				req.URL.Host = f
			},
		}
	}
	return nil
}
<|endoftext|>