<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	"context"
	"fmt"
	"net"
	"net/url"
	"strings"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/tools/pager"
	"k8s.io/kops/upup/pkg/fi"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/cloudinstances"
	"k8s.io/kops/pkg/dns"
)

// ValidationCluster uses a cluster to validate.
type ValidationCluster struct {
	Failures []*ValidationError `json:"failures,omitempty"`

	Nodes []*ValidationNode `json:"nodes,omitempty"`
}

// ValidationError holds a validation failure
type ValidationError struct {
	Kind    string `json:"type,omitempty"`
	Name    string `json:"name,omitempty"`
	Message string `json:"message,omitempty"`
}

type ClusterValidator interface {
	// Validate validates a k8s cluster
	Validate() (*ValidationCluster, error)
}

type clusterValidatorImpl struct {
	cluster        *kops.Cluster
	cloud          fi.Cloud
	instanceGroups []*kops.InstanceGroup
	k8sClient      kubernetes.Interface
}

func (v *ValidationCluster) addError(failure *ValidationError) {
	v.Failures = append(v.Failures, failure)
}

// ValidationNode represents the validation status for a node
type ValidationNode struct {
	Name     string             `json:"name,omitempty"`
	Zone     string             `json:"zone,omitempty"`
	Role     string             `json:"role,omitempty"`
	Hostname string             `json:"hostname,omitempty"`
	Status   v1.ConditionStatus `json:"status,omitempty"`
}

// hasPlaceHolderIP checks if the API DNS has been updated.
func hasPlaceHolderIP(clusterName string) (bool, error) {

	config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		clientcmd.NewDefaultClientConfigLoadingRules(),
		&clientcmd.ConfigOverrides{CurrentContext: clusterName}).ClientConfig()
	if err != nil {
		return false, fmt.Errorf("error building configuration: %v", err)
	}

	apiAddr, err := url.Parse(config.Host)
	if err != nil {
		return true, fmt.Errorf("unable to parse Kubernetes cluster API URL: %v", err)
	}
	hostAddrs, err := net.LookupHost(apiAddr.Hostname())
	if err != nil {
		return true, fmt.Errorf("unable to resolve Kubernetes cluster API URL dns: %v", err)
	}

	for _, h := range hostAddrs {
		if h == "203.0.113.123" {
			return true, nil
		}
	}

	return false, nil
}

func NewClusterValidator(cluster *kops.Cluster, cloud fi.Cloud, instanceGroupList *kops.InstanceGroupList, k8sClient kubernetes.Interface) (ClusterValidator, error) {
	var instanceGroups []*kops.InstanceGroup

	for i := range instanceGroupList.Items {
		ig := &instanceGroupList.Items[i]
		instanceGroups = append(instanceGroups, ig)
	}

	if len(instanceGroups) == 0 {
		return nil, fmt.Errorf("no InstanceGroup objects found")
	}

	return &clusterValidatorImpl{
		cluster:        cluster,
		cloud:          cloud,
		instanceGroups: instanceGroups,
		k8sClient:      k8sClient,
	}, nil
}

func (v *clusterValidatorImpl) Validate() (*ValidationCluster, error) {
	ctx := context.TODO()

	clusterName := v.cluster.Name

	validation := &ValidationCluster{}

	// Do not use if we are running gossip
	if !dns.IsGossipHostname(clusterName) {
		contextName := clusterName

		hasPlaceHolderIPAddress, err := hasPlaceHolderIP(contextName)
		if err != nil {
			return nil, err
		}

		if hasPlaceHolderIPAddress {
			message := "Validation Failed\n\n" +
				"The dns-controller Kubernetes deployment has not updated the Kubernetes cluster's API DNS entry to the correct IP address." +
				"  The API DNS IP address is the placeholder address that kops creates: 203.0.113.123." +
				"  Please wait about 5-10 minutes for a master to start, dns-controller to launch, and DNS to propagate." +
				"  The protokube container and dns-controller deployment logs may contain more diagnostic information." +
				"  Etcd and the API DNS entries must be updated for a kops Kubernetes cluster to start."
			validation.addError(&ValidationError{
				Kind:    "dns",
				Name:    "apiserver",
				Message: message,
			})
			return validation, nil
		}
	}

	nodeList, err := v.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
	if err != nil {
		return nil, fmt.Errorf("error listing nodes: %v", err)
	}

	warnUnmatched := false
	cloudGroups, err := v.cloud.GetCloudGroups(v.cluster, v.instanceGroups, warnUnmatched, nodeList.Items)
	if err != nil {
		return nil, err
	}
	readyNodes := validation.validateNodes(cloudGroups, v.instanceGroups)

	if err := validation.collectComponentFailures(ctx, v.k8sClient); err != nil {
		return nil, fmt.Errorf("cannot get component status for %q: %v", clusterName, err)
	}

	if err := validation.collectPodFailures(ctx, v.k8sClient, readyNodes); err != nil {
		return nil, fmt.Errorf("cannot get pod health for %q: %v", clusterName, err)
	}

	return validation, nil
}

func (v *ValidationCluster) collectComponentFailures(ctx context.Context, client kubernetes.Interface) error {
	componentList, err := client.CoreV1().ComponentStatuses().List(ctx, metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("error listing ComponentStatuses: %v", err)
	}

	for _, component := range componentList.Items {
		for _, condition := range component.Conditions {
			if condition.Status != v1.ConditionTrue {
				v.addError(&ValidationError{
					Kind:    "ComponentStatus",
					Name:    component.Name,
					Message: fmt.Sprintf("component %q is unhealthy", component.Name),
				})
			}
		}
	}
	return nil
}

var masterStaticPods = []string{
	"kube-apiserver",
	"kube-controller-manager",
	"kube-scheduler",
}

func (v *ValidationCluster) collectPodFailures(ctx context.Context, client kubernetes.Interface, nodes []v1.Node) error {
	masterWithoutPod := map[string]map[string]bool{}
	nodeByAddress := map[string]string{}
	for _, node := range nodes {
		labels := node.GetLabels()
		if labels != nil && labels["kubernetes.io/role"] == "master" {
			masterWithoutPod[node.Name] = map[string]bool{}
			for _, pod := range masterStaticPods {
				masterWithoutPod[node.Name][pod] = true
			}
		}
		for _, nodeAddress := range node.Status.Addresses {
			nodeByAddress[nodeAddress.Address] = node.Name
		}
	}

	err := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
		return client.CoreV1().Pods(metav1.NamespaceAll).List(ctx, opts)
	})).EachListItem(context.TODO(), metav1.ListOptions{}, func(obj runtime.Object) error {
		pod := obj.(*v1.Pod)
		priority := pod.Spec.PriorityClassName
		if priority != "system-cluster-critical" && priority != "system-node-critical" {
			return nil
		}
		if pod.Status.Phase == v1.PodSucceeded {
			return nil
		}
		if pod.Status.Phase == v1.PodPending {
			v.addError(&ValidationError{
				Kind:    "Pod",
				Name:    pod.Namespace + "/" + pod.Name,
				Message: fmt.Sprintf("%s pod %q is pending", priority, pod.Name),
			})
			return nil
		}
		if pod.Status.Phase == v1.PodUnknown {
			v.addError(&ValidationError{
				Kind:    "Pod",
				Name:    pod.Namespace + "/" + pod.Name,
				Message: fmt.Sprintf("%s pod %q is unknown phase", priority, pod.Name),
			})
			return nil
		}
		var notready []string
		for _, container := range pod.Status.ContainerStatuses {
			if !container.Ready {
				notready = append(notready, container.Name)
			}
		}
		if len(notready) != 0 {
			v.addError(&ValidationError{
				Kind:    "Pod",
				Name:    pod.Namespace + "/" + pod.Name,
				Message: fmt.Sprintf("%s pod %q is not ready (%s)", priority, pod.Name, strings.Join(notready, ",")),
			})

		}

		app := pod.GetLabels()["k8s-app"]
		if pod.Namespace == "kube-system" && masterWithoutPod[nodeByAddress[pod.Status.HostIP]][app] {
			delete(masterWithoutPod[nodeByAddress[pod.Status.HostIP]], app)
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error listing Pods: %v", err)
	}

	for node, nodeMap := range masterWithoutPod {
		for app := range nodeMap {
			v.addError(&ValidationError{
				Kind:    "Node",
				Name:    node,
				Message: fmt.Sprintf("master %q is missing %s pod", node, app),
			})
		}
	}

	return nil
}

func (v *ValidationCluster) validateNodes(cloudGroups map[string]*cloudinstances.CloudInstanceGroup, groups []*kops.InstanceGroup) []v1.Node {
	var readyNodes []v1.Node
	groupsSeen := map[string]bool{}

	for _, cloudGroup := range cloudGroups {
		var allMembers []*cloudinstances.CloudInstance
		allMembers = append(allMembers, cloudGroup.Ready...)
		allMembers = append(allMembers, cloudGroup.NeedUpdate...)

		groupsSeen[cloudGroup.InstanceGroup.Name] = true
		numNodes := 0
		for _, m := range allMembers {
			if m.Status != cloudinstances.CloudInstanceStatusDetached {
				numNodes++
			}
		}
		if numNodes < cloudGroup.TargetSize {
			v.addError(&ValidationError{
				Kind: "InstanceGroup",
				Name: cloudGroup.InstanceGroup.Name,
				Message: fmt.Sprintf("InstanceGroup %q did not have enough nodes %d vs %d",
					cloudGroup.InstanceGroup.Name,
					numNodes,
					cloudGroup.TargetSize),
			})
		}

		for _, member := range allMembers {
			node := member.Node

			if node == nil {
				nodeExpectedToJoin := true
				if cloudGroup.InstanceGroup.Spec.Role == kops.InstanceGroupRoleBastion {
					// bastion nodes don't join the cluster
					nodeExpectedToJoin = false
				}

				if nodeExpectedToJoin {
					v.addError(&ValidationError{
						Kind:    "Machine",
						Name:    member.ID,
						Message: fmt.Sprintf("machine %q has not yet joined cluster", member.ID),
					})
				}
				continue
			}

			role := strings.ToLower(string(cloudGroup.InstanceGroup.Spec.Role))
			if role == "" {
				role = "node"
			}

			n := &ValidationNode{
				Name:     node.Name,
				Zone:     node.ObjectMeta.Labels["topology.kubernetes.io/zone"],
				Hostname: node.ObjectMeta.Labels["kubernetes.io/hostname"],
				Role:     role,
				Status:   getNodeReadyStatus(node),
			}
			if n.Zone == "" {
				n.Zone = node.ObjectMeta.Labels["failure-domain.beta.kubernetes.io/zone"]
			}

			ready := isNodeReady(node)
			if ready {
				readyNodes = append(readyNodes, *node)
			}

			if n.Role == "master" {
				if !ready {
					v.addError(&ValidationError{
						Kind:    "Node",
						Name:    node.Name,
						Message: fmt.Sprintf("master %q is not ready", node.Name),
					})
				}

				v.Nodes = append(v.Nodes, n)
			} else if n.Role == "node" {
				if !ready {
					v.addError(&ValidationError{
						Kind:    "Node",
						Name:    node.Name,
						Message: fmt.Sprintf("node %q is not ready", node.Name),
					})
				}

				v.Nodes = append(v.Nodes, n)
			} else {
				klog.Warningf("ignoring node with role %q", n.Role)
			}
		}
	}

	for _, ig := range groups {
		if !groupsSeen[ig.Name] {
			v.addError(&ValidationError{
				Kind:    "InstanceGroup",
				Name:    ig.Name,
				Message: fmt.Sprintf("InstanceGroup %q is missing from the cloud provider", ig.Name),
			})
		}
	}

	return readyNodes
}
<commit_msg>Don't require PriorityClassName to pass missing-static-pod checks<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	"context"
	"fmt"
	"net"
	"net/url"
	"strings"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/tools/pager"
	"k8s.io/kops/upup/pkg/fi"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/cloudinstances"
	"k8s.io/kops/pkg/dns"
)

// ValidationCluster uses a cluster to validate.
type ValidationCluster struct {
	Failures []*ValidationError `json:"failures,omitempty"`

	Nodes []*ValidationNode `json:"nodes,omitempty"`
}

// ValidationError holds a validation failure
type ValidationError struct {
	Kind    string `json:"type,omitempty"`
	Name    string `json:"name,omitempty"`
	Message string `json:"message,omitempty"`
}

type ClusterValidator interface {
	// Validate validates a k8s cluster
	Validate() (*ValidationCluster, error)
}

type clusterValidatorImpl struct {
	cluster        *kops.Cluster
	cloud          fi.Cloud
	instanceGroups []*kops.InstanceGroup
	k8sClient      kubernetes.Interface
}

func (v *ValidationCluster) addError(failure *ValidationError) {
	v.Failures = append(v.Failures, failure)
}

// ValidationNode represents the validation status for a node
type ValidationNode struct {
	Name     string             `json:"name,omitempty"`
	Zone     string             `json:"zone,omitempty"`
	Role     string             `json:"role,omitempty"`
	Hostname string             `json:"hostname,omitempty"`
	Status   v1.ConditionStatus `json:"status,omitempty"`
}

// hasPlaceHolderIP checks if the API DNS has been updated.
func hasPlaceHolderIP(clusterName string) (bool, error) {

	config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		clientcmd.NewDefaultClientConfigLoadingRules(),
		&clientcmd.ConfigOverrides{CurrentContext: clusterName}).ClientConfig()
	if err != nil {
		return false, fmt.Errorf("error building configuration: %v", err)
	}

	apiAddr, err := url.Parse(config.Host)
	if err != nil {
		return true, fmt.Errorf("unable to parse Kubernetes cluster API URL: %v", err)
	}
	hostAddrs, err := net.LookupHost(apiAddr.Hostname())
	if err != nil {
		return true, fmt.Errorf("unable to resolve Kubernetes cluster API URL dns: %v", err)
	}

	for _, h := range hostAddrs {
		if h == "203.0.113.123" {
			return true, nil
		}
	}

	return false, nil
}

func NewClusterValidator(cluster *kops.Cluster, cloud fi.Cloud, instanceGroupList *kops.InstanceGroupList, k8sClient kubernetes.Interface) (ClusterValidator, error) {
	var instanceGroups []*kops.InstanceGroup

	for i := range instanceGroupList.Items {
		ig := &instanceGroupList.Items[i]
		instanceGroups = append(instanceGroups, ig)
	}

	if len(instanceGroups) == 0 {
		return nil, fmt.Errorf("no InstanceGroup objects found")
	}

	return &clusterValidatorImpl{
		cluster:        cluster,
		cloud:          cloud,
		instanceGroups: instanceGroups,
		k8sClient:      k8sClient,
	}, nil
}

func (v *clusterValidatorImpl) Validate() (*ValidationCluster, error) {
	ctx := context.TODO()

	clusterName := v.cluster.Name

	validation := &ValidationCluster{}

	// Do not use if we are running gossip
	if !dns.IsGossipHostname(clusterName) {
		contextName := clusterName

		hasPlaceHolderIPAddress, err := hasPlaceHolderIP(contextName)
		if err != nil {
			return nil, err
		}

		if hasPlaceHolderIPAddress {
			message := "Validation Failed\n\n" +
				"The dns-controller Kubernetes deployment has not updated the Kubernetes cluster's API DNS entry to the correct IP address." +
				"  The API DNS IP address is the placeholder address that kops creates: 203.0.113.123." +
				"  Please wait about 5-10 minutes for a master to start, dns-controller to launch, and DNS to propagate." +
				"  The protokube container and dns-controller deployment logs may contain more diagnostic information." +
				"  Etcd and the API DNS entries must be updated for a kops Kubernetes cluster to start."
			validation.addError(&ValidationError{
				Kind:    "dns",
				Name:    "apiserver",
				Message: message,
			})
			return validation, nil
		}
	}

	nodeList, err := v.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
	if err != nil {
		return nil, fmt.Errorf("error listing nodes: %v", err)
	}

	warnUnmatched := false
	cloudGroups, err := v.cloud.GetCloudGroups(v.cluster, v.instanceGroups, warnUnmatched, nodeList.Items)
	if err != nil {
		return nil, err
	}
	readyNodes := validation.validateNodes(cloudGroups, v.instanceGroups)

	if err := validation.collectComponentFailures(ctx, v.k8sClient); err != nil {
		return nil, fmt.Errorf("cannot get component status for %q: %v", clusterName, err)
	}

	if err := validation.collectPodFailures(ctx, v.k8sClient, readyNodes); err != nil {
		return nil, fmt.Errorf("cannot get pod health for %q: %v", clusterName, err)
	}

	return validation, nil
}

func (v *ValidationCluster) collectComponentFailures(ctx context.Context, client kubernetes.Interface) error {
	componentList, err := client.CoreV1().ComponentStatuses().List(ctx, metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("error listing ComponentStatuses: %v", err)
	}

	for _, component := range componentList.Items {
		for _, condition := range component.Conditions {
			if condition.Status != v1.ConditionTrue {
				v.addError(&ValidationError{
					Kind:    "ComponentStatus",
					Name:    component.Name,
					Message: fmt.Sprintf("component %q is unhealthy", component.Name),
				})
			}
		}
	}
	return nil
}

var masterStaticPods = []string{
	"kube-apiserver",
	"kube-controller-manager",
	"kube-scheduler",
}

func (v *ValidationCluster) collectPodFailures(ctx context.Context, client kubernetes.Interface, nodes []v1.Node) error {
	masterWithoutPod := map[string]map[string]bool{}
	nodeByAddress := map[string]string{}
	for _, node := range nodes {
		labels := node.GetLabels()
		if labels != nil && labels["kubernetes.io/role"] == "master" {
			masterWithoutPod[node.Name] = map[string]bool{}
			for _, pod := range masterStaticPods {
				masterWithoutPod[node.Name][pod] = true
			}
		}
		for _, nodeAddress := range node.Status.Addresses {
			nodeByAddress[nodeAddress.Address] = node.Name
		}
	}

	err := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
		return client.CoreV1().Pods(metav1.NamespaceAll).List(ctx, opts)
	})).EachListItem(context.TODO(), metav1.ListOptions{}, func(obj runtime.Object) error {
		pod := obj.(*v1.Pod)

		app := pod.GetLabels()["k8s-app"]
		if pod.Namespace == "kube-system" && masterWithoutPod[nodeByAddress[pod.Status.HostIP]][app] {
			delete(masterWithoutPod[nodeByAddress[pod.Status.HostIP]], app)
		}

		priority := pod.Spec.PriorityClassName
		if priority != "system-cluster-critical" && priority != "system-node-critical" {
			return nil
		}
		if pod.Status.Phase == v1.PodSucceeded {
			return nil
		}
		if pod.Status.Phase == v1.PodPending {
			v.addError(&ValidationError{
				Kind:    "Pod",
				Name:    pod.Namespace + "/" + pod.Name,
				Message: fmt.Sprintf("%s pod %q is pending", priority, pod.Name),
			})
			return nil
		}
		if pod.Status.Phase == v1.PodUnknown {
			v.addError(&ValidationError{
				Kind:    "Pod",
				Name:    pod.Namespace + "/" + pod.Name,
				Message: fmt.Sprintf("%s pod %q is unknown phase", priority, pod.Name),
			})
			return nil
		}
		var notready []string
		for _, container := range pod.Status.ContainerStatuses {
			if !container.Ready {
				notready = append(notready, container.Name)
			}
		}
		if len(notready) != 0 {
			v.addError(&ValidationError{
				Kind:    "Pod",
				Name:    pod.Namespace + "/" + pod.Name,
				Message: fmt.Sprintf("%s pod %q is not ready (%s)", priority, pod.Name, strings.Join(notready, ",")),
			})

		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error listing Pods: %v", err)
	}

	for node, nodeMap := range masterWithoutPod {
		for app := range nodeMap {
			v.addError(&ValidationError{
				Kind:    "Node",
				Name:    node,
				Message: fmt.Sprintf("master %q is missing %s pod", node, app),
			})
		}
	}

	return nil
}

func (v *ValidationCluster) validateNodes(cloudGroups map[string]*cloudinstances.CloudInstanceGroup, groups []*kops.InstanceGroup) []v1.Node {
	var readyNodes []v1.Node
	groupsSeen := map[string]bool{}

	for _, cloudGroup := range cloudGroups {
		var allMembers []*cloudinstances.CloudInstance
		allMembers = append(allMembers, cloudGroup.Ready...)
		allMembers = append(allMembers, cloudGroup.NeedUpdate...)

		groupsSeen[cloudGroup.InstanceGroup.Name] = true
		numNodes := 0
		for _, m := range allMembers {
			if m.Status != cloudinstances.CloudInstanceStatusDetached {
				numNodes++
			}
		}
		if numNodes < cloudGroup.TargetSize {
			v.addError(&ValidationError{
				Kind: "InstanceGroup",
				Name: cloudGroup.InstanceGroup.Name,
				Message: fmt.Sprintf("InstanceGroup %q did not have enough nodes %d vs %d",
					cloudGroup.InstanceGroup.Name,
					numNodes,
					cloudGroup.TargetSize),
			})
		}

		for _, member := range allMembers {
			node := member.Node

			if node == nil {
				nodeExpectedToJoin := true
				if cloudGroup.InstanceGroup.Spec.Role == kops.InstanceGroupRoleBastion {
					// bastion nodes don't join the cluster
					nodeExpectedToJoin = false
				}

				if nodeExpectedToJoin {
					v.addError(&ValidationError{
						Kind:    "Machine",
						Name:    member.ID,
						Message: fmt.Sprintf("machine %q has not yet joined cluster", member.ID),
					})
				}
				continue
			}

			role := strings.ToLower(string(cloudGroup.InstanceGroup.Spec.Role))
			if role == "" {
				role = "node"
			}

			n := &ValidationNode{
				Name:     node.Name,
				Zone:     node.ObjectMeta.Labels["topology.kubernetes.io/zone"],
				Hostname: node.ObjectMeta.Labels["kubernetes.io/hostname"],
				Role:     role,
				Status:   getNodeReadyStatus(node),
			}
			if n.Zone == "" {
				n.Zone = node.ObjectMeta.Labels["failure-domain.beta.kubernetes.io/zone"]
			}

			ready := isNodeReady(node)
			if ready {
				readyNodes = append(readyNodes, *node)
			}

			if n.Role == "master" {
				if !ready {
					v.addError(&ValidationError{
						Kind:    "Node",
						Name:    node.Name,
						Message: fmt.Sprintf("master %q is not ready", node.Name),
					})
				}

				v.Nodes = append(v.Nodes, n)
			} else if n.Role == "node" {
				if !ready {
					v.addError(&ValidationError{
						Kind:    "Node",
						Name:    node.Name,
						Message: fmt.Sprintf("node %q is not ready", node.Name),
					})
				}

				v.Nodes = append(v.Nodes, n)
			} else {
				klog.Warningf("ignoring node with role %q", n.Role)
			}
		}
	}

	for _, ig := range groups {
		if !groupsSeen[ig.Name] {
			v.addError(&ValidationError{
				Kind:    "InstanceGroup",
				Name:    ig.Name,
				Message: fmt.Sprintf("InstanceGroup %q is missing from the cloud provider", ig.Name),
			})
		}
	}

	return readyNodes
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	"context"
	"fmt"
	"net"
	"net/url"
	"strings"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/tools/pager"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/cloudinstances"
	"k8s.io/kops/pkg/dns"
)

// ValidationCluster uses a cluster to validate.
type ValidationCluster struct {
	Failures []*ValidationError `json:"failures,omitempty"`

	Nodes []*ValidationNode `json:"nodes,omitempty"`
}

// ValidationError holds a validation failure
type ValidationError struct {
	Kind    string `json:"type,omitempty"`
	Name    string `json:"name,omitempty"`
	Message string `json:"message,omitempty"`
	// The InstanceGroup field is used to indicate which instance group this validation error is coming from
	InstanceGroup *kops.InstanceGroup `json:"instanceGroup,omitempty"`
}

type ClusterValidator interface {
	// Validate validates a k8s cluster
	Validate() (*ValidationCluster, error)
}

type clusterValidatorImpl struct {
	cluster        *kops.Cluster
	cloud          fi.Cloud
	instanceGroups []*kops.InstanceGroup
	host           string
	k8sClient      kubernetes.Interface
}

func (v *ValidationCluster) addError(failure *ValidationError) {
	v.Failures = append(v.Failures, failure)
}

// ValidationNode represents the validation status for a node
type ValidationNode struct {
	Name     string             `json:"name,omitempty"`
	Zone     string             `json:"zone,omitempty"`
	Role     string             `json:"role,omitempty"`
	Hostname string             `json:"hostname,omitempty"`
	Status   v1.ConditionStatus `json:"status,omitempty"`
}

// hasPlaceHolderIP checks if the API DNS has been updated.
func hasPlaceHolderIP(host string) (bool, error) {
	apiAddr, err := url.Parse(host)
	if err != nil {
		return true, fmt.Errorf("unable to parse Kubernetes cluster API URL: %v", err)
	}
	hostAddrs, err := net.LookupHost(apiAddr.Hostname())
	if err != nil {
		return true, fmt.Errorf("unable to resolve Kubernetes cluster API URL dns: %v", err)
	}

	for _, h := range hostAddrs {
		if h == "203.0.113.123" {
			return true, nil
		}
	}

	return false, nil
}

func NewClusterValidator(cluster *kops.Cluster, cloud fi.Cloud, instanceGroupList *kops.InstanceGroupList, host string, k8sClient kubernetes.Interface) (ClusterValidator, error) {
	var instanceGroups []*kops.InstanceGroup

	for i := range instanceGroupList.Items {
		ig := &instanceGroupList.Items[i]
		instanceGroups = append(instanceGroups, ig)
	}

	if len(instanceGroups) == 0 {
		return nil, fmt.Errorf("no InstanceGroup objects found")
	}

	return &clusterValidatorImpl{
		cluster:        cluster,
		cloud:          cloud,
		instanceGroups: instanceGroups,
		host:           host,
		k8sClient:      k8sClient,
	}, nil
}

func (v *clusterValidatorImpl) Validate() (*ValidationCluster, error) {
	ctx := context.TODO()

	clusterName := v.cluster.Name

	validation := &ValidationCluster{}

	// Do not use if we are running gossip
	if !dns.IsGossipHostname(clusterName) {
		hasPlaceHolderIPAddress, err := hasPlaceHolderIP(v.host)
		if err != nil {
			return nil, err
		}

		if hasPlaceHolderIPAddress {
			message := "Validation Failed\n\n" +
				"The dns-controller Kubernetes deployment has not updated the Kubernetes cluster's API DNS entry to the correct IP address." +
				"  The API DNS IP address is the placeholder address that kops creates: 203.0.113.123." +
				"  Please wait about 5-10 minutes for a master to start, dns-controller to launch, and DNS to propagate." +
				"  The protokube container and dns-controller deployment logs may contain more diagnostic information." +
				"  Etcd and the API DNS entries must be updated for a kops Kubernetes cluster to start."
			validation.addError(&ValidationError{
				Kind:    "dns",
				Name:    "apiserver",
				Message: message,
			})
			return validation, nil
		}
	}

	nodeList, err := v.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
	if err != nil {
		return nil, fmt.Errorf("error listing nodes: %v", err)
	}

	warnUnmatched := false
	cloudGroups, err := v.cloud.GetCloudGroups(v.cluster, v.instanceGroups, warnUnmatched, nodeList.Items)
	if err != nil {
		return nil, err
	}
	readyNodes, nodeInstanceGroupMapping := validation.validateNodes(cloudGroups, v.instanceGroups)

	if err := validation.collectPodFailures(ctx, v.k8sClient, readyNodes, nodeInstanceGroupMapping); err != nil {
		return nil, fmt.Errorf("cannot get pod health for %q: %v", clusterName, err)
	}

	return validation, nil
}

var masterStaticPods = []string{
	"kube-apiserver",
	"kube-controller-manager",
	"kube-scheduler",
}

func (v *ValidationCluster) collectPodFailures(ctx context.Context, client kubernetes.Interface, nodes []v1.Node,
	nodeInstanceGroupMapping map[string]*kops.InstanceGroup) error {
	masterWithoutPod := map[string]map[string]bool{}
	nodeByAddress := map[string]string{}

	for _, node := range nodes {
		labels := node.GetLabels()
		if labels != nil && labels["kubernetes.io/role"] == "master" {
			masterWithoutPod[node.Name] = map[string]bool{}
			for _, pod := range masterStaticPods {
				masterWithoutPod[node.Name][pod] = true
			}
		}

		for _, nodeAddress := range node.Status.Addresses {
			nodeByAddress[nodeAddress.Address] = node.Name
		}
	}

	err := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
		return client.CoreV1().Pods(metav1.NamespaceAll).List(ctx, opts)
	})).EachListItem(context.TODO(), metav1.ListOptions{}, func(obj runtime.Object) error {
		pod := obj.(*v1.Pod)

		app := pod.GetLabels()["k8s-app"]
		if pod.Namespace == "kube-system" && masterWithoutPod[nodeByAddress[pod.Status.HostIP]][app] {
			delete(masterWithoutPod[nodeByAddress[pod.Status.HostIP]], app)
		}

		priority := pod.Spec.PriorityClassName
		if priority != "system-cluster-critical" && priority != "system-node-critical" {
			return nil
		}
		if pod.Status.Phase == v1.PodSucceeded {
			return nil
		}

		var podNode *kops.InstanceGroup
		if priority == "system-node-critical" {
			podNode = nodeInstanceGroupMapping[nodeByAddress[pod.Status.HostIP]]
		}

		if pod.Status.Phase == v1.PodPending {
			v.addError(&ValidationError{
				Kind:          "Pod",
				Name:          pod.Namespace + "/" + pod.Name,
				Message:       fmt.Sprintf("%s pod %q is pending", priority, pod.Name),
				InstanceGroup: podNode,
			})
			return nil
		}
		if pod.Status.Phase == v1.PodUnknown {
			v.addError(&ValidationError{
				Kind:          "Pod",
				Name:          pod.Namespace + "/" + pod.Name,
				Message:       fmt.Sprintf("%s pod %q is unknown phase", priority, pod.Name),
				InstanceGroup: podNode,
			})
			return nil
		}
		var notready []string
		for _, container := range pod.Status.ContainerStatuses {
			if !container.Ready {
				notready = append(notready, container.Name)
			}
		}
		if len(notready) != 0 {
			v.addError(&ValidationError{
				Kind:          "Pod",
				Name:          pod.Namespace + "/" + pod.Name,
				Message:       fmt.Sprintf("%s pod %q is not ready (%s)", priority, pod.Name, strings.Join(notready, ",")),
				InstanceGroup: podNode,
			})

		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error listing Pods: %v", err)
	}

	for node, nodeMap := range masterWithoutPod {
		for app := range nodeMap {
			v.addError(&ValidationError{
				Kind:          "Node",
				Name:          node,
				Message:       fmt.Sprintf("master %q is missing %s pod", node, app),
				InstanceGroup: nodeInstanceGroupMapping[node],
			})
		}
	}

	return nil
}

func (v *ValidationCluster) validateNodes(cloudGroups map[string]*cloudinstances.CloudInstanceGroup, groups []*kops.InstanceGroup) ([]v1.Node, map[string]*kops.InstanceGroup) {
	var readyNodes []v1.Node
	groupsSeen := map[string]bool{}
	nodeInstanceGroupMapping := map[string]*kops.InstanceGroup{}

	for _, cloudGroup := range cloudGroups {
		var allMembers []*cloudinstances.CloudInstance
		allMembers = append(allMembers, cloudGroup.Ready...)
		allMembers = append(allMembers, cloudGroup.NeedUpdate...)

		groupsSeen[cloudGroup.InstanceGroup.Name] = true
		numNodes := 0
		for _, m := range allMembers {
			if m.Status != cloudinstances.CloudInstanceStatusDetached {
				numNodes++
			}
		}
		if numNodes < cloudGroup.TargetSize {
			v.addError(&ValidationError{
				Kind: "InstanceGroup",
				Name: cloudGroup.InstanceGroup.Name,
				Message: fmt.Sprintf("InstanceGroup %q did not have enough nodes %d vs %d",
					cloudGroup.InstanceGroup.Name,
					numNodes,
					cloudGroup.TargetSize),
				InstanceGroup: cloudGroup.InstanceGroup,
			})
		}

		for _, member := range allMembers {
			node := member.Node

			if node == nil {
				nodeExpectedToJoin := true
				if cloudGroup.InstanceGroup.Spec.Role == kops.InstanceGroupRoleBastion {
					// bastion nodes don't join the cluster
					nodeExpectedToJoin = false
				}
				if member.State == cloudinstances.WarmPool {
					nodeExpectedToJoin = false
				}

				if nodeExpectedToJoin {
					v.addError(&ValidationError{
						Kind:          "Machine",
						Name:          member.ID,
						Message:       fmt.Sprintf("machine %q has not yet joined cluster", member.ID),
						InstanceGroup: cloudGroup.InstanceGroup,
					})
				}
				continue
			}

			nodeInstanceGroupMapping[node.Name] = cloudGroup.InstanceGroup

			role := strings.ToLower(string(cloudGroup.InstanceGroup.Spec.Role))
			if role == "" {
				role = "node"
			}

			n := &ValidationNode{
				Name:     node.Name,
				Zone:     node.ObjectMeta.Labels["topology.kubernetes.io/zone"],
				Hostname: node.ObjectMeta.Labels["kubernetes.io/hostname"],
				Role:     role,
				Status:   getNodeReadyStatus(node),
			}
			if n.Zone == "" {
				n.Zone = node.ObjectMeta.Labels["failure-domain.beta.kubernetes.io/zone"]
			}

			ready := isNodeReady(node)
			if ready {
				readyNodes = append(readyNodes, *node)
			}

			switch n.Role {
			case "master", "apiserver", "node":
				if !ready {
					v.addError(&ValidationError{
						Kind:          "Node",
						Name:          node.Name,
						Message:       fmt.Sprintf("node %q of role %q is not ready", node.Name, n.Role),
						InstanceGroup: cloudGroup.InstanceGroup,
					})
				}

				v.Nodes = append(v.Nodes, n)
			default:
				klog.Warningf("ignoring node with role %q", n.Role)

			}
		}
	}

	for _, ig := range groups {
		if !groupsSeen[ig.Name] {
			v.addError(&ValidationError{
				Kind:          "InstanceGroup",
				Name:          ig.Name,
				Message:       fmt.Sprintf("InstanceGroup %q is missing from the cloud provider", ig.Name),
				InstanceGroup: ig,
			})
		}
	}

	return readyNodes, nodeInstanceGroupMapping
}
<commit_msg>do not validate detached nodes<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	"context"
	"fmt"
	"net"
	"net/url"
	"strings"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/tools/pager"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/cloudinstances"
	"k8s.io/kops/pkg/dns"
)

// ValidationCluster uses a cluster to validate.
type ValidationCluster struct {
	Failures []*ValidationError `json:"failures,omitempty"`

	Nodes []*ValidationNode `json:"nodes,omitempty"`
}

// ValidationError holds a validation failure
type ValidationError struct {
	Kind    string `json:"type,omitempty"`
	Name    string `json:"name,omitempty"`
	Message string `json:"message,omitempty"`
	// The InstanceGroup field is used to indicate which instance group this validation error is coming from
	InstanceGroup *kops.InstanceGroup `json:"instanceGroup,omitempty"`
}

type ClusterValidator interface {
	// Validate validates a k8s cluster
	Validate() (*ValidationCluster, error)
}

type clusterValidatorImpl struct {
	cluster        *kops.Cluster
	cloud          fi.Cloud
	instanceGroups []*kops.InstanceGroup
	host           string
	k8sClient      kubernetes.Interface
}

func (v *ValidationCluster) addError(failure *ValidationError) {
	v.Failures = append(v.Failures, failure)
}

// ValidationNode represents the validation status for a node
type ValidationNode struct {
	Name     string             `json:"name,omitempty"`
	Zone     string             `json:"zone,omitempty"`
	Role     string             `json:"role,omitempty"`
	Hostname string             `json:"hostname,omitempty"`
	Status   v1.ConditionStatus `json:"status,omitempty"`
}

// hasPlaceHolderIP checks if the API DNS has been updated.
func hasPlaceHolderIP(host string) (bool, error) {
	apiAddr, err := url.Parse(host)
	if err != nil {
		return true, fmt.Errorf("unable to parse Kubernetes cluster API URL: %v", err)
	}
	hostAddrs, err := net.LookupHost(apiAddr.Hostname())
	if err != nil {
		return true, fmt.Errorf("unable to resolve Kubernetes cluster API URL dns: %v", err)
	}

	for _, h := range hostAddrs {
		if h == "203.0.113.123" {
			return true, nil
		}
	}

	return false, nil
}

func NewClusterValidator(cluster *kops.Cluster, cloud fi.Cloud, instanceGroupList *kops.InstanceGroupList, host string, k8sClient kubernetes.Interface) (ClusterValidator, error) {
	var instanceGroups []*kops.InstanceGroup

	for i := range instanceGroupList.Items {
		ig := &instanceGroupList.Items[i]
		instanceGroups = append(instanceGroups, ig)
	}

	if len(instanceGroups) == 0 {
		return nil, fmt.Errorf("no InstanceGroup objects found")
	}

	return &clusterValidatorImpl{
		cluster:        cluster,
		cloud:          cloud,
		instanceGroups: instanceGroups,
		host:           host,
		k8sClient:      k8sClient,
	}, nil
}

func (v *clusterValidatorImpl) Validate() (*ValidationCluster, error) {
	ctx := context.TODO()

	clusterName := v.cluster.Name

	validation := &ValidationCluster{}

	// Do not use if we are running gossip
	if !dns.IsGossipHostname(clusterName) {
		hasPlaceHolderIPAddress, err := hasPlaceHolderIP(v.host)
		if err != nil {
			return nil, err
		}

		if hasPlaceHolderIPAddress {
			message := "Validation Failed\n\n" +
				"The dns-controller Kubernetes deployment has not updated the Kubernetes cluster's API DNS entry to the correct IP address." +
				"  The API DNS IP address is the placeholder address that kops creates: 203.0.113.123." +
				"  Please wait about 5-10 minutes for a master to start, dns-controller to launch, and DNS to propagate." +
				"  The protokube container and dns-controller deployment logs may contain more diagnostic information." +
				"  Etcd and the API DNS entries must be updated for a kops Kubernetes cluster to start."
			validation.addError(&ValidationError{
				Kind:    "dns",
				Name:    "apiserver",
				Message: message,
			})
			return validation, nil
		}
	}

	nodeList, err := v.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
	if err != nil {
		return nil, fmt.Errorf("error listing nodes: %v", err)
	}

	warnUnmatched := false
	cloudGroups, err := v.cloud.GetCloudGroups(v.cluster, v.instanceGroups, warnUnmatched, nodeList.Items)
	if err != nil {
		return nil, err
	}
	readyNodes, nodeInstanceGroupMapping := validation.validateNodes(cloudGroups, v.instanceGroups)

	if err := validation.collectPodFailures(ctx, v.k8sClient, readyNodes, nodeInstanceGroupMapping); err != nil {
		return nil, fmt.Errorf("cannot get pod health for %q: %v", clusterName, err)
	}

	return validation, nil
}

var masterStaticPods = []string{
	"kube-apiserver",
	"kube-controller-manager",
	"kube-scheduler",
}

func (v *ValidationCluster) collectPodFailures(ctx context.Context, client kubernetes.Interface, nodes []v1.Node,
	nodeInstanceGroupMapping map[string]*kops.InstanceGroup) error {
	masterWithoutPod := map[string]map[string]bool{}
	nodeByAddress := map[string]string{}

	for _, node := range nodes {
		labels := node.GetLabels()
		if labels != nil && labels["kubernetes.io/role"] == "master" {
			masterWithoutPod[node.Name] = map[string]bool{}
			for _, pod := range masterStaticPods {
				masterWithoutPod[node.Name][pod] = true
			}
		}

		for _, nodeAddress := range node.Status.Addresses {
			nodeByAddress[nodeAddress.Address] = node.Name
		}
	}

	err := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
		return client.CoreV1().Pods(metav1.NamespaceAll).List(ctx, opts)
	})).EachListItem(context.TODO(), metav1.ListOptions{}, func(obj runtime.Object) error {
		pod := obj.(*v1.Pod)

		app := pod.GetLabels()["k8s-app"]
		if pod.Namespace == "kube-system" && masterWithoutPod[nodeByAddress[pod.Status.HostIP]][app] {
			delete(masterWithoutPod[nodeByAddress[pod.Status.HostIP]], app)
		}

		priority := pod.Spec.PriorityClassName
		if priority != "system-cluster-critical" && priority != "system-node-critical" {
			return nil
		}
		if pod.Status.Phase == v1.PodSucceeded {
			return nil
		}

		var podNode *kops.InstanceGroup
		if priority == "system-node-critical" {
			podNode = nodeInstanceGroupMapping[nodeByAddress[pod.Status.HostIP]]
		}

		if pod.Status.Phase == v1.PodPending {
			v.addError(&ValidationError{
				Kind:          "Pod",
				Name:          pod.Namespace + "/" + pod.Name,
				Message:       fmt.Sprintf("%s pod %q is pending", priority, pod.Name),
				InstanceGroup: podNode,
			})
			return nil
		}
		if pod.Status.Phase == v1.PodUnknown {
			v.addError(&ValidationError{
				Kind:          "Pod",
				Name:          pod.Namespace + "/" + pod.Name,
				Message:       fmt.Sprintf("%s pod %q is unknown phase", priority, pod.Name),
				InstanceGroup: podNode,
			})
			return nil
		}
		var notready []string
		for _, container := range pod.Status.ContainerStatuses {
			if !container.Ready {
				notready = append(notready, container.Name)
			}
		}
		if len(notready) != 0 {
			v.addError(&ValidationError{
				Kind:          "Pod",
				Name:          pod.Namespace + "/" + pod.Name,
				Message:       fmt.Sprintf("%s pod %q is not ready (%s)", priority, pod.Name, strings.Join(notready, ",")),
				InstanceGroup: podNode,
			})

		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error listing Pods: %v", err)
	}

	for node, nodeMap := range masterWithoutPod {
		for app := range nodeMap {
			v.addError(&ValidationError{
				Kind:          "Node",
				Name:          node,
				Message:       fmt.Sprintf("master %q is missing %s pod", node, app),
				InstanceGroup: nodeInstanceGroupMapping[node],
			})
		}
	}

	return nil
}

func (v *ValidationCluster) validateNodes(cloudGroups map[string]*cloudinstances.CloudInstanceGroup, groups []*kops.InstanceGroup) ([]v1.Node, map[string]*kops.InstanceGroup) {
	var readyNodes []v1.Node
	groupsSeen := map[string]bool{}
	nodeInstanceGroupMapping := map[string]*kops.InstanceGroup{}

	for _, cloudGroup := range cloudGroups {
		var allMembers []*cloudinstances.CloudInstance
		allMembers = append(allMembers, cloudGroup.Ready...)
		allMembers = append(allMembers, cloudGroup.NeedUpdate...)

		groupsSeen[cloudGroup.InstanceGroup.Name] = true
		numNodes := 0
		for _, m := range allMembers {
			if m.Status != cloudinstances.CloudInstanceStatusDetached {
				numNodes++
			}
		}
		if numNodes < cloudGroup.TargetSize {
			v.addError(&ValidationError{
				Kind: "InstanceGroup",
				Name: cloudGroup.InstanceGroup.Name,
				Message: fmt.Sprintf("InstanceGroup %q did not have enough nodes %d vs %d",
					cloudGroup.InstanceGroup.Name,
					numNodes,
					cloudGroup.TargetSize),
				InstanceGroup: cloudGroup.InstanceGroup,
			})
		}

		for _, member := range allMembers {
			node := member.Node

			if node == nil {
				nodeExpectedToJoin := true
				if cloudGroup.InstanceGroup.Spec.Role == kops.InstanceGroupRoleBastion {
					// bastion nodes don't join the cluster
					nodeExpectedToJoin = false
				}
				if member.State == cloudinstances.WarmPool {
					nodeExpectedToJoin = false
				}

				if member.Status == cloudinstances.CloudInstanceStatusDetached {
					nodeExpectedToJoin = false
				}

				if nodeExpectedToJoin {
					v.addError(&ValidationError{
						Kind:          "Machine",
						Name:          member.ID,
						Message:       fmt.Sprintf("machine %q has not yet joined cluster", member.ID),
						InstanceGroup: cloudGroup.InstanceGroup,
					})
				}
				continue
			}

			nodeInstanceGroupMapping[node.Name] = cloudGroup.InstanceGroup

			role := strings.ToLower(string(cloudGroup.InstanceGroup.Spec.Role))
			if role == "" {
				role = "node"
			}

			n := &ValidationNode{
				Name:     node.Name,
				Zone:     node.ObjectMeta.Labels["topology.kubernetes.io/zone"],
				Hostname: node.ObjectMeta.Labels["kubernetes.io/hostname"],
				Role:     role,
				Status:   getNodeReadyStatus(node),
			}
			if n.Zone == "" {
				n.Zone = node.ObjectMeta.Labels["failure-domain.beta.kubernetes.io/zone"]
			}

			ready := isNodeReady(node)
			if ready {
				readyNodes = append(readyNodes, *node)
			}

			switch n.Role {
			case "master", "apiserver", "node":
				if !ready {
					v.addError(&ValidationError{
						Kind:          "Node",
						Name:          node.Name,
						Message:       fmt.Sprintf("node %q of role %q is not ready", node.Name, n.Role),
						InstanceGroup: cloudGroup.InstanceGroup,
					})
				}

				v.Nodes = append(v.Nodes, n)
			default:
				klog.Warningf("ignoring node with role %q", n.Role)

			}
		}
	}

	for _, ig := range groups {
		if !groupsSeen[ig.Name] {
			v.addError(&ValidationError{
				Kind:          "InstanceGroup",
				Name:          ig.Name,
				Message:       fmt.Sprintf("InstanceGroup %q is missing from the cloud provider", ig.Name),
				InstanceGroup: ig,
			})
		}
	}

	return readyNodes, nodeInstanceGroupMapping
}
<|endoftext|>


<commit_before>package services

import (
	"bytes"
	"fmt"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/levels"
	"github.com/satori/go.uuid"
	"kubevirt/core/pkg/api"
	"kubevirt/core/pkg/kubecli"
	"kubevirt/core/pkg/middleware"
	"kubevirt/core/pkg/precond"
	"regexp"
)

type VMService interface {
	StartVMRaw(*api.VM, []byte) error
	DeleteVM(*api.VM) error
	PrepareMigration(*api.VM) error
}

type vmService struct {
	logger          levels.Levels
	KubeCli         kubecli.KubeCli `inject:""`
	TemplateService TemplateService `inject:""`
}

func (v *vmService) StartVMRaw(vm *api.VM, rawXML []byte) error {
	precond.MustNotBeNil(vm)
	precond.MustNotBeNil(rawXML)
	precond.MustNotBeEmpty(vm.Name)

	// Broken racy approach to at least not start multiple VMs during demos
	pods, err := v.KubeCli.GetPodsByLabel("domain", vm.Name)
	if err != nil {
		return err
	}

	// Pod for VM already exists
	if len(pods) > 0 {
		return middleware.NewResourceExistsError("VM", vm.Name)
	}

	if vm.UUID == uuid.Nil {
		vm.UUID = uuid.NewV4()
		//TODO when we can serialize VMs to XML, we can get rid of this
		r := regexp.MustCompile("</domain[\\s]*>")
		rawXML = r.ReplaceAll(rawXML, []byte(fmt.Sprintf("<uuid>%s</uuid></domain>", vm.UUID.String())))
	}

	templateBuffer := new(bytes.Buffer)
	if err := v.TemplateService.RenderLaunchManifest(vm, rawXML, templateBuffer); err != nil {
		return err
	}

	if err := v.KubeCli.CreatePod(templateBuffer); err != nil {
		return err
	}
	v.logger.Info().Log("action", "StartVMRaw", "object", "VM", "UUID", vm.UUID, "name", vm.Name)
	return nil
}

func (v *vmService) DeleteVM(vm *api.VM) error {
	precond.MustNotBeNil(vm)
	precond.MustNotBeEmpty(vm.Name)

	if err := v.KubeCli.DeletePodsByLabel("domain", vm.Name); err != nil {
		return err
	}
	v.logger.Info().Log("action", "DeleteVM", "object", "VM", "UUID", vm.UUID, "name", vm.Name)
	return nil
}

func NewVMService(logger log.Logger) VMService {
	precond.MustNotBeNil(logger)

	svc := vmService{logger: levels.New(logger).With("component", "VMService")}
	return &svc
}

func (v *vmService) PrepareMigration(vm *api.VM) error {
	precond.MustNotBeNil(vm)
	precond.MustNotBeEmpty(vm.Name)
	precond.MustBeTrue(len(vm.NodeSelector) > 0)

	// Broken racy approach to at least not start multiple VMs during demos
	pods, err := v.KubeCli.GetPodsByLabel("domain", vm.Name)
	if err != nil {
		return err
	}

	// Pod for VM does not exist
	if len(pods) == 0 {
		return middleware.NewResourceNotFoundError("VM", vm.Name)
	}

	// If there is more then there is already a migration going on
	if len(pods) > 1 {
		return middleware.NewResourceConflictError(fmt.Sprintf("VM %s is already migrating", vm.Name))
	}

	templateBuffer := new(bytes.Buffer)
	if err := v.TemplateService.RenderMigrationManifest(vm, templateBuffer); err != nil {
		return err
	}
	if err := v.KubeCli.CreatePod(templateBuffer); err != nil {
		return err
	}
	v.logger.Info().Log("action", "PrepareMigration", "object", "VM", "UUID", vm.UUID, "name", vm.Name)
	return nil
}
<commit_msg>Only count running or uknown pods when evaluating the cluster situation<commit_after>package services

import (
	"bytes"
	"fmt"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/levels"
	"github.com/satori/go.uuid"
	"kubevirt/core/pkg/api"
	"kubevirt/core/pkg/kubecli"
	"kubevirt/core/pkg/kubecli/v1.2"
	"kubevirt/core/pkg/middleware"
	"kubevirt/core/pkg/precond"
	"regexp"
)

type VMService interface {
	StartVMRaw(*api.VM, []byte) error
	DeleteVM(*api.VM) error
	PrepareMigration(*api.VM) error
}

type vmService struct {
	logger          levels.Levels
	KubeCli         kubecli.KubeCli `inject:""`
	TemplateService TemplateService `inject:""`
}

func (v *vmService) StartVMRaw(vm *api.VM, rawXML []byte) error {
	precond.MustNotBeNil(vm)
	precond.MustNotBeNil(rawXML)
	precond.MustNotBeEmpty(vm.Name)

	// Broken racy approach to at least not start multiple VMs during demos
	pods, err := v.KubeCli.GetPodsByLabel("domain", vm.Name)
	if err != nil {
		return err
	}

	// Search for accepted and not finished pods
	c := getUnfinishedPods(pods)

	// Pod for VM already exists
	if c > 0 {
		return middleware.NewResourceExistsError("VM", vm.Name)
	}

	if vm.UUID == uuid.Nil {
		vm.UUID = uuid.NewV4()
		//TODO when we can serialize VMs to XML, we can get rid of this
		r := regexp.MustCompile("</domain[\\s]*>")
		rawXML = r.ReplaceAll(rawXML, []byte(fmt.Sprintf("<uuid>%s</uuid></domain>", vm.UUID.String())))
	}

	templateBuffer := new(bytes.Buffer)
	if err := v.TemplateService.RenderLaunchManifest(vm, rawXML, templateBuffer); err != nil {
		return err
	}

	if err := v.KubeCli.CreatePod(templateBuffer); err != nil {
		return err
	}
	v.logger.Info().Log("action", "StartVMRaw", "object", "VM", "UUID", vm.UUID, "name", vm.Name)
	return nil
}

func (v *vmService) DeleteVM(vm *api.VM) error {
	precond.MustNotBeNil(vm)
	precond.MustNotBeEmpty(vm.Name)

	if err := v.KubeCli.DeletePodsByLabel("domain", vm.Name); err != nil {
		return err
	}
	v.logger.Info().Log("action", "DeleteVM", "object", "VM", "UUID", vm.UUID, "name", vm.Name)
	return nil
}

func NewVMService(logger log.Logger) VMService {
	precond.MustNotBeNil(logger)

	svc := vmService{logger: levels.New(logger).With("component", "VMService")}
	return &svc
}

func (v *vmService) PrepareMigration(vm *api.VM) error {
	precond.MustNotBeNil(vm)
	precond.MustNotBeEmpty(vm.Name)
	precond.MustBeTrue(len(vm.NodeSelector) > 0)

	// Broken racy approach to at least not start multiple VMs during demos
	pods, err := v.KubeCli.GetPodsByLabel("domain", vm.Name)
	if err != nil {
		return err
	}

	// Pod for VM does not exist
	if len(pods) == 0 {
		return middleware.NewResourceNotFoundError("VM", vm.Name)
	}

	// Search for accepted and not finished pods
	c := getUnfinishedPods(pods)

	// If there are more than one pod in other states than Succeeded or Failed we can't go on
	if c > 1 {
		return middleware.NewResourceConflictError(fmt.Sprintf("VM %s is already migrating", vm.Name))
	}

	templateBuffer := new(bytes.Buffer)
	if err := v.TemplateService.RenderMigrationManifest(vm, templateBuffer); err != nil {
		return err
	}
	if err := v.KubeCli.CreatePod(templateBuffer); err != nil {
		return err
	}
	v.logger.Info().Log("action", "PrepareMigration", "object", "VM", "UUID", vm.UUID, "name", vm.Name)
	return nil
}

func getUnfinishedPods(pods []v1_2.Pod) int {
	c := 0
	states := []string{"Running", "Pending", "Unknown"}
	for _, pod := range pods {
		for _, state := range states {
			if pod.Status.Phase == state {
				c++
			}
		}
	}
	return c
}
<|endoftext|>


<commit_before>package auth_test

import (
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
	"time"

	"github.com/casbin/casbin/v2"
	qt "github.com/frankban/quicktest"
	"github.com/google/uuid"
	"github.com/gorilla/mux"
	"github.com/rs/zerolog"

	"github.com/gilcrest/go-api-basic/domain/app"
	"github.com/gilcrest/go-api-basic/domain/audit"
	"github.com/gilcrest/go-api-basic/domain/auth"
	"github.com/gilcrest/go-api-basic/domain/logger"
	"github.com/gilcrest/go-api-basic/domain/org"
	"github.com/gilcrest/go-api-basic/domain/person"
	"github.com/gilcrest/go-api-basic/domain/user"
)

func TestNewProvider(t *testing.T) {
	t.Run("google", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("GoOgLe")
		c.Assert(p, qt.Equals, auth.Google)
	})
	t.Run("apple", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("ApPlE")
		c.Assert(p, qt.Equals, auth.Apple)
	})
	t.Run("invalid", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("anything else!")
		c.Assert(p, qt.Equals, auth.Invalid)
	})
}

func TestProvider_String(t *testing.T) {
	t.Run("google", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("GoOgLe")
		provider := p.String()
		c.Assert(provider, qt.Equals, "google")
	})
	t.Run("apple", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("APPLe")
		provider := p.String()
		c.Assert(provider, qt.Equals, "apple")
	})
	t.Run("invalid", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("anything else")
		provider := p.String()
		c.Assert(provider, qt.Equals, "invalid_provider")
	})
}

func TestCasbinAuthorizer_Authorize(t *testing.T) {
	t.Run("valid user", func(t *testing.T) {
		c := qt.New(t)

		lgr := logger.NewLogger(os.Stdout, zerolog.DebugLevel, true)
		req := httptest.NewRequest(http.MethodGet, "/api/v1/ping", nil)
		a := app.App{}
		u := user.User{
			ID:       uuid.Nil,
			Username: "dan@dangillis.dev",
			Org:      org.Org{},
			Profile:  person.Profile{},
		}
		adt := audit.Audit{
			App:    a,
			User:   u,
			Moment: time.Now(),
		}
		// initialize casbin enforcer (using config files for now, will migrate to db)
		casbinEnforcer, err := casbin.NewEnforcer("../../config/rbac_model.conf", "../../config/rbac_policy.csv")
		if err != nil {
			c.Fatal("casbin.NewEnforcer error")
		}
		ca := auth.CasbinAuthorizer{Enforcer: casbinEnforcer}

		// Authorize must be tested inside a handler as it uses mux.CurrentRoute
		testAuthorizeHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			err = ca.Authorize(lgr, r, adt)
			c.Assert(err, qt.IsNil)
		})

		rr := httptest.NewRecorder()

		rtr := mux.NewRouter()
		rtr.Handle("/api/v1/ping", testAuthorizeHandler).Methods(http.MethodGet)
		rtr.ServeHTTP(rr, req)

		// If there is any issues with the Access Token, the body
		// should be empty and the status code should be 401
		c.Assert(rr.Code, qt.Equals, http.StatusOK)

	})
}
<commit_msg>switch email<commit_after>package auth_test

import (
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
	"time"

	"github.com/casbin/casbin/v2"
	qt "github.com/frankban/quicktest"
	"github.com/google/uuid"
	"github.com/gorilla/mux"
	"github.com/rs/zerolog"

	"github.com/gilcrest/go-api-basic/domain/app"
	"github.com/gilcrest/go-api-basic/domain/audit"
	"github.com/gilcrest/go-api-basic/domain/auth"
	"github.com/gilcrest/go-api-basic/domain/logger"
	"github.com/gilcrest/go-api-basic/domain/org"
	"github.com/gilcrest/go-api-basic/domain/person"
	"github.com/gilcrest/go-api-basic/domain/user"
)

func TestNewProvider(t *testing.T) {
	t.Run("google", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("GoOgLe")
		c.Assert(p, qt.Equals, auth.Google)
	})
	t.Run("apple", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("ApPlE")
		c.Assert(p, qt.Equals, auth.Apple)
	})
	t.Run("invalid", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("anything else!")
		c.Assert(p, qt.Equals, auth.Invalid)
	})
}

func TestProvider_String(t *testing.T) {
	t.Run("google", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("GoOgLe")
		provider := p.String()
		c.Assert(provider, qt.Equals, "google")
	})
	t.Run("apple", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("APPLe")
		provider := p.String()
		c.Assert(provider, qt.Equals, "apple")
	})
	t.Run("invalid", func(t *testing.T) {
		c := qt.New(t)
		p := auth.NewProvider("anything else")
		provider := p.String()
		c.Assert(provider, qt.Equals, "invalid_provider")
	})
}

func TestCasbinAuthorizer_Authorize(t *testing.T) {
	t.Run("valid user", func(t *testing.T) {
		c := qt.New(t)

		lgr := logger.NewLogger(os.Stdout, zerolog.DebugLevel, true)
		req := httptest.NewRequest(http.MethodGet, "/api/v1/ping", nil)
		a := app.App{}
		u := user.User{
			ID:       uuid.Nil,
			Username: "otto.maddox711@gmail.com",
			Org:      org.Org{},
			Profile:  person.Profile{},
		}
		adt := audit.Audit{
			App:    a,
			User:   u,
			Moment: time.Now(),
		}
		// initialize casbin enforcer (using config files for now, will migrate to db)
		casbinEnforcer, err := casbin.NewEnforcer("../../config/rbac_model.conf", "../../config/rbac_policy.csv")
		if err != nil {
			c.Fatal("casbin.NewEnforcer error")
		}
		ca := auth.CasbinAuthorizer{Enforcer: casbinEnforcer}

		// Authorize must be tested inside a handler as it uses mux.CurrentRoute
		testAuthorizeHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			err = ca.Authorize(lgr, r, adt)
			c.Assert(err, qt.IsNil)
		})

		rr := httptest.NewRecorder()

		rtr := mux.NewRouter()
		rtr.Handle("/api/v1/ping", testAuthorizeHandler).Methods(http.MethodGet)
		rtr.ServeHTTP(rr, req)

		// If there is any issues with the Access Token, the body
		// should be empty and the status code should be 401
		c.Assert(rr.Code, qt.Equals, http.StatusOK)

	})
}
<|endoftext|>


<commit_before>package database

import (
	"strings"

	"github.com/paultag/go-dictd/dictd"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/util"

	"github.com/jamesturk/go-jellyfish"
)

/*
 *
 */
func NewLevelDBDatabase(path string, description string) (*LevelDBDatabase, error) {
	db, err := leveldb.OpenFile(path, nil)
	if err != nil {
		return nil, err
	}

	databaseBackend := LevelDBDatabase{
		description: description,
		db:          db,
	}

	return &databaseBackend, nil
}

/*
 *
 */
type LevelDBDatabase struct {
	dictd.Database

	description string
	db          *leveldb.DB
}

/*
 *
 */
func (this *LevelDBDatabase) Match(name string, query string, strat string) (defs []*dictd.Definition) {
	query = strings.ToLower(query)
	var results []string

	switch strat {
	case "prefix":
		results = this.scanPrefix(query)
	case "soundex":
		results = this.matchSoundex(query)
	case "levenshtein":
		results = this.scanLevenshtein(query, 1)
	}

	for _, el := range results {
		def := &dictd.Definition{
			DictDatabase:     this,
			DictDatabaseName: name,
			Word:             el,
		}
		defs = append(defs, def)
	}

	return
}

/*
 *
 */
func (this *LevelDBDatabase) Define(name string, query string) []*dictd.Definition {
	query = strings.ToLower(query)
	data, err := this.db.Get([]byte("\n"+query), nil)
	if err != nil {
		/* If we don't have the key, let's bail out. */
		return make([]*dictd.Definition, 0)
	}
	els := make([]*dictd.Definition, 1)
	els[0] = &dictd.Definition{
		DictDatabase:     this,
		DictDatabaseName: name,
		Word:             query,
		Definition:       string(data),
	}
	return els
}

/*
 *
 */
func (this *LevelDBDatabase) Info(name string) string {
	return "Foo"
}

/*
 *
 */
func (this *LevelDBDatabase) Description(name string) string {
	return this.description
}

/*
 * DB Specific calls below
 */

func (this *LevelDBDatabase) writeIndex(namespace string, key string, word string) {
	var values []string

	data, err := this.db.Get([]byte(namespace+"\n"+key), nil)

	if err != nil {
		values = []string{}
	} else {
		/* Values are newline delimed */
		values = strings.Split(string(data), "\n")
	}

	for _, el := range values {
		if el == word {
			return
		}
	}

	values = append(values, word)

	this.db.Put(
		[]byte(namespace+"\n"+key),
		[]byte(strings.Join(values, "\n")),
		nil,
	)

	/* Right, so key's not in the list. */
}

func (this *LevelDBDatabase) WriteDefinition(word string, definition string) {
	/* Right, now let's build up indexes on the word
	 *
	 * Critically, RFC2229 forbids commands to have newlines in them, even
	 * escaped. So, we'll use newlines to write out a prefix. This lets us
	 * work all sorts of magic on the keys and "namespace" them. */

	this.db.Put([]byte("\n"+word), []byte(definition), nil)

	/* Right, now let's build up some indexes */
	this.writeIndex("soundex", jellyfish.Soundex(word), word)
}

/*
 *
 *     MATCHERS
 *
 *
 *
 *
 */

/*
 */
func (this *LevelDBDatabase) scanLevenshtein(query string, threshold int) (ret []string) {
	iter := this.db.NewIterator(util.BytesPrefix([]byte("\n")), nil)
	for iter.Next() {
		key := string(iter.Key())[1:]
		distance := jellyfish.Levenshtein(query, key)
		if distance <= threshold {
			/* XXX: Return ordered by distance? */
			ret = append(ret, key)
		}
	}
	iter.Release()
	return
}

/*
 */
func (this *LevelDBDatabase) scanPrefix(query string) (ret []string) {
	query = "\n" + query /* See namespacing code */

	iter := this.db.NewIterator(util.BytesPrefix([]byte(query)), nil)

	for iter.Next() {
		word := string(iter.Key())[1:]
		ret = append(ret, word)
	}
	iter.Release()
	return
}

/*
 */
func (this *LevelDBDatabase) matchSoundex(query string) (ret []string) {
	query = "soundex\n" + jellyfish.Soundex(query) /* See namespacing code */
	data, err := this.db.Get([]byte(query), nil)
	if err != nil {
		return []string{}
	}
	return strings.Split(string(data), "\n")
}
<commit_msg>Add in metaphone voodoo<commit_after>package database

import (
	"strings"

	"github.com/paultag/go-dictd/dictd"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/util"

	"github.com/jamesturk/go-jellyfish"
)

/*
 *
 */
func NewLevelDBDatabase(path string, description string) (*LevelDBDatabase, error) {
	db, err := leveldb.OpenFile(path, nil)
	if err != nil {
		return nil, err
	}

	databaseBackend := LevelDBDatabase{
		description: description,
		db:          db,
	}

	return &databaseBackend, nil
}

/*
 *
 */
type LevelDBDatabase struct {
	dictd.Database

	description string
	db          *leveldb.DB
}

/*
 *
 */
func (this *LevelDBDatabase) Match(name string, query string, strat string) (defs []*dictd.Definition) {
	query = strings.ToLower(query)
	var results []string

	switch strat {
	case "metaphone", ".":
		results = this.matchMetaphone(query)
	case "prefix":
		results = this.scanPrefix(query)
	case "soundex":
		results = this.matchSoundex(query)
	case "levenshtein":
		results = this.scanLevenshtein(query, 1)
	}

	for _, el := range results {
		def := &dictd.Definition{
			DictDatabase:     this,
			DictDatabaseName: name,
			Word:             el,
		}
		defs = append(defs, def)
	}

	return
}

/*
 *
 */
func (this *LevelDBDatabase) Define(name string, query string) []*dictd.Definition {
	query = strings.ToLower(query)
	data, err := this.get("", query)
	if err != nil {
		/* If we don't have the key, let's bail out. */
		return make([]*dictd.Definition, 0)
	}
	els := make([]*dictd.Definition, 1)
	els[0] = &dictd.Definition{
		DictDatabase:     this,
		DictDatabaseName: name,
		Word:             query,
		Definition:       string(data),
	}
	return els
}

/*
 *
 */
func (this *LevelDBDatabase) Info(name string) string {
	return "Foo"
}

/*
 *
 */
func (this *LevelDBDatabase) Description(name string) string {
	return this.description
}

/*
 * DB Specific calls below
 */

/*
 */
func (this *LevelDBDatabase) write(namespace string, key string, value string) {
	query := namespace + "\n" + key
	this.db.Put([]byte(query), []byte(value), nil)
}

/*
 */
func (this *LevelDBDatabase) get(namespace string, key string) (value string, err error) {
	data, err := this.db.Get([]byte(namespace+"\n"+key), nil)
	return string(data), err
}

/*
 */
func (this *LevelDBDatabase) writeIndex(namespace string, key string, word string) {
	var values []string

	data, err := this.get(namespace, key)

	if err != nil {
		values = []string{}
	} else {
		/* Values are newline delimed */
		values = strings.Split(string(data), "\n")
	}

	for _, el := range values {
		if el == word {
			return
		}
	}

	values = append(values, word)
	this.write(namespace, key, strings.Join(values, "\n"))
}

func (this *LevelDBDatabase) WriteDefinition(word string, definition string) {
	/* Right, now let's build up indexes on the word */

	this.write("", word, definition) /* no namespace for words */

	/* Right, now let's build up some indexes */
	this.writeIndex("soundex", jellyfish.Soundex(word), word)

	if len(word) > 2 { /* Fixme? */
		metaWords := jellyfish.Metaphone(word)

		/* FO BA BAR BAZ */
		for _, el := range strings.Split(metaWords, " ") {
			this.writeIndex("metaphone", el, word)
		}
	}
}

/*
 *
 *     MATCHERS
 *
 *
 *
 *
 */

/*
 */
func (this *LevelDBDatabase) scanLevenshtein(query string, threshold int) (ret []string) {
	iter := this.db.NewIterator(util.BytesPrefix([]byte("\n")), nil)
	for iter.Next() {
		key := string(iter.Key())[1:]
		distance := jellyfish.Levenshtein(query, key)
		if distance <= threshold {
			/* XXX: Return ordered by distance? */
			ret = append(ret, key)
		}
	}
	iter.Release()
	return
}

/*
 */
func (this *LevelDBDatabase) scanPrefix(query string) (ret []string) {
	query = "\n" + query /* See namespacing code */

	iter := this.db.NewIterator(util.BytesPrefix([]byte(query)), nil)

	for iter.Next() {
		word := string(iter.Key())[1:]
		ret = append(ret, word)
	}
	iter.Release()
	return
}

/*
 */
func (this *LevelDBDatabase) matchFromIndex(namespace string, key string) (ret []string) {
	data, err := this.get(namespace, key)
	if err != nil {
		return []string{}
	}
	return strings.Split(string(data), "\n")
}

/*
 */
func (this *LevelDBDatabase) matchSoundex(query string) (ret []string) {
	return this.matchFromIndex("soundex", jellyfish.Soundex(query))
}

/*
 */
func (this *LevelDBDatabase) matchMetaphone(query string) (ret []string) {
	meta := jellyfish.Metaphone(query)
	for _, el := range strings.Split(meta, " ") {
		ret = append(ret, this.matchFromIndex("metaphone", el)...)
	}

	/* right, so ret may have multiples */
	ordering := map[string]int{}
	for _, el := range ret {
		ordering[el] = 0 /* update this to count / sort */
	}

	r := []string{}
	for k, _ := range ordering {
		r = append(ret, k)
	}
	return r
}
<|endoftext|>


<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
)

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type systemMessage struct {
	level  systemMessageLevel
	line   int
	source string
	items  []item
}

type sectionLevel struct {
	char     rune // The adornment character used to describe the section
	overline bool // The section contains an overline
	length   int  // The length of the adornment lines
}

type sectionLevels []sectionLevel

func (s *sectionLevels) String() string {
	var out string
	for lvl, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			lvl+1, sec.char, sec.overline, sec.length)
	}
	return out
}

func (s *sectionLevels) Add(adornChar rune, overline bool, length int) int {
	lvl := s.Find(adornChar)
	if lvl > 0 {
		return lvl
	}
	*s = append(*s, sectionLevel{char: adornChar, overline: overline, length: length})
	return len(*s)
}

// Returns -1 if not found
func (s *sectionLevels) Find(adornChar rune) int {
	for lvl, sec := range *s {
		if sec.char == adornChar {
			return lvl + 1
		}
	}
	return -1
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

// Parse is the entry point for the reStructuredText parser.
func Parse(name, text string) (t *Tree, errors []error) {
	t = New(name)
	t.text = text
	_, errors = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{Name: name, sectionLevels: new(sectionLevels)}
}

type Tree struct {
	Name          string
	Nodes         *NodeList // The root node list
	Errors        []error
	text          string
	branch        *NodeList // The current branch to add nodes to
	lex           *lexer
	peekCount     int
	token         [3]item        // three-token look-ahead for parser.
	sectionLevel  int            // The current section level of parsing
	sectionLevels *sectionLevels // Encountered section levels
	id            int            // The unique id of the node in the tree
}

func (t *Tree) errorf(format string, args ...interface{}) {
	format = fmt.Sprintf("go-rst: %s:%d: %s\n", t.Name, t.lex.lineNumber(), format)
	t.Errors = append(t.Errors, fmt.Errorf(format, args...))
}

func (t *Tree) error(err error) {
	t.errorf("%s\n", err)
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.Nodes = nil
	t.branch = nil
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.lex = nil
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, errors []error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, t.Errors
}

func (t *Tree) parse(tree *Tree) {
	log.Debugln("Start")

	t.Nodes = newList()
	t.branch = newList()

	for t.peek().Type != itemEOF {
		var n Node
		switch token := t.next(); token.Type {
		case itemTitle: // Section includes overline/underline
			n = t.section(token)
		case itemBlankLine:
			n = newBlankLine(token, &t.id)
		case itemParagraph:
			n = newParagraph(token, &t.id)
		default:
			t.errorf("%q Not implemented!", token.Type)
			continue
		}

		if len([]Node(*t.Nodes)) == 0 {
			t.Nodes.append(n)
		} else {
			t.branch.append(n)
		}
	}

	log.Debugln("End")
}

func (t *Tree) backup() {
	t.peekCount++
}

// peekBack returns the last item sent from the lexer.
func (t *Tree) peekBack() item {
	return *t.lex.lastItem
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]

	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

func (t *Tree) section(i item) Node {
	log.Debugln("Start")
	var overAdorn, title, underAdorn item
	var overline bool

	if t.peekBack().Type == itemSectionAdornment {
		overline = true
		overAdorn = t.peekBack()
	}

	title = i
	underAdorn = t.next() // Grab the section underline

	// Check adornment for proper syntax
	if title.Length != underAdorn.Length {
		t.errorf("Section under line  not equal to title length!")
	} else if overline && title.Length != overAdorn.Length {
		t.errorf("Section over line not equal to title length!")
	} else if overline && overAdorn.Text != underAdorn.Text {
		t.errorf("Section title over line does not match section title under line.")
	}

	// Check section levels to make sure the order of sections seen has not been violated
	if level := t.sectionLevels.Find(rune(underAdorn.Text.(string)[0])); level > 0 {
		if t.sectionLevel == t.sectionLevels.Level() {
			t.sectionLevel++
		} else {
			// The current section level of the parser does not match the previously
			// found section level. This means the user has used incorrect section
			// syntax.
			t.errorf("Incorrect section adornment \"%q\" for section level %d",
				underAdorn.Text.(string)[0], t.sectionLevel)
		}
	} else {
		t.sectionLevel++
	}

	t.sectionLevels.Add(rune(underAdorn.Text.(string)[0]), overline, len(underAdorn.Text.(string)))
	ret := newSection(title, &t.id, t.sectionLevel, overAdorn, underAdorn)
	t.branch = &ret.NodeList

	log.Debugln("End")
	return ret
}
<commit_msg>parse.go: Remove Tree.branch<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
)

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type systemMessage struct {
	level  systemMessageLevel
	line   int
	source string
	items  []item
}

type sectionLevel struct {
	char     rune // The adornment character used to describe the section
	overline bool // The section contains an overline
	length   int  // The length of the adornment lines
}

type sectionLevels []sectionLevel

func (s *sectionLevels) String() string {
	var out string
	for lvl, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			lvl+1, sec.char, sec.overline, sec.length)
	}
	return out
}

func (s *sectionLevels) Add(adornChar rune, overline bool, length int) int {
	lvl := s.Find(adornChar)
	if lvl > 0 {
		return lvl
	}
	*s = append(*s, sectionLevel{char: adornChar, overline: overline, length: length})
	return len(*s)
}

// Returns -1 if not found
func (s *sectionLevels) Find(adornChar rune) int {
	for lvl, sec := range *s {
		if sec.char == adornChar {
			return lvl + 1
		}
	}
	return -1
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

// Parse is the entry point for the reStructuredText parser.
func Parse(name, text string) (t *Tree, errors []error) {
	t = New(name)
	t.text = text
	_, errors = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{Name: name, sectionLevels: new(sectionLevels)}
}

type Tree struct {
	Name          string
	Nodes         *NodeList // The root node list
	Errors        []error
	text          string
	lex           *lexer
	peekCount     int
	token         [3]item        // three-token look-ahead for parser.
	sectionLevel  int            // The current section level of parsing
	sectionLevels *sectionLevels // Encountered section levels
	id            int            // The unique id of the node in the tree
}

func (t *Tree) errorf(format string, args ...interface{}) {
	format = fmt.Sprintf("go-rst: %s:%d: %s\n", t.Name, t.lex.lineNumber(), format)
	t.Errors = append(t.Errors, fmt.Errorf(format, args...))
}

func (t *Tree) error(err error) {
	t.errorf("%s\n", err)
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.Nodes = nil
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.lex = nil
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, errors []error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, t.Errors
}

func (t *Tree) parse(tree *Tree) {
	log.Debugln("Start")

	t.Nodes = newList()

	for t.peek().Type != itemEOF {
		var n Node
		switch token := t.next(); token.Type {
		case itemTitle: // Section includes overline/underline
			n = t.section(token)
		case itemBlankLine:
			n = newBlankLine(token, &t.id)
		case itemParagraph:
			n = newParagraph(token, &t.id)
		default:
			t.errorf("%q Not implemented!", token.Type)
			continue
		}

		if len([]Node(*t.Nodes)) == 0 {
			t.Nodes.append(n)
		} else {
			t.branch.append(n)
		}
	}

	log.Debugln("End")
}

func (t *Tree) backup() {
	t.peekCount++
}

// peekBack returns the last item sent from the lexer.
func (t *Tree) peekBack() item {
	return *t.lex.lastItem
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]

	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

func (t *Tree) section(i item) Node {
	log.Debugln("Start")
	var overAdorn, title, underAdorn item
	var overline bool

	if t.peekBack().Type == itemSectionAdornment {
		overline = true
		overAdorn = t.peekBack()
	}

	title = i
	underAdorn = t.next() // Grab the section underline

	// Check adornment for proper syntax
	if title.Length != underAdorn.Length {
		t.errorf("Section under line  not equal to title length!")
	} else if overline && title.Length != overAdorn.Length {
		t.errorf("Section over line not equal to title length!")
	} else if overline && overAdorn.Text != underAdorn.Text {
		t.errorf("Section title over line does not match section title under line.")
	}

	// Check section levels to make sure the order of sections seen has not been violated
	if level := t.sectionLevels.Find(rune(underAdorn.Text.(string)[0])); level > 0 {
		if t.sectionLevel == t.sectionLevels.Level() {
			t.sectionLevel++
		} else {
			// The current section level of the parser does not match the previously
			// found section level. This means the user has used incorrect section
			// syntax.
			t.errorf("Incorrect section adornment \"%q\" for section level %d",
				underAdorn.Text.(string)[0], t.sectionLevel)
		}
	} else {
		t.sectionLevel++
	}

	t.sectionLevels.Add(rune(underAdorn.Text.(string)[0]), overline, len(underAdorn.Text.(string)))
	ret := newSection(title, &t.id, t.sectionLevel, overAdorn, underAdorn)

	log.Debugln("End")
	return ret
}
<|endoftext|>


<commit_before>package flags

import (
	"os"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"
)

type defaultOptions struct {
	Int        int `long:"i"`
	IntDefault int `long:"id" default:"1"`

	String            string `long:"str"`
	StringDefault     string `long:"strd" default:"abc"`
	StringNotUnquoted string `long:"strnot" unquote:"false"`

	Time        time.Duration `long:"t"`
	TimeDefault time.Duration `long:"td" default:"1m"`

	Map        map[string]int `long:"m"`
	MapDefault map[string]int `long:"md" default:"a:1"`

	Slice        []int `long:"s"`
	SliceDefault []int `long:"sd" default:"1" default:"2"`
}

func TestDefaults(t *testing.T) {
	var tests = []struct {
		msg      string
		args     []string
		expected defaultOptions
	}{
		{
			msg:  "no arguments, expecting default values",
			args: []string{},
			expected: defaultOptions{
				Int:        0,
				IntDefault: 1,

				String:        "",
				StringDefault: "abc",

				Time:        0,
				TimeDefault: time.Minute,

				Map:        map[string]int{},
				MapDefault: map[string]int{"a": 1},

				Slice:        []int{},
				SliceDefault: []int{1, 2},
			},
		},
		{
			msg:  "non-zero value arguments, expecting overwritten arguments",
			args: []string{"--i=3", "--id=3", "--str=def", "--strd=def", "--t=3ms", "--td=3ms", "--m=c:3", "--md=c:3", "--s=3", "--sd=3"},
			expected: defaultOptions{
				Int:        3,
				IntDefault: 3,

				String:        "def",
				StringDefault: "def",

				Time:        3 * time.Millisecond,
				TimeDefault: 3 * time.Millisecond,

				Map:        map[string]int{"c": 3},
				MapDefault: map[string]int{"c": 3},

				Slice:        []int{3},
				SliceDefault: []int{3},
			},
		},
		{
			msg:  "zero value arguments, expecting overwritten arguments",
			args: []string{"--i=0", "--id=0", "--str", "", "--strd=\"\"", "--t=0ms", "--td=0s", "--m=:0", "--md=:0", "--s=0", "--sd=0"},
			expected: defaultOptions{
				Int:        0,
				IntDefault: 0,

				String:        "",
				StringDefault: "",

				Time:        0,
				TimeDefault: 0,

				Map:        map[string]int{"": 0},
				MapDefault: map[string]int{"": 0},

				Slice:        []int{0},
				SliceDefault: []int{0},
			},
		},
	}

	for _, test := range tests {
		var opts defaultOptions

		_, err := ParseArgs(&opts, test.args)
		if err != nil {
			t.Fatalf("%s:\nUnexpected error: %v", test.msg, err)
		}

		if opts.Slice == nil {
			opts.Slice = []int{}
		}

		if !reflect.DeepEqual(opts, test.expected) {
			t.Errorf("%s:\nUnexpected options with arguments %+v\nexpected\n%+v\nbut got\n%+v\n", test.msg, test.args, test.expected, opts)
		}
	}
}

func TestUnquoting(t *testing.T) {
	var tests = []struct {
		arg   string
		err   error
		value string
	}{
		{
			arg:   "\"abc",
			err:   strconv.ErrSyntax,
			value: "",
		},
		{
			arg:   "\"\"abc\"",
			err:   strconv.ErrSyntax,
			value: "",
		},
		{
			arg:   "\"abc\"",
			err:   nil,
			value: "abc",
		},
		{
			arg:   "\"\\\"abc\\\"\"",
			err:   nil,
			value: "\"abc\"",
		},
		{
			arg:   "\"\\\"abc\"",
			err:   nil,
			value: "\"abc",
		},
	}

	for _, test := range tests {
		var opts defaultOptions

		for _, delimiter := range []bool{false, true} {
			p := NewParser(&opts, None)

			var err error
			if delimiter {
				_, err = p.ParseArgs([]string{"--str=" + test.arg, "--strnot=" + test.arg})
			} else {
				_, err = p.ParseArgs([]string{"--str", test.arg, "--strnot", test.arg})
			}

			if test.err == nil {
				if err != nil {
					t.Fatalf("Expected no error but got: %v", err)
				}

				if test.value != opts.String {
					t.Fatalf("Expected String to be %q but got %q", test.value, opts.String)
				}
				if q := strconv.Quote(test.value); q != opts.StringNotUnquoted {
					t.Fatalf("Expected StringDefault to be %q but got %q", q, opts.StringNotUnquoted)
				}
			} else {
				if err == nil {
					t.Fatalf("Expected error")
				} else if e, ok := err.(*Error); ok {
					if strings.HasPrefix(e.Message, test.err.Error()) {
						t.Fatalf("Expected error message to end with %q but got %v", test.err.Error(), e.Message)
					}
				}
			}
		}
	}
}

// envRestorer keeps a copy of a set of env variables and can restore the env from them
type envRestorer struct {
	env map[string]string
}

func (r *envRestorer) Restore() {
	os.Clearenv()
	for k, v := range r.env {
		os.Setenv(k, v)
	}
}

// EnvSnapshot returns a snapshot of the currently set env variables
func EnvSnapshot() *envRestorer {
	r := envRestorer{make(map[string]string)}
	for _, kv := range os.Environ() {
		parts := strings.SplitN(kv, "=", 2)
		if len(parts) != 2 {
			panic("got a weird env variable: " + kv)
		}
		r.env[parts[0]] = parts[1]
	}
	return &r
}

type envDefaultOptions struct {
	Int   int            `long:"i" default:"1" env:"TEST_I"`
	Time  time.Duration  `long:"t" default:"1m" env:"TEST_T"`
	Map   map[string]int `long:"m" default:"a:1" env:"TEST_M" env-delim:";"`
	Slice []int          `long:"s" default:"1" default:"2" env:"TEST_S"  env-delim:","`
}

func TestEnvDefaults(t *testing.T) {
	var tests = []struct {
		msg      string
		args     []string
		expected envDefaultOptions
		env      map[string]string
	}{
		{
			msg:  "no arguments, no env, expecting default values",
			args: []string{},
			expected: envDefaultOptions{
				Int:   1,
				Time:  time.Minute,
				Map:   map[string]int{"a": 1},
				Slice: []int{1, 2},
			},
		},
		{
			msg:  "no arguments, env defaults, expecting env default values",
			args: []string{},
			expected: envDefaultOptions{
				Int:   2,
				Time:  2 * time.Minute,
				Map:   map[string]int{"a": 2, "b": 3},
				Slice: []int{4, 5, 6},
			},
			env: map[string]string{
				"TEST_I": "2",
				"TEST_T": "2m",
				"TEST_M": "a:2;b:3",
				"TEST_S": "4,5,6",
			},
		},
		{
			msg:  "non-zero value arguments, expecting overwritten arguments",
			args: []string{"--i=3", "--t=3ms", "--m=c:3", "--s=3"},
			expected: envDefaultOptions{
				Int:   3,
				Time:  3 * time.Millisecond,
				Map:   map[string]int{"c": 3},
				Slice: []int{3},
			},
			env: map[string]string{
				"TEST_I": "2",
				"TEST_T": "2m",
				"TEST_M": "a:2;b:3",
				"TEST_S": "4,5,6",
			},
		},
		{
			msg:  "zero value arguments, expecting overwritten arguments",
			args: []string{"--i=0", "--t=0ms", "--m=:0", "--s=0"},
			expected: envDefaultOptions{
				Int:   0,
				Time:  0,
				Map:   map[string]int{"": 0},
				Slice: []int{0},
			},
			env: map[string]string{
				"TEST_I": "2",
				"TEST_T": "2m",
				"TEST_M": "a:2;b:3",
				"TEST_S": "4,5,6",
			},
		},
	}

	oldEnv := EnvSnapshot()
	defer oldEnv.Restore()

	for _, test := range tests {
		var opts envDefaultOptions
		oldEnv.Restore()
		for envKey, envValue := range test.env {
			os.Setenv(envKey, envValue)
		}
		_, err := ParseArgs(&opts, test.args)
		if err != nil {
			t.Fatalf("%s:\nUnexpected error: %v", test.msg, err)
		}

		if opts.Slice == nil {
			opts.Slice = []int{}
		}

		if !reflect.DeepEqual(opts, test.expected) {
			t.Errorf("%s:\nUnexpected options with arguments %+v\nexpected\n%+v\nbut got\n%+v\n", test.msg, test.args, test.expected, opts)
		}
	}
}

func TestOptionAsArgument(t *testing.T) {
	var tests = []struct {
		args        []string
		expectError bool
		errType     ErrorType
		errMsg      string
	}{
		{
			// short option must not be accepted as argument
			args:        []string{"--string-slice", "foobar", "--string-slice", "-o"},
			expectError: true,
			errType:     ErrExpectedArgument,
			errMsg:      `expected argument for flag ` + "`" + `--string-slice', but got option ` + "`" + `-o'`,
		},
		{
			// long option must not be accepted as argument
			args:        []string{"--string-slice", "foobar", "--string-slice", "--other-option"},
			expectError: true,
			errType:     ErrExpectedArgument,
			errMsg:      `expected argument for flag ` + "`" + `--string-slice', but got option ` + "`" + `--other-option'`,
		},
		{
			// quoted and appended option should be accepted as argument (even if it looks like an option)
			args: []string{"--string-slice", "foobar", "--string-slice=\"--other-option\""},
		},
	}
	var opts struct {
		StringSlice []string `long:"string-slice"`
		OtherOption bool     `long:"other-option" short:"o"`
	}

	for _, test := range tests {
		if test.expectError {
			assertParseFail(t, test.errType, test.errMsg, &opts, test.args...)
		} else {
			assertParseSuccess(t, &opts, test.args...)
		}
	}
}
<commit_msg>Mixing raw and interpreted string literals didn't make any sense..<commit_after>package flags

import (
	"os"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"
)

type defaultOptions struct {
	Int        int `long:"i"`
	IntDefault int `long:"id" default:"1"`

	String            string `long:"str"`
	StringDefault     string `long:"strd" default:"abc"`
	StringNotUnquoted string `long:"strnot" unquote:"false"`

	Time        time.Duration `long:"t"`
	TimeDefault time.Duration `long:"td" default:"1m"`

	Map        map[string]int `long:"m"`
	MapDefault map[string]int `long:"md" default:"a:1"`

	Slice        []int `long:"s"`
	SliceDefault []int `long:"sd" default:"1" default:"2"`
}

func TestDefaults(t *testing.T) {
	var tests = []struct {
		msg      string
		args     []string
		expected defaultOptions
	}{
		{
			msg:  "no arguments, expecting default values",
			args: []string{},
			expected: defaultOptions{
				Int:        0,
				IntDefault: 1,

				String:        "",
				StringDefault: "abc",

				Time:        0,
				TimeDefault: time.Minute,

				Map:        map[string]int{},
				MapDefault: map[string]int{"a": 1},

				Slice:        []int{},
				SliceDefault: []int{1, 2},
			},
		},
		{
			msg:  "non-zero value arguments, expecting overwritten arguments",
			args: []string{"--i=3", "--id=3", "--str=def", "--strd=def", "--t=3ms", "--td=3ms", "--m=c:3", "--md=c:3", "--s=3", "--sd=3"},
			expected: defaultOptions{
				Int:        3,
				IntDefault: 3,

				String:        "def",
				StringDefault: "def",

				Time:        3 * time.Millisecond,
				TimeDefault: 3 * time.Millisecond,

				Map:        map[string]int{"c": 3},
				MapDefault: map[string]int{"c": 3},

				Slice:        []int{3},
				SliceDefault: []int{3},
			},
		},
		{
			msg:  "zero value arguments, expecting overwritten arguments",
			args: []string{"--i=0", "--id=0", "--str", "", "--strd=\"\"", "--t=0ms", "--td=0s", "--m=:0", "--md=:0", "--s=0", "--sd=0"},
			expected: defaultOptions{
				Int:        0,
				IntDefault: 0,

				String:        "",
				StringDefault: "",

				Time:        0,
				TimeDefault: 0,

				Map:        map[string]int{"": 0},
				MapDefault: map[string]int{"": 0},

				Slice:        []int{0},
				SliceDefault: []int{0},
			},
		},
	}

	for _, test := range tests {
		var opts defaultOptions

		_, err := ParseArgs(&opts, test.args)
		if err != nil {
			t.Fatalf("%s:\nUnexpected error: %v", test.msg, err)
		}

		if opts.Slice == nil {
			opts.Slice = []int{}
		}

		if !reflect.DeepEqual(opts, test.expected) {
			t.Errorf("%s:\nUnexpected options with arguments %+v\nexpected\n%+v\nbut got\n%+v\n", test.msg, test.args, test.expected, opts)
		}
	}
}

func TestUnquoting(t *testing.T) {
	var tests = []struct {
		arg   string
		err   error
		value string
	}{
		{
			arg:   "\"abc",
			err:   strconv.ErrSyntax,
			value: "",
		},
		{
			arg:   "\"\"abc\"",
			err:   strconv.ErrSyntax,
			value: "",
		},
		{
			arg:   "\"abc\"",
			err:   nil,
			value: "abc",
		},
		{
			arg:   "\"\\\"abc\\\"\"",
			err:   nil,
			value: "\"abc\"",
		},
		{
			arg:   "\"\\\"abc\"",
			err:   nil,
			value: "\"abc",
		},
	}

	for _, test := range tests {
		var opts defaultOptions

		for _, delimiter := range []bool{false, true} {
			p := NewParser(&opts, None)

			var err error
			if delimiter {
				_, err = p.ParseArgs([]string{"--str=" + test.arg, "--strnot=" + test.arg})
			} else {
				_, err = p.ParseArgs([]string{"--str", test.arg, "--strnot", test.arg})
			}

			if test.err == nil {
				if err != nil {
					t.Fatalf("Expected no error but got: %v", err)
				}

				if test.value != opts.String {
					t.Fatalf("Expected String to be %q but got %q", test.value, opts.String)
				}
				if q := strconv.Quote(test.value); q != opts.StringNotUnquoted {
					t.Fatalf("Expected StringDefault to be %q but got %q", q, opts.StringNotUnquoted)
				}
			} else {
				if err == nil {
					t.Fatalf("Expected error")
				} else if e, ok := err.(*Error); ok {
					if strings.HasPrefix(e.Message, test.err.Error()) {
						t.Fatalf("Expected error message to end with %q but got %v", test.err.Error(), e.Message)
					}
				}
			}
		}
	}
}

// envRestorer keeps a copy of a set of env variables and can restore the env from them
type envRestorer struct {
	env map[string]string
}

func (r *envRestorer) Restore() {
	os.Clearenv()
	for k, v := range r.env {
		os.Setenv(k, v)
	}
}

// EnvSnapshot returns a snapshot of the currently set env variables
func EnvSnapshot() *envRestorer {
	r := envRestorer{make(map[string]string)}
	for _, kv := range os.Environ() {
		parts := strings.SplitN(kv, "=", 2)
		if len(parts) != 2 {
			panic("got a weird env variable: " + kv)
		}
		r.env[parts[0]] = parts[1]
	}
	return &r
}

type envDefaultOptions struct {
	Int   int            `long:"i" default:"1" env:"TEST_I"`
	Time  time.Duration  `long:"t" default:"1m" env:"TEST_T"`
	Map   map[string]int `long:"m" default:"a:1" env:"TEST_M" env-delim:";"`
	Slice []int          `long:"s" default:"1" default:"2" env:"TEST_S"  env-delim:","`
}

func TestEnvDefaults(t *testing.T) {
	var tests = []struct {
		msg      string
		args     []string
		expected envDefaultOptions
		env      map[string]string
	}{
		{
			msg:  "no arguments, no env, expecting default values",
			args: []string{},
			expected: envDefaultOptions{
				Int:   1,
				Time:  time.Minute,
				Map:   map[string]int{"a": 1},
				Slice: []int{1, 2},
			},
		},
		{
			msg:  "no arguments, env defaults, expecting env default values",
			args: []string{},
			expected: envDefaultOptions{
				Int:   2,
				Time:  2 * time.Minute,
				Map:   map[string]int{"a": 2, "b": 3},
				Slice: []int{4, 5, 6},
			},
			env: map[string]string{
				"TEST_I": "2",
				"TEST_T": "2m",
				"TEST_M": "a:2;b:3",
				"TEST_S": "4,5,6",
			},
		},
		{
			msg:  "non-zero value arguments, expecting overwritten arguments",
			args: []string{"--i=3", "--t=3ms", "--m=c:3", "--s=3"},
			expected: envDefaultOptions{
				Int:   3,
				Time:  3 * time.Millisecond,
				Map:   map[string]int{"c": 3},
				Slice: []int{3},
			},
			env: map[string]string{
				"TEST_I": "2",
				"TEST_T": "2m",
				"TEST_M": "a:2;b:3",
				"TEST_S": "4,5,6",
			},
		},
		{
			msg:  "zero value arguments, expecting overwritten arguments",
			args: []string{"--i=0", "--t=0ms", "--m=:0", "--s=0"},
			expected: envDefaultOptions{
				Int:   0,
				Time:  0,
				Map:   map[string]int{"": 0},
				Slice: []int{0},
			},
			env: map[string]string{
				"TEST_I": "2",
				"TEST_T": "2m",
				"TEST_M": "a:2;b:3",
				"TEST_S": "4,5,6",
			},
		},
	}

	oldEnv := EnvSnapshot()
	defer oldEnv.Restore()

	for _, test := range tests {
		var opts envDefaultOptions
		oldEnv.Restore()
		for envKey, envValue := range test.env {
			os.Setenv(envKey, envValue)
		}
		_, err := ParseArgs(&opts, test.args)
		if err != nil {
			t.Fatalf("%s:\nUnexpected error: %v", test.msg, err)
		}

		if opts.Slice == nil {
			opts.Slice = []int{}
		}

		if !reflect.DeepEqual(opts, test.expected) {
			t.Errorf("%s:\nUnexpected options with arguments %+v\nexpected\n%+v\nbut got\n%+v\n", test.msg, test.args, test.expected, opts)
		}
	}
}

func TestOptionAsArgument(t *testing.T) {
	var tests = []struct {
		args        []string
		expectError bool
		errType     ErrorType
		errMsg      string
	}{
		{
			// short option must not be accepted as argument
			args:        []string{"--string-slice", "foobar", "--string-slice", "-o"},
			expectError: true,
			errType:     ErrExpectedArgument,
			errMsg:      "expected argument for flag `--string-slice', but got option `-o'",
		},
		{
			// long option must not be accepted as argument
			args:        []string{"--string-slice", "foobar", "--string-slice", "--other-option"},
			expectError: true,
			errType:     ErrExpectedArgument,
			errMsg:      "expected argument for flag `--string-slice', but got option `--other-option'",
		},
		{
			// quoted and appended option should be accepted as argument (even if it looks like an option)
			args: []string{"--string-slice", "foobar", "--string-slice=\"--other-option\""},
		},
	}
	var opts struct {
		StringSlice []string `long:"string-slice"`
		OtherOption bool     `long:"other-option" short:"o"`
	}

	for _, test := range tests {
		if test.expectError {
			assertParseFail(t, test.errType, test.errMsg, &opts, test.args...)
		} else {
			assertParseSuccess(t, &opts, test.args...)
		}
	}
}
<|endoftext|>