<commit_before>/*
 * This file is part of the kubevirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package watch

import (
	"errors"
	"strings"
	"time"

	"github.com/jeevatkm/go-model"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"

	kubev1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/logging"
	registrydisk "kubevirt.io/kubevirt/pkg/registry-disk"
	"kubevirt.io/kubevirt/pkg/virt-controller/services"
)

func NewVMController(restClient *rest.RESTClient, vmService services.VMService, queue workqueue.RateLimitingInterface, vmCache cache.Store, vmInformer cache.SharedIndexInformer, podInformer cache.SharedIndexInformer, recorder record.EventRecorder, clientset *kubernetes.Clientset) *VMController {
	return &VMController{
		restClient:  restClient,
		vmService:   vmService,
		queue:       queue,
		store:       vmCache,
		vmInformer:  vmInformer,
		podInformer: podInformer,
		recorder:    recorder,
		clientset:   clientset,
	}
}

type VMController struct {
	restClient  *rest.RESTClient
	vmService   services.VMService
	clientset   *kubernetes.Clientset
	queue       workqueue.RateLimitingInterface
	store       cache.Store
	vmInformer  cache.SharedIndexInformer
	podInformer cache.SharedIndexInformer
	recorder    record.EventRecorder
}

func (c *VMController) Run(threadiness int, stopCh chan struct{}) {
	defer kubecli.HandlePanic()
	defer c.queue.ShutDown()
	logging.DefaultLogger().Info().Msg("Starting controller.")

	// Wait for cache sync before we start the pod controller
	cache.WaitForCacheSync(stopCh, c.vmInformer.HasSynced, c.podInformer.HasSynced)

	// Start the actual work
	for i := 0; i < threadiness; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	<-stopCh
	logging.DefaultLogger().Info().Msg("Stopping controller.")
}

func (c *VMController) runWorker() {
	for c.Execute() {
	}
}

func (c *VMController) Execute() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)
	if err := c.execute(key.(string)); err != nil {
		logging.DefaultLogger().Info().Reason(err).Msgf("reenqueuing VM %v", key)
		c.queue.AddRateLimited(key)
	} else {
		logging.DefaultLogger().Info().V(4).Msgf("processed VM %v", key)
		c.queue.Forget(key)
	}
	return true
}

func (c *VMController) execute(key string) error {

	// Fetch the latest Vm state from cache
	obj, exists, err := c.store.GetByKey(key)

	if err != nil {
		return err
	}

	// Retrieve the VM
	var vm *kubev1.VM
	if !exists {
		namespace, name, err := cache.SplitMetaNamespaceKey(key)
		if err != nil {
			return err
		}
		vm = kubev1.NewVMReferenceFromNameWithNS(namespace, name)
	} else {
		vm = obj.(*kubev1.VM)
	}
	logger := logging.DefaultLogger().Object(vm)

	if !exists {
		// Delete VM Pods
		err := c.vmService.DeleteVMPod(vm)
		if err != nil {
			logger.Error().Reason(err).Msg("Deleting VM target Pod failed.")
			return err
		}
		logger.Info().Msg("Deleting VM target Pod succeeded.")
		return nil
	}

	switch vm.Status.Phase {
	case kubev1.VmPhaseUnset, kubev1.Pending:
		// Schedule the VM

		// Deep copy the object, so that we can safely manipulate it
		vmCopy := kubev1.VM{}
		model.Copy(&vmCopy, vm)
		logger = logging.DefaultLogger().Object(&vmCopy)

		// Check if there are already outdated VM Pods
		pods, err := c.vmService.GetRunningVMPods(&vmCopy)
		if err != nil {
			logger.Error().Reason(err).Msg("Fetching VM pods failed.")
			return err
		}

		// If there are already pods, delete them before continuing ...
		if len(pods.Items) > 0 {
			logger.Error().Msg("VM Pods do already exist, will clean up before continuing.")
			if err := c.vmService.DeleteVMPod(&vmCopy); err != nil {
				logger.Error().Reason(err).Msg("Deleting VM pods failed.")
				return err
			}
		}

		// Defaulting and setting constants
		// TODO move defaulting to virt-api
		// TODO move constants to virt-handler and remove from the spec
		if vmCopy.Spec.Domain == nil {
			spec := kubev1.NewMinimalDomainSpec()
			vmCopy.Spec.Domain = spec
		}

		// TODO when we move this to virt-api, we have to block that they are set on POST or changed on PUT
		graphics := vmCopy.Spec.Domain.Devices.Graphics
		for i, _ := range graphics {
			if strings.ToLower(graphics[i].Type) == "spice" {
				graphics[i].Port = int32(4000) + int32(i)
				graphics[i].Listen = kubev1.Listen{
					Address: "0.0.0.0",
					Type:    "address",
				}

			}
		}

		registrydisk.ApplyPorts(&vmCopy)

		// Create a Pod which will be the VM destination
		if err := c.vmService.StartVMPod(&vmCopy); err != nil {
			logger.Error().Reason(err).Msg("Defining a target pod for the VM failed.")
			return err
		}

		// Mark the VM as "initialized". After the created Pod above is scheduled by
		// kubernetes, virt-handler can take over.
		vmCopy.Status.Phase = kubev1.Scheduling
		if err := c.restClient.Put().Resource("vms").Body(&vmCopy).Name(vmCopy.ObjectMeta.Name).Namespace(vmCopy.ObjectMeta.Namespace).Do().Error(); err != nil {
			logger.Error().Reason(err).Msg("Updating the VM state to 'Scheduling' failed.")
			return err
		}
		logger.Info().Msg("Handing over the VM to the scheduler succeeded.")
	case kubev1.Scheduling:
		// Target Pod for the VM was already created, check if it is  running and update the VM to Scheduled

		// Deep copy the object, so that we can safely manipulate it
		vmCopy := kubev1.VM{}
		model.Copy(&vmCopy, vm)
		logger = logging.DefaultLogger().Object(&vmCopy)

		pods, err := c.vmService.GetRunningVMPods(&vmCopy)
		if err != nil {
			logger.Error().Reason(err).Msg("Fetching VM pods failed.")
			return err
		}

		//TODO, we can improve the pod checks here, for now they are as good as before the refactoring
		// So far, no running Pod found, we will sooner or later get a started event.
		// If not, something is wrong and the VM, stay stuck in the Scheduling phase
		if len(pods.Items) == 0 {
			logger.Info().V(3).Msg("No VM target pod in running state found.")
			return nil
		}

		// Whatever is going on here, I don't know what to do, don't reprocess this
		if len(pods.Items) > 1 {
			logger.Error().V(3).Msg("More than one VM target pods found.")
			return nil
		}

		// Pod is not yet running
		if pods.Items[0].Status.Phase != k8sv1.PodRunning {
			return nil
		}

		// Ensure registry disks are online before placing VM
		if registrydisk.DisksAreReady(&pods.Items[0]) == false {
			return errors.New("waiting on image wrapper disks to become ready")
		}

		// VM got scheduled
		vmCopy.Status.Phase = kubev1.Scheduled

		// Fill in host info for container registry disks
		registrydisk.ApplyHost(&vmCopy, &pods.Items[0])

		// FIXME we store this in the metadata since field selectors are currently not working for TPRs
		if vmCopy.GetObjectMeta().GetLabels() == nil {
			vmCopy.ObjectMeta.Labels = map[string]string{}
		}
		vmCopy.ObjectMeta.Labels[kubev1.NodeNameLabel] = pods.Items[0].Spec.NodeName
		vmCopy.Status.NodeName = pods.Items[0].Spec.NodeName
		if _, err := c.vmService.PutVm(&vmCopy); err != nil {
			logger.Error().Reason(err).Msg("Updating the VM state to 'Scheduled' failed.")
			return err
		}
		logger.Info().Msgf("VM successfully scheduled to %s.", vmCopy.Status.NodeName)
	}
	return nil
}

func vmLabelHandler(vmQueue workqueue.RateLimitingInterface) func(obj interface{}) {
	return func(obj interface{}) {
		phase := obj.(*k8sv1.Pod).Status.Phase
		namespace := obj.(*k8sv1.Pod).ObjectMeta.Namespace
		appLabel, hasAppLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.AppLabel]
		domainLabel, hasDomainLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.DomainLabel]
		_, hasMigrationLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.MigrationLabel]

		if phase != k8sv1.PodRunning {
			// Filter out non running pods from queue
			return
		} else if hasMigrationLabel {
			// Filter out migration target pods
			return
		} else if hasDomainLabel == false || hasAppLabel == false {
			// missing required labels
			return
		} else if appLabel != "virt-launcher" {
			// ensure we're looking just for virt-launcher pods
			return
		}
		vmQueue.Add(namespace + "/" + domainLabel)
	}
}
<commit_msg>Do not return error while waiting for registry disk readiness<commit_after>/*
 * This file is part of the kubevirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package watch

import (
	"strings"
	"time"

	"github.com/jeevatkm/go-model"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"

	kubev1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/logging"
	registrydisk "kubevirt.io/kubevirt/pkg/registry-disk"
	"kubevirt.io/kubevirt/pkg/virt-controller/services"
)

func NewVMController(restClient *rest.RESTClient, vmService services.VMService, queue workqueue.RateLimitingInterface, vmCache cache.Store, vmInformer cache.SharedIndexInformer, podInformer cache.SharedIndexInformer, recorder record.EventRecorder, clientset *kubernetes.Clientset) *VMController {
	return &VMController{
		restClient:  restClient,
		vmService:   vmService,
		queue:       queue,
		store:       vmCache,
		vmInformer:  vmInformer,
		podInformer: podInformer,
		recorder:    recorder,
		clientset:   clientset,
	}
}

type VMController struct {
	restClient  *rest.RESTClient
	vmService   services.VMService
	clientset   *kubernetes.Clientset
	queue       workqueue.RateLimitingInterface
	store       cache.Store
	vmInformer  cache.SharedIndexInformer
	podInformer cache.SharedIndexInformer
	recorder    record.EventRecorder
}

func (c *VMController) Run(threadiness int, stopCh chan struct{}) {
	defer kubecli.HandlePanic()
	defer c.queue.ShutDown()
	logging.DefaultLogger().Info().Msg("Starting controller.")

	// Wait for cache sync before we start the pod controller
	cache.WaitForCacheSync(stopCh, c.vmInformer.HasSynced, c.podInformer.HasSynced)

	// Start the actual work
	for i := 0; i < threadiness; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	<-stopCh
	logging.DefaultLogger().Info().Msg("Stopping controller.")
}

func (c *VMController) runWorker() {
	for c.Execute() {
	}
}

func (c *VMController) Execute() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)
	if err := c.execute(key.(string)); err != nil {
		logging.DefaultLogger().Info().Reason(err).Msgf("reenqueuing VM %v", key)
		c.queue.AddRateLimited(key)
	} else {
		logging.DefaultLogger().Info().V(4).Msgf("processed VM %v", key)
		c.queue.Forget(key)
	}
	return true
}

func (c *VMController) execute(key string) error {

	// Fetch the latest Vm state from cache
	obj, exists, err := c.store.GetByKey(key)

	if err != nil {
		return err
	}

	// Retrieve the VM
	var vm *kubev1.VM
	if !exists {
		namespace, name, err := cache.SplitMetaNamespaceKey(key)
		if err != nil {
			return err
		}
		vm = kubev1.NewVMReferenceFromNameWithNS(namespace, name)
	} else {
		vm = obj.(*kubev1.VM)
	}
	logger := logging.DefaultLogger().Object(vm)

	if !exists {
		// Delete VM Pods
		err := c.vmService.DeleteVMPod(vm)
		if err != nil {
			logger.Error().Reason(err).Msg("Deleting VM target Pod failed.")
			return err
		}
		logger.Info().Msg("Deleting VM target Pod succeeded.")
		return nil
	}

	switch vm.Status.Phase {
	case kubev1.VmPhaseUnset, kubev1.Pending:
		// Schedule the VM

		// Deep copy the object, so that we can safely manipulate it
		vmCopy := kubev1.VM{}
		model.Copy(&vmCopy, vm)
		logger = logging.DefaultLogger().Object(&vmCopy)

		// Check if there are already outdated VM Pods
		pods, err := c.vmService.GetRunningVMPods(&vmCopy)
		if err != nil {
			logger.Error().Reason(err).Msg("Fetching VM pods failed.")
			return err
		}

		// If there are already pods, delete them before continuing ...
		if len(pods.Items) > 0 {
			logger.Error().Msg("VM Pods do already exist, will clean up before continuing.")
			if err := c.vmService.DeleteVMPod(&vmCopy); err != nil {
				logger.Error().Reason(err).Msg("Deleting VM pods failed.")
				return err
			}
		}

		// Defaulting and setting constants
		// TODO move defaulting to virt-api
		// TODO move constants to virt-handler and remove from the spec
		if vmCopy.Spec.Domain == nil {
			spec := kubev1.NewMinimalDomainSpec()
			vmCopy.Spec.Domain = spec
		}

		// TODO when we move this to virt-api, we have to block that they are set on POST or changed on PUT
		graphics := vmCopy.Spec.Domain.Devices.Graphics
		for i, _ := range graphics {
			if strings.ToLower(graphics[i].Type) == "spice" {
				graphics[i].Port = int32(4000) + int32(i)
				graphics[i].Listen = kubev1.Listen{
					Address: "0.0.0.0",
					Type:    "address",
				}

			}
		}

		registrydisk.ApplyPorts(&vmCopy)

		// Create a Pod which will be the VM destination
		if err := c.vmService.StartVMPod(&vmCopy); err != nil {
			logger.Error().Reason(err).Msg("Defining a target pod for the VM failed.")
			return err
		}

		// Mark the VM as "initialized". After the created Pod above is scheduled by
		// kubernetes, virt-handler can take over.
		vmCopy.Status.Phase = kubev1.Scheduling
		if err := c.restClient.Put().Resource("vms").Body(&vmCopy).Name(vmCopy.ObjectMeta.Name).Namespace(vmCopy.ObjectMeta.Namespace).Do().Error(); err != nil {
			logger.Error().Reason(err).Msg("Updating the VM state to 'Scheduling' failed.")
			return err
		}
		logger.Info().Msg("Handing over the VM to the scheduler succeeded.")
	case kubev1.Scheduling:
		// Target Pod for the VM was already created, check if it is  running and update the VM to Scheduled

		// Deep copy the object, so that we can safely manipulate it
		vmCopy := kubev1.VM{}
		model.Copy(&vmCopy, vm)
		logger = logging.DefaultLogger().Object(&vmCopy)

		pods, err := c.vmService.GetRunningVMPods(&vmCopy)
		if err != nil {
			logger.Error().Reason(err).Msg("Fetching VM pods failed.")
			return err
		}

		//TODO, we can improve the pod checks here, for now they are as good as before the refactoring
		// So far, no running Pod found, we will sooner or later get a started event.
		// If not, something is wrong and the VM, stay stuck in the Scheduling phase
		if len(pods.Items) == 0 {
			logger.Info().V(3).Msg("No VM target pod in running state found.")
			return nil
		}

		// Whatever is going on here, I don't know what to do, don't reprocess this
		if len(pods.Items) > 1 {
			logger.Error().V(3).Msg("More than one VM target pods found.")
			return nil
		}

		// Pod is not yet running
		if pods.Items[0].Status.Phase != k8sv1.PodRunning {
			return nil
		}

		// Ensure registry disks are online before placing VM
		if registrydisk.DisksAreReady(&pods.Items[0]) == false {
			logger.Info().V(2).Msg("Waiting on image wrapper disks to become ready.")
			return nil
		}

		// VM got scheduled
		vmCopy.Status.Phase = kubev1.Scheduled

		// Fill in host info for container registry disks
		registrydisk.ApplyHost(&vmCopy, &pods.Items[0])

		// FIXME we store this in the metadata since field selectors are currently not working for TPRs
		if vmCopy.GetObjectMeta().GetLabels() == nil {
			vmCopy.ObjectMeta.Labels = map[string]string{}
		}
		vmCopy.ObjectMeta.Labels[kubev1.NodeNameLabel] = pods.Items[0].Spec.NodeName
		vmCopy.Status.NodeName = pods.Items[0].Spec.NodeName
		if _, err := c.vmService.PutVm(&vmCopy); err != nil {
			logger.Error().Reason(err).Msg("Updating the VM state to 'Scheduled' failed.")
			return err
		}
		logger.Info().Msgf("VM successfully scheduled to %s.", vmCopy.Status.NodeName)
	}
	return nil
}

func vmLabelHandler(vmQueue workqueue.RateLimitingInterface) func(obj interface{}) {
	return func(obj interface{}) {
		phase := obj.(*k8sv1.Pod).Status.Phase
		namespace := obj.(*k8sv1.Pod).ObjectMeta.Namespace
		appLabel, hasAppLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.AppLabel]
		domainLabel, hasDomainLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.DomainLabel]
		_, hasMigrationLabel := obj.(*k8sv1.Pod).ObjectMeta.Labels[kubev1.MigrationLabel]

		if phase != k8sv1.PodRunning {
			// Filter out non running pods from queue
			return
		} else if hasMigrationLabel {
			// Filter out migration target pods
			return
		} else if hasDomainLabel == false || hasAppLabel == false {
			// missing required labels
			return
		} else if appLabel != "virt-launcher" {
			// ensure we're looking just for virt-launcher pods
			return
		}
		vmQueue.Add(namespace + "/" + domainLabel)
	}
}
<|endoftext|>


<commit_before>package dns

import (
	"testing"

	r "github.com/hashicorp/terraform/helper/resource"
)

func TestAccDnsCnameRecordSet_Basic(t *testing.T) {
	tests := []struct {
		DataSourceBlock string
		Expected        string
	}{
		{
			`
			data "dns_cname_record_set" "foo" {
			  host = "www.hashicorp.com"
			}
			`,
			"s.shared.global.fastly.net.",
		},
	}

	for _, test := range tests {
		r.UnitTest(t, r.TestCase{
			Providers: testAccProviders,
			Steps: []r.TestStep{
				r.TestStep{
					Config: test.DataSourceBlock,
					Check: r.ComposeTestCheckFunc(
						r.TestCheckResourceAttr("data.dns_cname_record_set.foo", "cname", test.Expected),
					),
				},
			},
		})
	}
}
<commit_msg>provider/dns: update hard-coded DNS result in acctests<commit_after>package dns

import (
	"testing"

	r "github.com/hashicorp/terraform/helper/resource"
)

func TestAccDnsCnameRecordSet_Basic(t *testing.T) {
	tests := []struct {
		DataSourceBlock string
		Expected        string
	}{
		{
			`
			data "dns_cname_record_set" "foo" {
			  host = "www.hashicorp.com"
			}
			`,
			"dualstack.s.shared.global.fastly.net.",
		},
	}

	for _, test := range tests {
		r.UnitTest(t, r.TestCase{
			Providers: testAccProviders,
			Steps: []r.TestStep{
				r.TestStep{
					Config: test.DataSourceBlock,
					Check: r.ComposeTestCheckFunc(
						r.TestCheckResourceAttr("data.dns_cname_record_set.foo", "cname", test.Expected),
					),
				},
			},
		})
	}
}
<|endoftext|>


<commit_before>package core
<commit_msg>TBR: veyron/lib/modules/core: remove empty file.<commit_after><|endoftext|>


<commit_before>package rancher

import (
	"fmt"
	"math"
	"strconv"
	"strings"
	"text/template"

	"github.com/BurntSushi/ty/fun"
	"github.com/containous/traefik/log"
	"github.com/containous/traefik/provider"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/types"
)

var _ provider.Provider = (*Provider)(nil)

// Provider holds configurations of the provider.
type Provider struct {
	provider.BaseProvider     `mapstructure:",squash" export:"true"`
	APIConfiguration          `mapstructure:",squash" export:"true"` // Provide backwards compatibility
	API                       *APIConfiguration                      `description:"Enable the Rancher API provider" export:"true"`
	Metadata                  *MetadataConfiguration                 `description:"Enable the Rancher metadata service provider" export:"true"`
	Domain                    string                                 `description:"Default domain used"`
	RefreshSeconds            int                                    `description:"Polling interval (in seconds)" export:"true"`
	ExposedByDefault          bool                                   `description:"Expose services by default" export:"true"`
	EnableServiceHealthFilter bool                                   `description:"Filter services with unhealthy states and inactive states" export:"true"`
}

type rancherData struct {
	Name       string
	Labels     map[string]string // List of labels set to container or service
	Containers []string
	Health     string
	State      string
}

func (r rancherData) String() string {
	return fmt.Sprintf("{name:%s, labels:%v, containers: %v, health: %s, state: %s}", r.Name, r.Labels, r.Containers, r.Health, r.State)
}

// Frontend Labels
func (p *Provider) getPassHostHeader(service rancherData) string {
	if passHostHeader, err := getServiceLabel(service, types.LabelFrontendPassHostHeader); err == nil {
		return passHostHeader
	}
	return "true"
}

func (p *Provider) getPriority(service rancherData) string {
	if priority, err := getServiceLabel(service, types.LabelFrontendPriority); err == nil {
		return priority
	}
	return "0"
}

func (p *Provider) getEntryPoints(service rancherData) []string {
	if entryPoints, err := getServiceLabel(service, types.LabelFrontendEntryPoints); err == nil {
		return strings.Split(entryPoints, ",")
	}
	return []string{}
}

func (p *Provider) getFrontendRule(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelFrontendRule); err == nil {
		return label
	}
	return "Host:" + strings.ToLower(strings.Replace(service.Name, "/", ".", -1)) + "." + p.Domain
}

func (p *Provider) getBasicAuth(service rancherData) []string {
	if basicAuth, err := getServiceLabel(service, types.LabelFrontendAuthBasic); err == nil {
		return strings.Split(basicAuth, ",")
	}
	return []string{}
}

func (p *Provider) getFrontendName(service rancherData) string {
	// Replace '.' with '-' in quoted keys because of this issue https://github.com/BurntSushi/toml/issues/78
	return provider.Normalize(p.getFrontendRule(service))
}

// Backend Labels
func (p *Provider) getLoadBalancerMethod(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelBackendLoadbalancerMethod); err == nil {
		return label
	}
	return "wrr"
}

func (p *Provider) hasLoadBalancerLabel(service rancherData) bool {
	_, errMethod := getServiceLabel(service, types.LabelBackendLoadbalancerMethod)
	_, errSticky := getServiceLabel(service, types.LabelBackendLoadbalancerSticky)
	_, errStickiness := getServiceLabel(service, types.LabelBackendLoadbalancerStickiness)
	_, errCookieName := getServiceLabel(service, types.LabelBackendLoadbalancerStickinessCookieName)

	return errMethod == nil || errSticky == nil || errStickiness == nil || errCookieName == nil
}

func (p *Provider) hasCircuitBreakerLabel(service rancherData) bool {
	if _, err := getServiceLabel(service, types.LabelBackendCircuitbreakerExpression); err != nil {
		return false
	}
	return true
}

func (p *Provider) getCircuitBreakerExpression(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelBackendCircuitbreakerExpression); err == nil {
		return label
	}
	return "NetworkErrorRatio() > 1"
}

func (p *Provider) getSticky(service rancherData) string {
	if _, err := getServiceLabel(service, types.LabelBackendLoadbalancerSticky); err == nil {
		log.Warnf("Deprecated configuration found: %s. Please use %s.", types.LabelBackendLoadbalancerSticky, types.LabelBackendLoadbalancerStickiness)
		return "true"
	}
	return "false"
}

func (p *Provider) hasStickinessLabel(service rancherData) bool {
	labelStickiness, errStickiness := getServiceLabel(service, types.LabelBackendLoadbalancerStickiness)

	return errStickiness == nil && len(labelStickiness) > 0 && strings.EqualFold(strings.TrimSpace(labelStickiness), "true")
}

func (p *Provider) getStickinessCookieName(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelBackendLoadbalancerStickinessCookieName); err == nil {
		return label
	}
	return ""
}

func (p *Provider) getBackend(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelBackend); err == nil {
		return provider.Normalize(label)
	}
	return provider.Normalize(service.Name)
}

// General Application Stuff
func (p *Provider) getPort(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelPort); err == nil {
		return label
	}
	return ""
}

func (p *Provider) getProtocol(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelProtocol); err == nil {
		return label
	}
	return "http"
}

func (p *Provider) getWeight(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelWeight); err == nil {
		return label
	}
	return "0"
}

func (p *Provider) getDomain(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelDomain); err == nil {
		return label
	}
	return p.Domain
}

func (p *Provider) hasMaxConnLabels(service rancherData) bool {
	if _, err := getServiceLabel(service, types.LabelBackendMaxconnAmount); err != nil {
		return false
	}
	if _, err := getServiceLabel(service, types.LabelBackendMaxconnExtractorfunc); err != nil {
		return false
	}
	return true
}

func (p *Provider) getMaxConnAmount(service rancherData) int64 {
	if label, err := getServiceLabel(service, types.LabelBackendMaxconnAmount); err == nil {
		i, errConv := strconv.ParseInt(label, 10, 64)
		if errConv != nil {
			log.Errorf("Unable to parse %s %s", types.LabelBackendMaxconnAmount, label)
			return math.MaxInt64
		}
		return i
	}
	return math.MaxInt64
}

func (p *Provider) getMaxConnExtractorFunc(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelBackendMaxconnExtractorfunc); err == nil {
		return label
	}
	return "request.host"
}

func getServiceLabel(service rancherData, label string) (string, error) {
	for key, value := range service.Labels {
		if key == label {
			return value, nil
		}
	}
	return "", fmt.Errorf("label not found: %s", label)
}

// Provide allows either the Rancher API or metadata service provider to
// seed configuration into Traefik using the given configuration channel.
func (p *Provider) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
	if p.Metadata == nil {
		return p.apiProvide(configurationChan, pool, constraints)
	}
	return p.metadataProvide(configurationChan, pool, constraints)
}

func (p *Provider) loadRancherConfig(services []rancherData) *types.Configuration {

	var RancherFuncMap = template.FuncMap{
		"getPort":                     p.getPort,
		"getBackend":                  p.getBackend,
		"getWeight":                   p.getWeight,
		"getDomain":                   p.getDomain,
		"getProtocol":                 p.getProtocol,
		"getPassHostHeader":           p.getPassHostHeader,
		"getPriority":                 p.getPriority,
		"getEntryPoints":              p.getEntryPoints,
		"getBasicAuth":                p.getBasicAuth,
		"getFrontendRule":             p.getFrontendRule,
		"hasCircuitBreakerLabel":      p.hasCircuitBreakerLabel,
		"getCircuitBreakerExpression": p.getCircuitBreakerExpression,
		"hasLoadBalancerLabel":        p.hasLoadBalancerLabel,
		"getLoadBalancerMethod":       p.getLoadBalancerMethod,
		"hasMaxConnLabels":            p.hasMaxConnLabels,
		"getMaxConnAmount":            p.getMaxConnAmount,
		"getMaxConnExtractorFunc":     p.getMaxConnExtractorFunc,
		"getSticky":                   p.getSticky,
		"hasStickinessLabel":          p.hasStickinessLabel,
		"getStickinessCookieName":     p.getStickinessCookieName,
		"hasRedirect":                 hasRedirect,
		"getRedirectEntryPoint":       getRedirectEntryPoint,
		"getRedirectRegex":            getRedirectRegex,
		"getRedirectReplacement":      getRedirectReplacement,
	}

	// filter services
	filteredServices := fun.Filter(p.serviceFilter, services).([]rancherData)

	frontends := map[string]rancherData{}
	backends := map[string]rancherData{}

	for _, service := range filteredServices {
		frontendName := p.getFrontendName(service)
		frontends[frontendName] = service
		backendName := p.getBackend(service)
		backends[backendName] = service
	}

	templateObjects := struct {
		Frontends map[string]rancherData
		Backends  map[string]rancherData
		Domain    string
	}{
		frontends,
		backends,
		p.Domain,
	}

	configuration, err := p.GetConfiguration("templates/rancher.tmpl", RancherFuncMap, templateObjects)
	if err != nil {
		log.Error(err)
	}

	return configuration

}

func containerFilter(name, healthState, state string) bool {
	if healthState != "" && healthState != "healthy" && healthState != "updating-healthy" {
		log.Debugf("Filtering container %s with healthState of %s", name, healthState)
		return false
	}

	if state != "" && state != "running" && state != "updating-running" {
		log.Debugf("Filtering container %s with state of %s", name, state)
		return false
	}

	return true
}

func (p *Provider) serviceFilter(service rancherData) bool {

	if service.Labels[types.LabelPort] == "" {
		log.Debugf("Filtering service %s without traefik.port label", service.Name)
		return false
	}

	if !isServiceEnabled(service, p.ExposedByDefault) {
		log.Debugf("Filtering disabled service %s", service.Name)
		return false
	}

	constraintTags := strings.Split(service.Labels[types.LabelTags], ",")
	if ok, failingConstraint := p.MatchConstraints(constraintTags); !ok {
		if failingConstraint != nil {
			log.Debugf("Filtering service %s with constraint %s", service.Name, failingConstraint.String())
		}
		return false
	}

	// Only filter services by Health (HealthState) and State if EnableServiceHealthFilter is true
	if p.EnableServiceHealthFilter {

		if service.Health != "" && service.Health != "healthy" && service.Health != "updating-healthy" {
			log.Debugf("Filtering service %s with healthState of %s", service.Name, service.Health)
			return false
		}

		if service.State != "" && service.State != "active" && service.State != "updating-active" && service.State != "upgraded" {
			log.Debugf("Filtering service %s with state of %s", service.Name, service.State)
			return false
		}
	}

	return true
}

func isServiceEnabled(service rancherData, exposedByDefault bool) bool {

	if service.Labels[types.LabelEnable] != "" {
		var v = service.Labels[types.LabelEnable]
		return exposedByDefault && v != "false" || v == "true"
	}
	return exposedByDefault
}

// TODO will be rewrite when merge on master
func hasRedirect(service rancherData) bool {
	value, err := getServiceLabel(service, types.LabelFrontendRedirectEntryPoint)
	frep := err == nil && len(value) > 0
	value, err = getServiceLabel(service, types.LabelFrontendRedirectRegex)
	frrg := err == nil && len(value) > 0
	value, err = getServiceLabel(service, types.LabelFrontendRedirectReplacement)
	frrp := err == nil && len(value) > 0

	return frep || frrg && frrp
}

// TODO will be rewrite when merge on master
func getRedirectEntryPoint(service rancherData) string {
	value, err := getServiceLabel(service, types.LabelFrontendRedirectEntryPoint)
	if err != nil || len(value) == 0 {
		return ""
	}
	return value
}

// TODO will be rewrite when merge on master
func getRedirectRegex(service rancherData) string {
	value, err := getServiceLabel(service, types.LabelFrontendRedirectRegex)
	if err != nil || len(value) == 0 {
		return ""
	}
	return value
}

// TODO will be rewrite when merge on master
func getRedirectReplacement(service rancherData) string {
	value, err := getServiceLabel(service, types.LabelFrontendRedirectReplacement)
	if err != nil || len(value) == 0 {
		return ""
	}
	return value
}
<commit_msg>Fix Rancher Healthcheck when upgrading a service<commit_after>package rancher

import (
	"fmt"
	"math"
	"strconv"
	"strings"
	"text/template"

	"github.com/BurntSushi/ty/fun"
	"github.com/containous/traefik/log"
	"github.com/containous/traefik/provider"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/types"
)

var _ provider.Provider = (*Provider)(nil)

// Provider holds configurations of the provider.
type Provider struct {
	provider.BaseProvider     `mapstructure:",squash" export:"true"`
	APIConfiguration          `mapstructure:",squash" export:"true"` // Provide backwards compatibility
	API                       *APIConfiguration                      `description:"Enable the Rancher API provider" export:"true"`
	Metadata                  *MetadataConfiguration                 `description:"Enable the Rancher metadata service provider" export:"true"`
	Domain                    string                                 `description:"Default domain used"`
	RefreshSeconds            int                                    `description:"Polling interval (in seconds)" export:"true"`
	ExposedByDefault          bool                                   `description:"Expose services by default" export:"true"`
	EnableServiceHealthFilter bool                                   `description:"Filter services with unhealthy states and inactive states" export:"true"`
}

type rancherData struct {
	Name       string
	Labels     map[string]string // List of labels set to container or service
	Containers []string
	Health     string
	State      string
}

func (r rancherData) String() string {
	return fmt.Sprintf("{name:%s, labels:%v, containers: %v, health: %s, state: %s}", r.Name, r.Labels, r.Containers, r.Health, r.State)
}

// Frontend Labels
func (p *Provider) getPassHostHeader(service rancherData) string {
	if passHostHeader, err := getServiceLabel(service, types.LabelFrontendPassHostHeader); err == nil {
		return passHostHeader
	}
	return "true"
}

func (p *Provider) getPriority(service rancherData) string {
	if priority, err := getServiceLabel(service, types.LabelFrontendPriority); err == nil {
		return priority
	}
	return "0"
}

func (p *Provider) getEntryPoints(service rancherData) []string {
	if entryPoints, err := getServiceLabel(service, types.LabelFrontendEntryPoints); err == nil {
		return strings.Split(entryPoints, ",")
	}
	return []string{}
}

func (p *Provider) getFrontendRule(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelFrontendRule); err == nil {
		return label
	}
	return "Host:" + strings.ToLower(strings.Replace(service.Name, "/", ".", -1)) + "." + p.Domain
}

func (p *Provider) getBasicAuth(service rancherData) []string {
	if basicAuth, err := getServiceLabel(service, types.LabelFrontendAuthBasic); err == nil {
		return strings.Split(basicAuth, ",")
	}
	return []string{}
}

func (p *Provider) getFrontendName(service rancherData) string {
	// Replace '.' with '-' in quoted keys because of this issue https://github.com/BurntSushi/toml/issues/78
	return provider.Normalize(p.getFrontendRule(service))
}

// Backend Labels
func (p *Provider) getLoadBalancerMethod(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelBackendLoadbalancerMethod); err == nil {
		return label
	}
	return "wrr"
}

func (p *Provider) hasLoadBalancerLabel(service rancherData) bool {
	_, errMethod := getServiceLabel(service, types.LabelBackendLoadbalancerMethod)
	_, errSticky := getServiceLabel(service, types.LabelBackendLoadbalancerSticky)
	_, errStickiness := getServiceLabel(service, types.LabelBackendLoadbalancerStickiness)
	_, errCookieName := getServiceLabel(service, types.LabelBackendLoadbalancerStickinessCookieName)

	return errMethod == nil || errSticky == nil || errStickiness == nil || errCookieName == nil
}

func (p *Provider) hasCircuitBreakerLabel(service rancherData) bool {
	if _, err := getServiceLabel(service, types.LabelBackendCircuitbreakerExpression); err != nil {
		return false
	}
	return true
}

func (p *Provider) getCircuitBreakerExpression(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelBackendCircuitbreakerExpression); err == nil {
		return label
	}
	return "NetworkErrorRatio() > 1"
}

func (p *Provider) getSticky(service rancherData) string {
	if _, err := getServiceLabel(service, types.LabelBackendLoadbalancerSticky); err == nil {
		log.Warnf("Deprecated configuration found: %s. Please use %s.", types.LabelBackendLoadbalancerSticky, types.LabelBackendLoadbalancerStickiness)
		return "true"
	}
	return "false"
}

func (p *Provider) hasStickinessLabel(service rancherData) bool {
	labelStickiness, errStickiness := getServiceLabel(service, types.LabelBackendLoadbalancerStickiness)

	return errStickiness == nil && len(labelStickiness) > 0 && strings.EqualFold(strings.TrimSpace(labelStickiness), "true")
}

func (p *Provider) getStickinessCookieName(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelBackendLoadbalancerStickinessCookieName); err == nil {
		return label
	}
	return ""
}

func (p *Provider) getBackend(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelBackend); err == nil {
		return provider.Normalize(label)
	}
	return provider.Normalize(service.Name)
}

// General Application Stuff
func (p *Provider) getPort(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelPort); err == nil {
		return label
	}
	return ""
}

func (p *Provider) getProtocol(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelProtocol); err == nil {
		return label
	}
	return "http"
}

func (p *Provider) getWeight(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelWeight); err == nil {
		return label
	}
	return "0"
}

func (p *Provider) getDomain(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelDomain); err == nil {
		return label
	}
	return p.Domain
}

func (p *Provider) hasMaxConnLabels(service rancherData) bool {
	if _, err := getServiceLabel(service, types.LabelBackendMaxconnAmount); err != nil {
		return false
	}
	if _, err := getServiceLabel(service, types.LabelBackendMaxconnExtractorfunc); err != nil {
		return false
	}
	return true
}

func (p *Provider) getMaxConnAmount(service rancherData) int64 {
	if label, err := getServiceLabel(service, types.LabelBackendMaxconnAmount); err == nil {
		i, errConv := strconv.ParseInt(label, 10, 64)
		if errConv != nil {
			log.Errorf("Unable to parse %s %s", types.LabelBackendMaxconnAmount, label)
			return math.MaxInt64
		}
		return i
	}
	return math.MaxInt64
}

func (p *Provider) getMaxConnExtractorFunc(service rancherData) string {
	if label, err := getServiceLabel(service, types.LabelBackendMaxconnExtractorfunc); err == nil {
		return label
	}
	return "request.host"
}

func getServiceLabel(service rancherData, label string) (string, error) {
	for key, value := range service.Labels {
		if key == label {
			return value, nil
		}
	}
	return "", fmt.Errorf("label not found: %s", label)
}

// Provide allows either the Rancher API or metadata service provider to
// seed configuration into Traefik using the given configuration channel.
func (p *Provider) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
	if p.Metadata == nil {
		return p.apiProvide(configurationChan, pool, constraints)
	}
	return p.metadataProvide(configurationChan, pool, constraints)
}

func (p *Provider) loadRancherConfig(services []rancherData) *types.Configuration {

	var RancherFuncMap = template.FuncMap{
		"getPort":                     p.getPort,
		"getBackend":                  p.getBackend,
		"getWeight":                   p.getWeight,
		"getDomain":                   p.getDomain,
		"getProtocol":                 p.getProtocol,
		"getPassHostHeader":           p.getPassHostHeader,
		"getPriority":                 p.getPriority,
		"getEntryPoints":              p.getEntryPoints,
		"getBasicAuth":                p.getBasicAuth,
		"getFrontendRule":             p.getFrontendRule,
		"hasCircuitBreakerLabel":      p.hasCircuitBreakerLabel,
		"getCircuitBreakerExpression": p.getCircuitBreakerExpression,
		"hasLoadBalancerLabel":        p.hasLoadBalancerLabel,
		"getLoadBalancerMethod":       p.getLoadBalancerMethod,
		"hasMaxConnLabels":            p.hasMaxConnLabels,
		"getMaxConnAmount":            p.getMaxConnAmount,
		"getMaxConnExtractorFunc":     p.getMaxConnExtractorFunc,
		"getSticky":                   p.getSticky,
		"hasStickinessLabel":          p.hasStickinessLabel,
		"getStickinessCookieName":     p.getStickinessCookieName,
		"hasRedirect":                 hasRedirect,
		"getRedirectEntryPoint":       getRedirectEntryPoint,
		"getRedirectRegex":            getRedirectRegex,
		"getRedirectReplacement":      getRedirectReplacement,
	}

	// filter services
	filteredServices := fun.Filter(p.serviceFilter, services).([]rancherData)

	frontends := map[string]rancherData{}
	backends := map[string]rancherData{}

	for _, service := range filteredServices {
		frontendName := p.getFrontendName(service)
		frontends[frontendName] = service
		backendName := p.getBackend(service)
		backends[backendName] = service
	}

	templateObjects := struct {
		Frontends map[string]rancherData
		Backends  map[string]rancherData
		Domain    string
	}{
		frontends,
		backends,
		p.Domain,
	}

	configuration, err := p.GetConfiguration("templates/rancher.tmpl", RancherFuncMap, templateObjects)
	if err != nil {
		log.Error(err)
	}

	return configuration

}

func containerFilter(name, healthState, state string) bool {
	if healthState != "" && healthState != "healthy" && healthState != "updating-healthy" {
		log.Debugf("Filtering container %s with healthState of %s", name, healthState)
		return false
	}

	if state != "" && state != "running" && state != "updating-running" && state != "upgraded" {
		log.Debugf("Filtering container %s with state of %s", name, state)
		return false
	}

	return true
}

func (p *Provider) serviceFilter(service rancherData) bool {

	if service.Labels[types.LabelPort] == "" {
		log.Debugf("Filtering service %s without traefik.port label", service.Name)
		return false
	}

	if !isServiceEnabled(service, p.ExposedByDefault) {
		log.Debugf("Filtering disabled service %s", service.Name)
		return false
	}

	constraintTags := strings.Split(service.Labels[types.LabelTags], ",")
	if ok, failingConstraint := p.MatchConstraints(constraintTags); !ok {
		if failingConstraint != nil {
			log.Debugf("Filtering service %s with constraint %s", service.Name, failingConstraint.String())
		}
		return false
	}

	// Only filter services by Health (HealthState) and State if EnableServiceHealthFilter is true
	if p.EnableServiceHealthFilter {

		if service.Health != "" && service.Health != "healthy" && service.Health != "updating-healthy" {
			log.Debugf("Filtering service %s with healthState of %s", service.Name, service.Health)
			return false
		}

		if service.State != "" && service.State != "active" && service.State != "updating-active" && service.State != "upgraded" && service.State != "upgrading" {
			log.Debugf("Filtering service %s with state of %s", service.Name, service.State)
			return false
		}
	}

	return true
}

func isServiceEnabled(service rancherData, exposedByDefault bool) bool {

	if service.Labels[types.LabelEnable] != "" {
		var v = service.Labels[types.LabelEnable]
		return exposedByDefault && v != "false" || v == "true"
	}
	return exposedByDefault
}

// TODO will be rewrite when merge on master
func hasRedirect(service rancherData) bool {
	value, err := getServiceLabel(service, types.LabelFrontendRedirectEntryPoint)
	frep := err == nil && len(value) > 0
	value, err = getServiceLabel(service, types.LabelFrontendRedirectRegex)
	frrg := err == nil && len(value) > 0
	value, err = getServiceLabel(service, types.LabelFrontendRedirectReplacement)
	frrp := err == nil && len(value) > 0

	return frep || frrg && frrp
}

// TODO will be rewrite when merge on master
func getRedirectEntryPoint(service rancherData) string {
	value, err := getServiceLabel(service, types.LabelFrontendRedirectEntryPoint)
	if err != nil || len(value) == 0 {
		return ""
	}
	return value
}

// TODO will be rewrite when merge on master
func getRedirectRegex(service rancherData) string {
	value, err := getServiceLabel(service, types.LabelFrontendRedirectRegex)
	if err != nil || len(value) == 0 {
		return ""
	}
	return value
}

// TODO will be rewrite when merge on master
func getRedirectReplacement(service rancherData) string {
	value, err := getServiceLabel(service, types.LabelFrontendRedirectReplacement)
	if err != nil || len(value) == 0 {
		return ""
	}
	return value
}
<|endoftext|>


<commit_before>package desec

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/StackExchange/dnscontrol/v3/pkg/printer"
)

const apiBase = "https://desec.io/api/v1"

// Api layer for desec
type desecProvider struct {
	domainIndex map[string]uint32 //stores the minimum ttl of each domain. (key = domain and value = ttl)
	creds       struct {
		tokenid  string
		token    string
		user     string
		password string
	}
	mutex sync.Mutex
}

type domainObject struct {
	Created    time.Time   `json:"created,omitempty"`
	Keys       []dnssecKey `json:"keys,omitempty"`
	MinimumTTL uint32      `json:"minimum_ttl,omitempty"`
	Name       string      `json:"name,omitempty"`
	Published  time.Time   `json:"published,omitempty"`
}

type resourceRecord struct {
	Subname string   `json:"subname"`
	Records []string `json:"records"`
	TTL     uint32   `json:"ttl,omitempty"`
	Type    string   `json:"type"`
	Target  string   `json:"-"`
}

type rrResponse struct {
	resourceRecord
	Created time.Time `json:"created"`
	Domain  string    `json:"domain"`
	Name    string    `json:"name"`
}

type dnssecKey struct {
	Dnskey  string   `json:"dnskey"`
	Ds      []string `json:"ds"`
	Flags   int      `json:"flags"`
	Keytype string   `json:"keytype"`
}

type errorResponse struct {
	Detail string `json:"detail"`
}
type nonFieldError struct {
	Errors []string `json:"non_field_errors"`
}

func (c *desecProvider) authenticate() error {
	endpoint := "/auth/account/"
	var _, _, err = c.get(endpoint, "GET")
	if err != nil {
		return err
	}
	return nil
}
func (c *desecProvider) initializeDomainIndex() error {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	if c.domainIndex != nil {
		return nil
	}
	endpoint := "/domains/"
	var bodyString, resp, err = c.get(endpoint, "GET")
	if resp.StatusCode == 400 && resp.Header.Get("Link") != "" {
		//pagination is required
		links := c.convertLinks(resp.Header.Get("Link"))
		endpoint = links["first"]
		printer.Debugf("initial endpoint %s\n", endpoint)
		for endpoint != "" {
			bodyString, resp, err = c.get(endpoint, "GET")
			if err != nil {
				if resp.StatusCode == 404 {
					return nil
				}
				return fmt.Errorf("failed fetching domains: %s", err)
			}
			err = c.buildIndexFromResponse(bodyString)
			if err != nil {
				return fmt.Errorf("failed fetching domains: %s", err)
			}
			links = c.convertLinks(resp.Header.Get("Link"))
			endpoint = links["next"]
			printer.Debugf("next endpoint %s\n", endpoint)
		}
		printer.Debugf("Domain Index initilized with pagination (%d domains)\n", len(c.domainIndex))
		return nil //domainIndex was build using pagination without errors
	}

	//no pagination required
	if err != nil && resp.StatusCode != 400 {
		if resp.StatusCode == 404 {
			return nil
		}
		return fmt.Errorf("failed fetching domains: %s", err)
	}
	err = c.buildIndexFromResponse(bodyString)
	if err == nil {
		printer.Debugf("Domain Index initilized without pagination (%d domains)\n", len(c.domainIndex))
	}
	return err
}

// buildIndexFromResponse takes the bodyString from initializeDomainIndex and builds the domainIndex
func (c *desecProvider) buildIndexFromResponse(bodyString []byte) error {
	if c.domainIndex == nil {
		c.domainIndex = map[string]uint32{}
	}
	var dr []domainObject
	err := json.Unmarshal(bodyString, &dr)
	if err != nil {
		return err
	}
	for _, domain := range dr {
		//deSEC allows different minimum ttls per domain
		//we store the actual minimum ttl to use it in desecProvider.go GetDomainCorrections() to enforce the minimum ttl and avoid api errors.
		c.domainIndex[domain.Name] = domain.MinimumTTL
	}
	return nil
}

// Parses the Link Header into a map (https://github.com/desec-io/desec-tools/blob/master/fetch_zone.py#L13)
func (c *desecProvider) convertLinks(links string) map[string]string {
	mapping := make(map[string]string)
	printer.Debugf("Header: %s\n", links)
	for _, link := range strings.Split(links, ", ") {
		tmpurl := strings.Split(link, "; ")
		if len(tmpurl) != 2 {
			printer.Printf("unexpected link header %s", link)
			continue
		}
		r := regexp.MustCompile(`rel="(.*)"`)
		matches := r.FindStringSubmatch(tmpurl[1])
		if len(matches) != 2 {
			printer.Printf("unexpected label %s", tmpurl[1])
			continue
		}
		// mapping["$label"] = "$URL"
		//URL = https://desec.io/api/v1/domains/{domain}/rrsets/?cursor=:next_cursor
		mapping[matches[1]] = strings.TrimSuffix(strings.TrimPrefix(tmpurl[0], "<"), ">")
	}
	return mapping
}

func (c *desecProvider) getRecords(domain string) ([]resourceRecord, error) {
	endpoint := "/domains/%s/rrsets/"
	var rrsNew []resourceRecord
	var bodyString, resp, err = c.get(fmt.Sprintf(endpoint, domain), "GET")
	if resp.StatusCode == 400 && resp.Header.Get("Link") != "" {
		//pagination required
		links := c.convertLinks(resp.Header.Get("Link"))
		endpoint = links["first"]
		printer.Debugf("getRecords: initial endpoint %s\n", fmt.Sprintf(endpoint, domain))
		for endpoint != "" {
			bodyString, resp, err = c.get(endpoint, "GET")
			if err != nil {
				if resp.StatusCode == 404 {
					return rrsNew, nil
				}
				return rrsNew, fmt.Errorf("getRecords: failed fetching rrsets: %s", err)
			}
			tmp, err := generateRRSETfromResponse(bodyString)
			if err != nil {
				return rrsNew, fmt.Errorf("failed fetching records for domain %s (deSEC): %s", domain, err)
			}
			rrsNew = append(rrsNew, tmp...)
			links = c.convertLinks(resp.Header.Get("Link"))
			endpoint = links["next"]
			printer.Debugf("getRecords: next endpoint %s\n", endpoint)
		}
		printer.Debugf("Build rrset using pagination (%d rrs)\n", len(rrsNew))
		return rrsNew, nil //domainIndex was build using pagination without errors
	}
	//no pagination
	if err != nil {
		return rrsNew, fmt.Errorf("failed fetching records for domain %s (deSEC): %s", domain, err)
	}
	tmp, err := generateRRSETfromResponse(bodyString)
	if err != nil {
		return rrsNew, err
	}
	rrsNew = append(rrsNew, tmp...)
	printer.Debugf("Build rrset without pagination (%d rrs)\n", len(rrsNew))
	return rrsNew, nil
}

// generateRRSETfromResponse takes the response rrset api calls and returns []resourceRecord
func generateRRSETfromResponse(bodyString []byte) ([]resourceRecord, error) {
	var rrs []rrResponse
	var rrsNew []resourceRecord
	err := json.Unmarshal(bodyString, &rrs)
	if err != nil {
		return rrsNew, err
	}
	// deSEC returns round robin records as array but dnsconfig expects single entries for each record
	// we will create one object per record except of TXT records which are handled as array of string by dnscontrol aswell.
	for i := range rrs {
		tmp := resourceRecord{
			TTL:     rrs[i].TTL,
			Type:    rrs[i].Type,
			Subname: rrs[i].Subname,
			Records: rrs[i].Records,
		}
		rrsNew = append(rrsNew, tmp)
	}
	return rrsNew, nil
}

func (c *desecProvider) createDomain(domain string) error {
	endpoint := "/domains/"
	pl := domainObject{Name: domain}
	byt, _ := json.Marshal(pl)
	var resp []byte
	var err error
	if resp, err = c.post(endpoint, "POST", byt); err != nil {
		return fmt.Errorf("failed domain create (deSEC): %v", err)
	}
	dm := domainObject{}
	err = json.Unmarshal(resp, &dm)
	if err != nil {
		return err
	}
	printer.Printf("To enable DNSSEC validation for your domain, make sure to convey the DS record(s) to your registrar:\n")
	printer.Printf("%+q", dm.Keys)
	return nil
}

// upsertRR will create or override the RRSet with the provided resource record.
func (c *desecProvider) upsertRR(rr []resourceRecord, domain string) error {
	endpoint := fmt.Sprintf("/domains/%s/rrsets/", domain)
	byt, _ := json.Marshal(rr)
	if _, err := c.post(endpoint, "PUT", byt); err != nil {
		return fmt.Errorf("failed create RRset (deSEC): %v", err)
	}
	return nil
}

func (c *desecProvider) deleteRR(domain, shortname, t string) error {
	endpoint := fmt.Sprintf("/domains/%s/rrsets/%s/%s/", domain, shortname, t)
	if _, _, err := c.get(endpoint, "DELETE"); err != nil {
		return fmt.Errorf("failed delete RRset (deSEC): %v", err)
	}
	return nil
}

func (c *desecProvider) get(target, method string) ([]byte, *http.Response, error) {
	retrycnt := 0
	var endpoint string
	if strings.Contains(target, "http") {
		endpoint = target
	} else {
		endpoint = apiBase + target
	}
retry:
	client := &http.Client{}
	req, _ := http.NewRequest(method, endpoint, nil)
	q := req.URL.Query()
	req.Header.Add("Authorization", fmt.Sprintf("Token %s", c.creds.token))

	req.URL.RawQuery = q.Encode()

	resp, err := client.Do(req)
	if err != nil {
		return []byte{}, resp, err
	}

	bodyString, _ := io.ReadAll(resp.Body)
	// Got error from API ?
	if resp.StatusCode > 299 {
		if resp.StatusCode == 429 && retrycnt < 5 {
			retrycnt++
			//we've got rate limiting and will try to get the Retry-After Header if this fails we fallback to sleep for 500ms max. 5 retries.
			waitfor := resp.Header.Get("Retry-After")
			if waitfor != "" {
				wait, err := strconv.ParseInt(waitfor, 10, 64)
				if err == nil {
					if wait > 180 {
						return []byte{}, resp, fmt.Errorf("rate limiting exceeded")
					}
					printer.Warnf("Rate limiting.. waiting for %s seconds", waitfor)
					time.Sleep(time.Duration(wait+1) * time.Second)
					goto retry
				}
			}
			printer.Warnf("Rate limiting.. waiting for 500 milliseconds")
			time.Sleep(500 * time.Millisecond)
			goto retry
		}
		var errResp errorResponse
		var nfieldErrors []nonFieldError
		err = json.Unmarshal(bodyString, &errResp)
		if err == nil {
			return bodyString, resp, fmt.Errorf("%s", errResp.Detail)
		}
		err = json.Unmarshal(bodyString, &nfieldErrors)
		if err == nil && len(nfieldErrors) > 0 {
			if len(nfieldErrors[0].Errors) > 0 {
				return bodyString, resp, fmt.Errorf("%s", nfieldErrors[0].Errors[0])
			}
		}
		return bodyString, resp, fmt.Errorf("HTTP status %s Body: %s, the API does not provide more information", resp.Status, bodyString)
	}
	return bodyString, resp, nil
}

func (c *desecProvider) post(target, method string, payload []byte) ([]byte, error) {
	retrycnt := 0
	var endpoint string
	if strings.Contains(target, "http") {
		endpoint = target
	} else {
		endpoint = apiBase + target
	}
retry:
	client := &http.Client{}
	req, err := http.NewRequest(method, endpoint, bytes.NewReader(payload))
	if err != nil {
		return []byte{}, err
	}
	q := req.URL.Query()
	if endpoint != "/auth/login/" {
		req.Header.Add("Authorization", fmt.Sprintf("Token %s", c.creds.token))
	}
	req.Header.Set("Content-Type", "application/json")

	req.URL.RawQuery = q.Encode()

	resp, err := client.Do(req)
	if err != nil {
		return []byte{}, err
	}

	bodyString, _ := io.ReadAll(resp.Body)

	// Got error from API ?
	if resp.StatusCode > 299 {
		if resp.StatusCode == 429 && retrycnt < 5 {
			retrycnt++
			//we've got rate limiting and will try to get the Retry-After Header if this fails we fallback to sleep for 500ms max. 5 retries.
			waitfor := resp.Header.Get("Retry-After")
			if waitfor != "" {
				wait, err := strconv.ParseInt(waitfor, 10, 64)
				if err == nil {
					if wait > 180 {
						return []byte{}, fmt.Errorf("rate limiting exceeded")
					}
					printer.Warnf("Rate limiting.. waiting for %s seconds", waitfor)
					time.Sleep(time.Duration(wait+1) * time.Second)
					goto retry
				}
			}
			printer.Warnf("Rate limiting.. waiting for 500 milliseconds")
			time.Sleep(500 * time.Millisecond)
			goto retry
		}
		var errResp errorResponse
		var nfieldErrors []nonFieldError
		err = json.Unmarshal(bodyString, &errResp)
		if err == nil {
			return bodyString, fmt.Errorf("HTTP status %d %s details: %s", resp.StatusCode, resp.Status, errResp.Detail)
		}
		err = json.Unmarshal(bodyString, &nfieldErrors)
		if err == nil && len(nfieldErrors) > 0 {
			if len(nfieldErrors[0].Errors) > 0 {
				return bodyString, fmt.Errorf("%s", nfieldErrors[0].Errors[0])
			}
		}
		return bodyString, fmt.Errorf("HTTP status %s Body: %s, the API does not provide more information", resp.Status, bodyString)
	}
	//time.Sleep(334 * time.Millisecond)
	return bodyString, nil
}
<commit_msg>DESEC: Check authentication via domains API (#1723)<commit_after>package desec

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/StackExchange/dnscontrol/v3/pkg/printer"
)

const apiBase = "https://desec.io/api/v1"

// Api layer for desec
type desecProvider struct {
	domainIndex map[string]uint32 //stores the minimum ttl of each domain. (key = domain and value = ttl)
	creds       struct {
		tokenid  string
		token    string
		user     string
		password string
	}
	mutex sync.Mutex
}

type domainObject struct {
	Created    time.Time   `json:"created,omitempty"`
	Keys       []dnssecKey `json:"keys,omitempty"`
	MinimumTTL uint32      `json:"minimum_ttl,omitempty"`
	Name       string      `json:"name,omitempty"`
	Published  time.Time   `json:"published,omitempty"`
}

type resourceRecord struct {
	Subname string   `json:"subname"`
	Records []string `json:"records"`
	TTL     uint32   `json:"ttl,omitempty"`
	Type    string   `json:"type"`
	Target  string   `json:"-"`
}

type rrResponse struct {
	resourceRecord
	Created time.Time `json:"created"`
	Domain  string    `json:"domain"`
	Name    string    `json:"name"`
}

type dnssecKey struct {
	Dnskey  string   `json:"dnskey"`
	Ds      []string `json:"ds"`
	Flags   int      `json:"flags"`
	Keytype string   `json:"keytype"`
}

type errorResponse struct {
	Detail string `json:"detail"`
}
type nonFieldError struct {
	Errors []string `json:"non_field_errors"`
}

func (c *desecProvider) authenticate() error {
	endpoint := "/auth/account/"
	var _, resp, err = c.get(endpoint, "GET")
	//restricted tokens are valid, but get 403 on /auth/account
	//invalid tokens get 401
	if resp.StatusCode == 403 {
		return nil
	}
	if err != nil {
		return err
	}
	return nil
}
func (c *desecProvider) initializeDomainIndex() error {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	if c.domainIndex != nil {
		return nil
	}
	endpoint := "/domains/"
	var bodyString, resp, err = c.get(endpoint, "GET")
	if resp.StatusCode == 400 && resp.Header.Get("Link") != "" {
		//pagination is required
		links := c.convertLinks(resp.Header.Get("Link"))
		endpoint = links["first"]
		printer.Debugf("initial endpoint %s\n", endpoint)
		for endpoint != "" {
			bodyString, resp, err = c.get(endpoint, "GET")
			if err != nil {
				if resp.StatusCode == 404 {
					return nil
				}
				return fmt.Errorf("failed fetching domains: %s", err)
			}
			err = c.buildIndexFromResponse(bodyString)
			if err != nil {
				return fmt.Errorf("failed fetching domains: %s", err)
			}
			links = c.convertLinks(resp.Header.Get("Link"))
			endpoint = links["next"]
			printer.Debugf("next endpoint %s\n", endpoint)
		}
		printer.Debugf("Domain Index initilized with pagination (%d domains)\n", len(c.domainIndex))
		return nil //domainIndex was build using pagination without errors
	}

	//no pagination required
	if err != nil && resp.StatusCode != 400 {
		if resp.StatusCode == 404 {
			return nil
		}
		return fmt.Errorf("failed fetching domains: %s", err)
	}
	err = c.buildIndexFromResponse(bodyString)
	if err == nil {
		printer.Debugf("Domain Index initilized without pagination (%d domains)\n", len(c.domainIndex))
	}
	return err
}

// buildIndexFromResponse takes the bodyString from initializeDomainIndex and builds the domainIndex
func (c *desecProvider) buildIndexFromResponse(bodyString []byte) error {
	if c.domainIndex == nil {
		c.domainIndex = map[string]uint32{}
	}
	var dr []domainObject
	err := json.Unmarshal(bodyString, &dr)
	if err != nil {
		return err
	}
	for _, domain := range dr {
		//deSEC allows different minimum ttls per domain
		//we store the actual minimum ttl to use it in desecProvider.go GetDomainCorrections() to enforce the minimum ttl and avoid api errors.
		c.domainIndex[domain.Name] = domain.MinimumTTL
	}
	return nil
}

// Parses the Link Header into a map (https://github.com/desec-io/desec-tools/blob/master/fetch_zone.py#L13)
func (c *desecProvider) convertLinks(links string) map[string]string {
	mapping := make(map[string]string)
	printer.Debugf("Header: %s\n", links)
	for _, link := range strings.Split(links, ", ") {
		tmpurl := strings.Split(link, "; ")
		if len(tmpurl) != 2 {
			printer.Printf("unexpected link header %s", link)
			continue
		}
		r := regexp.MustCompile(`rel="(.*)"`)
		matches := r.FindStringSubmatch(tmpurl[1])
		if len(matches) != 2 {
			printer.Printf("unexpected label %s", tmpurl[1])
			continue
		}
		// mapping["$label"] = "$URL"
		//URL = https://desec.io/api/v1/domains/{domain}/rrsets/?cursor=:next_cursor
		mapping[matches[1]] = strings.TrimSuffix(strings.TrimPrefix(tmpurl[0], "<"), ">")
	}
	return mapping
}

func (c *desecProvider) getRecords(domain string) ([]resourceRecord, error) {
	endpoint := "/domains/%s/rrsets/"
	var rrsNew []resourceRecord
	var bodyString, resp, err = c.get(fmt.Sprintf(endpoint, domain), "GET")
	if resp.StatusCode == 400 && resp.Header.Get("Link") != "" {
		//pagination required
		links := c.convertLinks(resp.Header.Get("Link"))
		endpoint = links["first"]
		printer.Debugf("getRecords: initial endpoint %s\n", fmt.Sprintf(endpoint, domain))
		for endpoint != "" {
			bodyString, resp, err = c.get(endpoint, "GET")
			if err != nil {
				if resp.StatusCode == 404 {
					return rrsNew, nil
				}
				return rrsNew, fmt.Errorf("getRecords: failed fetching rrsets: %s", err)
			}
			tmp, err := generateRRSETfromResponse(bodyString)
			if err != nil {
				return rrsNew, fmt.Errorf("failed fetching records for domain %s (deSEC): %s", domain, err)
			}
			rrsNew = append(rrsNew, tmp...)
			links = c.convertLinks(resp.Header.Get("Link"))
			endpoint = links["next"]
			printer.Debugf("getRecords: next endpoint %s\n", endpoint)
		}
		printer.Debugf("Build rrset using pagination (%d rrs)\n", len(rrsNew))
		return rrsNew, nil //domainIndex was build using pagination without errors
	}
	//no pagination
	if err != nil {
		return rrsNew, fmt.Errorf("failed fetching records for domain %s (deSEC): %s", domain, err)
	}
	tmp, err := generateRRSETfromResponse(bodyString)
	if err != nil {
		return rrsNew, err
	}
	rrsNew = append(rrsNew, tmp...)
	printer.Debugf("Build rrset without pagination (%d rrs)\n", len(rrsNew))
	return rrsNew, nil
}

// generateRRSETfromResponse takes the response rrset api calls and returns []resourceRecord
func generateRRSETfromResponse(bodyString []byte) ([]resourceRecord, error) {
	var rrs []rrResponse
	var rrsNew []resourceRecord
	err := json.Unmarshal(bodyString, &rrs)
	if err != nil {
		return rrsNew, err
	}
	// deSEC returns round robin records as array but dnsconfig expects single entries for each record
	// we will create one object per record except of TXT records which are handled as array of string by dnscontrol aswell.
	for i := range rrs {
		tmp := resourceRecord{
			TTL:     rrs[i].TTL,
			Type:    rrs[i].Type,
			Subname: rrs[i].Subname,
			Records: rrs[i].Records,
		}
		rrsNew = append(rrsNew, tmp)
	}
	return rrsNew, nil
}

func (c *desecProvider) createDomain(domain string) error {
	endpoint := "/domains/"
	pl := domainObject{Name: domain}
	byt, _ := json.Marshal(pl)
	var resp []byte
	var err error
	if resp, err = c.post(endpoint, "POST", byt); err != nil {
		return fmt.Errorf("failed domain create (deSEC): %v", err)
	}
	dm := domainObject{}
	err = json.Unmarshal(resp, &dm)
	if err != nil {
		return err
	}
	printer.Printf("To enable DNSSEC validation for your domain, make sure to convey the DS record(s) to your registrar:\n")
	printer.Printf("%+q", dm.Keys)
	return nil
}

// upsertRR will create or override the RRSet with the provided resource record.
func (c *desecProvider) upsertRR(rr []resourceRecord, domain string) error {
	endpoint := fmt.Sprintf("/domains/%s/rrsets/", domain)
	byt, _ := json.Marshal(rr)
	if _, err := c.post(endpoint, "PUT", byt); err != nil {
		return fmt.Errorf("failed create RRset (deSEC): %v", err)
	}
	return nil
}

func (c *desecProvider) deleteRR(domain, shortname, t string) error {
	endpoint := fmt.Sprintf("/domains/%s/rrsets/%s/%s/", domain, shortname, t)
	if _, _, err := c.get(endpoint, "DELETE"); err != nil {
		return fmt.Errorf("failed delete RRset (deSEC): %v", err)
	}
	return nil
}

func (c *desecProvider) get(target, method string) ([]byte, *http.Response, error) {
	retrycnt := 0
	var endpoint string
	if strings.Contains(target, "http") {
		endpoint = target
	} else {
		endpoint = apiBase + target
	}
retry:
	client := &http.Client{}
	req, _ := http.NewRequest(method, endpoint, nil)
	q := req.URL.Query()
	req.Header.Add("Authorization", fmt.Sprintf("Token %s", c.creds.token))

	req.URL.RawQuery = q.Encode()

	resp, err := client.Do(req)
	if err != nil {
		return []byte{}, resp, err
	}

	bodyString, _ := io.ReadAll(resp.Body)
	// Got error from API ?
	if resp.StatusCode > 299 {
		if resp.StatusCode == 429 && retrycnt < 5 {
			retrycnt++
			//we've got rate limiting and will try to get the Retry-After Header if this fails we fallback to sleep for 500ms max. 5 retries.
			waitfor := resp.Header.Get("Retry-After")
			if waitfor != "" {
				wait, err := strconv.ParseInt(waitfor, 10, 64)
				if err == nil {
					if wait > 180 {
						return []byte{}, resp, fmt.Errorf("rate limiting exceeded")
					}
					printer.Warnf("Rate limiting.. waiting for %s seconds", waitfor)
					time.Sleep(time.Duration(wait+1) * time.Second)
					goto retry
				}
			}
			printer.Warnf("Rate limiting.. waiting for 500 milliseconds")
			time.Sleep(500 * time.Millisecond)
			goto retry
		}
		var errResp errorResponse
		var nfieldErrors []nonFieldError
		err = json.Unmarshal(bodyString, &errResp)
		if err == nil {
			return bodyString, resp, fmt.Errorf("%s", errResp.Detail)
		}
		err = json.Unmarshal(bodyString, &nfieldErrors)
		if err == nil && len(nfieldErrors) > 0 {
			if len(nfieldErrors[0].Errors) > 0 {
				return bodyString, resp, fmt.Errorf("%s", nfieldErrors[0].Errors[0])
			}
		}
		return bodyString, resp, fmt.Errorf("HTTP status %s Body: %s, the API does not provide more information", resp.Status, bodyString)
	}
	return bodyString, resp, nil
}

func (c *desecProvider) post(target, method string, payload []byte) ([]byte, error) {
	retrycnt := 0
	var endpoint string
	if strings.Contains(target, "http") {
		endpoint = target
	} else {
		endpoint = apiBase + target
	}
retry:
	client := &http.Client{}
	req, err := http.NewRequest(method, endpoint, bytes.NewReader(payload))
	if err != nil {
		return []byte{}, err
	}
	q := req.URL.Query()
	if endpoint != "/auth/login/" {
		req.Header.Add("Authorization", fmt.Sprintf("Token %s", c.creds.token))
	}
	req.Header.Set("Content-Type", "application/json")

	req.URL.RawQuery = q.Encode()

	resp, err := client.Do(req)
	if err != nil {
		return []byte{}, err
	}

	bodyString, _ := io.ReadAll(resp.Body)

	// Got error from API ?
	if resp.StatusCode > 299 {
		if resp.StatusCode == 429 && retrycnt < 5 {
			retrycnt++
			//we've got rate limiting and will try to get the Retry-After Header if this fails we fallback to sleep for 500ms max. 5 retries.
			waitfor := resp.Header.Get("Retry-After")
			if waitfor != "" {
				wait, err := strconv.ParseInt(waitfor, 10, 64)
				if err == nil {
					if wait > 180 {
						return []byte{}, fmt.Errorf("rate limiting exceeded")
					}
					printer.Warnf("Rate limiting.. waiting for %s seconds", waitfor)
					time.Sleep(time.Duration(wait+1) * time.Second)
					goto retry
				}
			}
			printer.Warnf("Rate limiting.. waiting for 500 milliseconds")
			time.Sleep(500 * time.Millisecond)
			goto retry
		}
		var errResp errorResponse
		var nfieldErrors []nonFieldError
		err = json.Unmarshal(bodyString, &errResp)
		if err == nil {
			return bodyString, fmt.Errorf("HTTP status %d %s details: %s", resp.StatusCode, resp.Status, errResp.Detail)
		}
		err = json.Unmarshal(bodyString, &nfieldErrors)
		if err == nil && len(nfieldErrors) > 0 {
			if len(nfieldErrors[0].Errors) > 0 {
				return bodyString, fmt.Errorf("%s", nfieldErrors[0].Errors[0])
			}
		}
		return bodyString, fmt.Errorf("HTTP status %s Body: %s, the API does not provide more information", resp.Status, bodyString)
	}
	//time.Sleep(334 * time.Millisecond)
	return bodyString, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package entrypoint

import (
	"errors"
	"flag"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"syscall"

	"github.com/sirupsen/logrus"
	"k8s.io/test-infra/prow/logrusutil"
	"k8s.io/test-infra/prow/pod-utils/wrapper"
)

type options struct {
	args []string

	wrapperOptions *wrapper.Options
}

func (o *options) Validate() error {
	if len(o.args) == 0 {
		return errors.New("no process to wrap specified")
	}

	return o.wrapperOptions.Validate()
}

func (o *options) Complete(args []string) {
	o.args = args
}

func gatherOptions() options {
	o := options{}
	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	o.wrapperOptions = wrapper.BindOptions(fs)
	fs.Parse(os.Args[1:])
	o.Complete(fs.Args())
	return o
}

func main() {
	o := gatherOptions()
	if err := o.Validate(); err != nil {
		logrus.Fatalf("Invalid options: %v", err)
	}

	logrus.SetFormatter(
		logrusutil.NewDefaultFieldsFormatter(nil, logrus.Fields{"component": "entrypoint"}),
	)

	processLogFile, err := os.Create(o.wrapperOptions.ProcessLog)
	if err != nil {
		logrus.WithError(err).Fatal("Could not open output process logfile")
	}
	output := io.MultiWriter(os.Stdout, processLogFile)
	logrus.SetOutput(output)

	executable := o.args[0]
	var arguments []string
	if len(o.args) > 1 {
		arguments = o.args[1:]
	}
	command := exec.Command(executable, arguments...)
	command.Stderr = output
	command.Stdout = output
	if err := command.Start(); err != nil {
		if err := ioutil.WriteFile(o.wrapperOptions.MarkerFile, []byte("127"), os.ModePerm); err != nil {
			logrus.WithError(err).Fatal("Could not write to marker file")
		}
		logrus.WithError(err).Fatal("Could not start the process")
	}

	commandErr := command.Wait()

	returnCode := "1"
	if commandErr == nil {
		returnCode = "0"
	} else if exitErr, ok := err.(*exec.ExitError); ok {
		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			returnCode = strconv.Itoa(status.ExitStatus())
		}
	}

	if err := ioutil.WriteFile(o.wrapperOptions.MarkerFile, []byte(returnCode), os.ModePerm); err != nil {
		logrus.WithError(err).Fatal("Could not write return code to marker file")
	}
	if commandErr != nil {
		logrus.WithError(err).Fatal("Wrapped process failed")
	}
}
<commit_msg>Add a timeout to the entrypoint wrapper<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package entrypoint

import (
	"errors"
	"flag"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"syscall"
	"time"

	"github.com/sirupsen/logrus"
	"k8s.io/test-infra/prow/logrusutil"
	"k8s.io/test-infra/prow/pod-utils/wrapper"
)

type options struct {
	args    []string
	timeout time.Duration

	wrapperOptions *wrapper.Options
}

func (o *options) Validate() error {
	if len(o.args) == 0 {
		return errors.New("no process to wrap specified")
	}

	return o.wrapperOptions.Validate()
}

func (o *options) Complete(args []string) {
	o.args = args
}

func gatherOptions() options {
	o := options{}
	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	o.wrapperOptions = wrapper.BindOptions(fs)
	fs.DurationVar(&o.timeout, "timeout", 5*time.Hour, "Timeout for the test command.")
	fs.Parse(os.Args[1:])
	o.Complete(fs.Args())
	return o
}

func main() {
	o := gatherOptions()
	if err := o.Validate(); err != nil {
		logrus.Fatalf("Invalid options: %v", err)
	}

	logrus.SetFormatter(
		logrusutil.NewDefaultFieldsFormatter(nil, logrus.Fields{"component": "entrypoint"}),
	)

	processLogFile, err := os.Create(o.wrapperOptions.ProcessLog)
	if err != nil {
		logrus.WithError(err).Fatal("Could not open output process logfile")
	}
	output := io.MultiWriter(os.Stdout, processLogFile)
	logrus.SetOutput(output)

	executable := o.args[0]
	var arguments []string
	if len(o.args) > 1 {
		arguments = o.args[1:]
	}
	command := exec.Command(executable, arguments...)
	command.Stderr = output
	command.Stdout = output
	if err := command.Start(); err != nil {
		if err := ioutil.WriteFile(o.wrapperOptions.MarkerFile, []byte("127"), os.ModePerm); err != nil {
			logrus.WithError(err).Fatal("Could not write to marker file")
		}
		logrus.WithError(err).Fatal("Could not start the process")
	}

	var commandErr error
	done := make(chan error)
	go func() {
		done <- command.Wait()
	}()
	select {
	case err := <-done:
		commandErr = err
	case <-time.After(o.timeout):
		logrus.Errorf("Process did not finish before %s timeout", o.timeout)
		if err := command.Process.Kill(); err != nil {
			logrus.WithError(err).Error("Could not kill process after timeout")
		}
		commandErr = errors.New("process timed out")
	}

	returnCode := "1"
	if commandErr == nil {
		returnCode = "0"
	} else if exitErr, ok := err.(*exec.ExitError); ok {
		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			returnCode = strconv.Itoa(status.ExitStatus())
		}
	}

	if err := ioutil.WriteFile(o.wrapperOptions.MarkerFile, []byte(returnCode), os.ModePerm); err != nil {
		logrus.WithError(err).Fatal("Could not write return code to marker file")
	}
	if commandErr != nil {
		logrus.WithError(err).Fatal("Wrapped process failed")
	}
}
<|endoftext|>


<commit_before>package spatialite

import (
	"database/sql"
	"errors"

	"github.com/mattn/go-sqlite3"
)

type entrypoint struct {
	lib  string
	proc string
}

var LibNames = []entrypoint{
	{"mod_spatialite", "sqlite3_modspatialite_init"},
	{"libspatialite.so", "sqlite3_modspatialite_init"},
}

var ErrSpatialiteNotFound = errors.New("shaxbee/go-spatialite: spatialite extension not found.")

func init() {
	sql.Register("spatialite", &sqlite3.SQLiteDriver{
		ConnectHook: func(conn *sqlite3.SQLiteConn) error {
			for _, v := range LibNames {
				if err := conn.LoadExtension(v.lib, v.proc); err == nil {
					return nil
				}
			}
			return ErrSpatialiteNotFound
		},
	})
}
<commit_msg>add spatialite_init_ex entry point for libspatialite.so<commit_after>package spatialite

import (
	"database/sql"
	"errors"

	"github.com/mattn/go-sqlite3"
)

type entrypoint struct {
	lib  string
	proc string
}

var LibNames = []entrypoint{
	{"mod_spatialite", "sqlite3_modspatialite_init"},
	{"libspatialite.so", "sqlite3_modspatialite_init"},
	{"libspatialite.so", "spatialite_init_ex"},
}

var ErrSpatialiteNotFound = errors.New("shaxbee/go-spatialite: spatialite extension not found.")

func init() {
	sql.Register("spatialite", &sqlite3.SQLiteDriver{
		ConnectHook: func(conn *sqlite3.SQLiteConn) error {
			for _, v := range LibNames {
				if err := conn.LoadExtension(v.lib, v.proc); err == nil {
					return nil
				}
			}
			return ErrSpatialiteNotFound
		},
	})
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"
	"testing"

	"github.com/golang/dep"
	"github.com/golang/dep/internal/gps"
	"github.com/golang/dep/internal/test"
	"github.com/pkg/errors"
)

const testGlideProjectRoot = "github.com/golang/notexist"

var (
	discardLogger  = log.New(ioutil.Discard, "", 0)
	discardLoggers = &dep.Loggers{Out: discardLogger, Err: discardLogger}
)

func newTestContext(h *test.Helper) *dep.Ctx {
	h.TempDir("src")
	pwd := h.Path(".")
	return &dep.Ctx{
		GOPATH:  pwd,
		Loggers: discardLoggers,
	}
}

func TestGlideConfig_Import(t *testing.T) {
	h := test.NewHelper(t)
	defer h.Cleanup()

	cacheDir := "gps-repocache"
	h.TempDir(cacheDir)
	h.TempDir("src")
	h.TempDir(filepath.Join("src", testGlideProjectRoot))
	h.TempCopy(filepath.Join(testGlideProjectRoot, glideYamlName), "glide/glide.yaml")
	h.TempCopy(filepath.Join(testGlideProjectRoot, glideLockName), "glide/glide.lock")

	projectRoot := h.Path(testGlideProjectRoot)
	sm, err := gps.NewSourceManager(h.Path(cacheDir))
	h.Must(err)
	defer sm.Release()

	// Capture stderr so we can verify output
	verboseOutput := &bytes.Buffer{}
	logger := log.New(verboseOutput, "", 0)

	g := newGlideImporter(logger, false, sm) // Disable verbose so that we don't print values that change each test run
	if !g.HasDepMetadata(projectRoot) {
		t.Fatal("Expected the importer to detect the glide configuration files")
	}

	m, l, err := g.Import(projectRoot, testGlideProjectRoot)
	h.Must(err)

	if m == nil {
		t.Fatal("Expected the manifest to be generated")
	}

	if l == nil {
		t.Fatal("Expected the lock to be generated")
	}

	goldenFile := "glide/expected_import_output.txt"
	got := verboseOutput.String()
	want := h.GetTestFileString(goldenFile)
	if want != got {
		if *test.UpdateGolden {
			if err := h.WriteTestFile(goldenFile, got); err != nil {
				t.Fatalf("%+v", errors.Wrapf(err, "Unable to write updated golden file %s", goldenFile))
			}
		} else {
			t.Fatalf("expected %s, got %s", want, got)
		}
	}
}

func TestGlideConfig_Import_MissingLockFile(t *testing.T) {

	h := test.NewHelper(t)
	defer h.Cleanup()

	cacheDir := "gps-repocache"
	h.TempDir(cacheDir)
	h.TempDir("src")
	h.TempDir(filepath.Join("src", "glidetest"))
	h.TempCopy(filepath.Join("glidetest", glideYamlName), "glide/glide.yaml")

	projectRoot := h.Path("glidetest")
	sm, err := gps.NewSourceManager(h.Path(cacheDir))
	h.Must(err)
	defer sm.Release()

	g := newGlideImporter(discardLogger, true, sm)
	if !g.HasDepMetadata(projectRoot) {
		t.Fatal("The glide importer should gracefully handle when only glide.yaml is present")
	}

	m, l, err := g.Import(projectRoot, testGlideProjectRoot)
	h.Must(err)

	if m == nil {
		t.Fatal("Expected the manifest to be generated")
	}

	if l != nil {
		t.Fatal("Expected the lock to not be generated")
	}
}

func TestGlideConfig_Convert_Project(t *testing.T) {

	h := test.NewHelper(t)
	defer h.Cleanup()

	pkg := "github.com/sdboyer/deptest"
	repo := "https://github.com/sdboyer/deptest.git"

	ctx := newTestContext(h)
	sm, err := ctx.SourceManager()
	h.Must(err)
	defer sm.Release()

	g := newGlideImporter(ctx.Err, true, sm)
	g.yaml = glideYaml{
		Imports: []glidePackage{
			{
				Name:       pkg,
				Repository: repo,
				Reference:  "1.0.0",
			},
		},
	}
	g.lock = &glideLock{
		Imports: []glideLockedPackage{
			{
				Name:       pkg,
				Repository: repo,
				Reference:  "ff2948a2ac8f538c4ecd55962e919d1e13e74baf",
			},
		},
	}

	manifest, lock, err := g.convert(testGlideProjectRoot)
	if err != nil {
		t.Fatal(err)
	}

	d, ok := manifest.Constraints[gps.ProjectRoot(pkg)]
	if !ok {
		t.Fatal("Expected the manifest to have a dependency for 'github.com/sdboyer/deptest' but got none")
	}

	wantC := "^1.0.0"
	gotC := d.Constraint.String()
	if gotC != wantC {
		t.Fatalf("Expected manifest constraint to be %s, got %s", wantC, gotC)
	}

	gotS := d.Source
	if gotS != repo {
		t.Fatalf("Expected manifest source to be %s, got %s", repo, gotS)
	}

	wantP := 1
	gotP := len(lock.P)
	if gotP != 1 {
		t.Fatalf("Expected the lock to contain %d project but got %d", wantP, gotP)
	}

	p := lock.P[0]
	gotPr := string(p.Ident().ProjectRoot)
	if gotPr != pkg {
		t.Fatalf("Expected the lock to have a project for %s but got '%s'", pkg, gotPr)
	}

	gotS = p.Ident().Source
	if gotS != repo {
		t.Fatalf("Expected locked source to be %s, got '%s'", repo, gotS)
	}

	lv := p.Version()
	lpv, ok := lv.(gps.PairedVersion)
	if !ok {
		t.Fatalf("Expected locked version to be a PairedVersion but got %T", lv)
	}

	wantRev := "ff2948a2ac8f538c4ecd55962e919d1e13e74baf"
	gotRev := lpv.Underlying().String()
	if gotRev != wantRev {
		t.Fatalf("Expected locked revision to be %s, got %s", wantRev, gotRev)
	}

	wantV := "v1.0.0"
	gotV := lpv.String()
	if gotV != wantV {
		t.Fatalf("Expected locked version to be %s, got %s", wantV, gotV)
	}
}

func TestGlideConfig_Convert_TestProject(t *testing.T) {
	h := test.NewHelper(t)
	defer h.Cleanup()

	pkg := "github.com/sdboyer/deptest"

	ctx := newTestContext(h)
	sm, err := ctx.SourceManager()
	h.Must(err)
	defer sm.Release()

	g := newGlideImporter(ctx.Err, true, sm)
	g.yaml = glideYaml{
		TestImports: []glidePackage{
			{
				Name:      pkg,
				Reference: "v1.0.0",
			},
		},
	}
	g.lock = &glideLock{
		TestImports: []glideLockedPackage{
			{
				Name:      pkg,
				Reference: "ff2948a2ac8f538c4ecd55962e919d1e13e74baf",
			},
		},
	}

	manifest, lock, err := g.convert(testGlideProjectRoot)
	if err != nil {
		t.Fatal(err)
	}

	_, ok := manifest.Constraints[gps.ProjectRoot(pkg)]
	if !ok {
		t.Fatalf("Expected the manifest to have a dependency for %s but got none", pkg)
	}

	if len(lock.P) != 1 {
		t.Fatalf("Expected the lock to contain 1 project but got %d", len(lock.P))
	}
	p := lock.P[0]
	if p.Ident().ProjectRoot != gps.ProjectRoot(pkg) {
		t.Fatalf("Expected the lock to have a project for %s but got '%s'", pkg, p.Ident().ProjectRoot)
	}
}

func TestGlideConfig_Convert_Ignore(t *testing.T) {
	pkg := "github.com/sdboyer/deptest"

	g := newGlideImporter(discardLogger, true, nil)
	g.yaml = glideYaml{
		Ignores: []string{pkg},
	}

	manifest, _, err := g.convert(testGlideProjectRoot)
	if err != nil {
		t.Fatal(err)
	}

	if len(manifest.Ignored) != 1 {
		t.Fatalf("Expected the manifest to contain 1 ignored project but got %d", len(manifest.Ignored))
	}
	i := manifest.Ignored[0]
	if i != pkg {
		t.Fatalf("Expected the manifest to ignore %s but got '%s'", pkg, i)
	}
}

func TestGlideConfig_Convert_ExcludeDir(t *testing.T) {
	g := newGlideImporter(discardLogger, true, nil)
	g.yaml = glideYaml{
		ExcludeDirs: []string{"samples"},
	}

	manifest, _, err := g.convert(testGlideProjectRoot)
	if err != nil {
		t.Fatal(err)
	}

	if len(manifest.Ignored) != 1 {
		t.Fatalf("Expected the manifest to contain 1 ignored project but got %d", len(manifest.Ignored))
	}
	i := manifest.Ignored[0]
	if i != "github.com/golang/notexist/samples" {
		t.Fatalf("Expected the manifest to ignore 'github.com/golang/notexist/samples' but got '%s'", i)
	}
}

func TestGlideConfig_Convert_ExcludeDir_IgnoresMismatchedPackageName(t *testing.T) {
	g := newGlideImporter(discardLogger, true, nil)
	g.yaml = glideYaml{
		Name:        "github.com/golang/mismatched-package-name",
		ExcludeDirs: []string{"samples"},
	}

	manifest, _, err := g.convert(testGlideProjectRoot)
	if err != nil {
		t.Fatal(err)
	}

	if len(manifest.Ignored) != 1 {
		t.Fatalf("Expected the manifest to contain 1 ignored project but got %d", len(manifest.Ignored))
	}
	i := manifest.Ignored[0]
	if i != "github.com/golang/notexist/samples" {
		t.Fatalf("Expected the manifest to ignore 'github.com/golang/notexist/samples' but got '%s'", i)
	}
}

func TestGlideConfig_Convert_WarnsForUnusedFields(t *testing.T) {
	testCases := map[string]glidePackage{
		"specified an os":   {OS: "windows"},
		"specified an arch": {Arch: "i686"},
	}

	for wantWarning, pkg := range testCases {
		t.Run(wantWarning, func(t *testing.T) {
			pkg.Name = "github.com/sdboyer/deptest"
			pkg.Reference = "v1.0.0"

			// Capture stderr so we can verify warnings
			verboseOutput := &bytes.Buffer{}
			logger := log.New(verboseOutput, "", 0)
			g := newGlideImporter(logger, true, nil)
			g.yaml = glideYaml{
				Imports: []glidePackage{pkg},
			}

			_, _, err := g.convert(testGlideProjectRoot)
			if err != nil {
				t.Fatal(err)
			}

			warnings := verboseOutput.String()
			if !strings.Contains(warnings, wantWarning) {
				t.Errorf("Expected the output to include the warning '%s'", wantWarning)
			}
		})
	}
}

func TestGlideConfig_Convert_BadInput_EmptyPackageName(t *testing.T) {
	g := newGlideImporter(discardLogger, true, nil)
	g.yaml = glideYaml{
		Imports: []glidePackage{{Name: ""}},
	}

	_, _, err := g.convert(testGlideProjectRoot)
	if err == nil {
		t.Fatal("Expected conversion to fail because the package name is empty")
	}
}
<commit_msg>Always set a source manager in the glide tests<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"
	"testing"

	"github.com/golang/dep"
	"github.com/golang/dep/internal/gps"
	"github.com/golang/dep/internal/test"
	"github.com/pkg/errors"
)

const testGlideProjectRoot = "github.com/golang/notexist"

var (
	discardLogger  = log.New(ioutil.Discard, "", 0)
	discardLoggers = &dep.Loggers{Out: discardLogger, Err: discardLogger}
)

func newTestContext(h *test.Helper) *dep.Ctx {
	h.TempDir("src")
	pwd := h.Path(".")
	return &dep.Ctx{
		GOPATH:  pwd,
		Loggers: discardLoggers,
	}
}

func TestGlideConfig_Import(t *testing.T) {
	h := test.NewHelper(t)
	defer h.Cleanup()

	ctx := newTestContext(h)
	sm, err := ctx.SourceManager()
	h.Must(err)
	defer sm.Release()

	h.TempDir(filepath.Join("src", testGlideProjectRoot))
	h.TempCopy(filepath.Join(testGlideProjectRoot, glideYamlName), "glide/glide.yaml")
	h.TempCopy(filepath.Join(testGlideProjectRoot, glideLockName), "glide/glide.lock")
	projectRoot := h.Path(testGlideProjectRoot)

	// Capture stderr so we can verify output
	verboseOutput := &bytes.Buffer{}
	ctx.Err = log.New(verboseOutput, "", 0)

	g := newGlideImporter(ctx.Err, false, sm) // Disable verbose so that we don't print values that change each test run
	if !g.HasDepMetadata(projectRoot) {
		t.Fatal("Expected the importer to detect the glide configuration files")
	}

	m, l, err := g.Import(projectRoot, testGlideProjectRoot)
	h.Must(err)

	if m == nil {
		t.Fatal("Expected the manifest to be generated")
	}

	if l == nil {
		t.Fatal("Expected the lock to be generated")
	}

	goldenFile := "glide/expected_import_output.txt"
	got := verboseOutput.String()
	want := h.GetTestFileString(goldenFile)
	if want != got {
		if *test.UpdateGolden {
			if err := h.WriteTestFile(goldenFile, got); err != nil {
				t.Fatalf("%+v", errors.Wrapf(err, "Unable to write updated golden file %s", goldenFile))
			}
		} else {
			t.Fatalf("expected %s, got %s", want, got)
		}
	}
}

func TestGlideConfig_Import_MissingLockFile(t *testing.T) {
	h := test.NewHelper(t)
	defer h.Cleanup()

	ctx := newTestContext(h)
	sm, err := ctx.SourceManager()
	h.Must(err)
	defer sm.Release()

	h.TempDir(filepath.Join("src", "glidetest"))
	h.TempCopy(filepath.Join("glidetest", glideYamlName), "glide/glide.yaml")
	projectRoot := h.Path("glidetest")

	g := newGlideImporter(ctx.Err, true, sm)
	if !g.HasDepMetadata(projectRoot) {
		t.Fatal("The glide importer should gracefully handle when only glide.yaml is present")
	}

	m, l, err := g.Import(projectRoot, testGlideProjectRoot)
	h.Must(err)

	if m == nil {
		t.Fatal("Expected the manifest to be generated")
	}

	if l != nil {
		t.Fatal("Expected the lock to not be generated")
	}
}

func TestGlideConfig_Convert_Project(t *testing.T) {
	h := test.NewHelper(t)
	defer h.Cleanup()

	ctx := newTestContext(h)
	sm, err := ctx.SourceManager()
	h.Must(err)
	defer sm.Release()

	pkg := "github.com/sdboyer/deptest"
	repo := "https://github.com/sdboyer/deptest.git"

	g := newGlideImporter(ctx.Err, true, sm)
	g.yaml = glideYaml{
		Imports: []glidePackage{
			{
				Name:       pkg,
				Repository: repo,
				Reference:  "1.0.0",
			},
		},
	}
	g.lock = &glideLock{
		Imports: []glideLockedPackage{
			{
				Name:       pkg,
				Repository: repo,
				Reference:  "ff2948a2ac8f538c4ecd55962e919d1e13e74baf",
			},
		},
	}

	manifest, lock, err := g.convert(testGlideProjectRoot)
	if err != nil {
		t.Fatal(err)
	}

	d, ok := manifest.Constraints[gps.ProjectRoot(pkg)]
	if !ok {
		t.Fatal("Expected the manifest to have a dependency for 'github.com/sdboyer/deptest' but got none")
	}

	wantC := "^1.0.0"
	gotC := d.Constraint.String()
	if gotC != wantC {
		t.Fatalf("Expected manifest constraint to be %s, got %s", wantC, gotC)
	}

	gotS := d.Source
	if gotS != repo {
		t.Fatalf("Expected manifest source to be %s, got %s", repo, gotS)
	}

	wantP := 1
	gotP := len(lock.P)
	if gotP != 1 {
		t.Fatalf("Expected the lock to contain %d project but got %d", wantP, gotP)
	}

	p := lock.P[0]
	gotPr := string(p.Ident().ProjectRoot)
	if gotPr != pkg {
		t.Fatalf("Expected the lock to have a project for %s but got '%s'", pkg, gotPr)
	}

	gotS = p.Ident().Source
	if gotS != repo {
		t.Fatalf("Expected locked source to be %s, got '%s'", repo, gotS)
	}

	lv := p.Version()
	lpv, ok := lv.(gps.PairedVersion)
	if !ok {
		t.Fatalf("Expected locked version to be a PairedVersion but got %T", lv)
	}

	wantRev := "ff2948a2ac8f538c4ecd55962e919d1e13e74baf"
	gotRev := lpv.Underlying().String()
	if gotRev != wantRev {
		t.Fatalf("Expected locked revision to be %s, got %s", wantRev, gotRev)
	}

	wantV := "v1.0.0"
	gotV := lpv.String()
	if gotV != wantV {
		t.Fatalf("Expected locked version to be %s, got %s", wantV, gotV)
	}
}

func TestGlideConfig_Convert_TestProject(t *testing.T) {
	h := test.NewHelper(t)
	defer h.Cleanup()

	ctx := newTestContext(h)
	sm, err := ctx.SourceManager()
	h.Must(err)
	defer sm.Release()

	pkg := "github.com/sdboyer/deptest"

	g := newGlideImporter(ctx.Err, true, sm)
	g.yaml = glideYaml{
		TestImports: []glidePackage{
			{
				Name:      pkg,
				Reference: "v1.0.0",
			},
		},
	}
	g.lock = &glideLock{
		TestImports: []glideLockedPackage{
			{
				Name:      pkg,
				Reference: "ff2948a2ac8f538c4ecd55962e919d1e13e74baf",
			},
		},
	}

	manifest, lock, err := g.convert(testGlideProjectRoot)
	if err != nil {
		t.Fatal(err)
	}

	_, ok := manifest.Constraints[gps.ProjectRoot(pkg)]
	if !ok {
		t.Fatalf("Expected the manifest to have a dependency for %s but got none", pkg)
	}

	if len(lock.P) != 1 {
		t.Fatalf("Expected the lock to contain 1 project but got %d", len(lock.P))
	}
	p := lock.P[0]
	if p.Ident().ProjectRoot != gps.ProjectRoot(pkg) {
		t.Fatalf("Expected the lock to have a project for %s but got '%s'", pkg, p.Ident().ProjectRoot)
	}
}

func TestGlideConfig_Convert_Ignore(t *testing.T) {
	h := test.NewHelper(t)
	defer h.Cleanup()

	ctx := newTestContext(h)
	sm, err := ctx.SourceManager()
	h.Must(err)
	defer sm.Release()

	pkg := "github.com/sdboyer/deptest"

	g := newGlideImporter(ctx.Err, true, sm)
	g.yaml = glideYaml{
		Ignores: []string{pkg},
	}

	manifest, _, err := g.convert(testGlideProjectRoot)
	if err != nil {
		t.Fatal(err)
	}

	if len(manifest.Ignored) != 1 {
		t.Fatalf("Expected the manifest to contain 1 ignored project but got %d", len(manifest.Ignored))
	}
	i := manifest.Ignored[0]
	if i != pkg {
		t.Fatalf("Expected the manifest to ignore %s but got '%s'", pkg, i)
	}
}

func TestGlideConfig_Convert_ExcludeDir(t *testing.T) {
	h := test.NewHelper(t)
	defer h.Cleanup()

	ctx := newTestContext(h)
	sm, err := ctx.SourceManager()
	h.Must(err)
	defer sm.Release()

	g := newGlideImporter(ctx.Err, true, sm)
	g.yaml = glideYaml{
		ExcludeDirs: []string{"samples"},
	}

	manifest, _, err := g.convert(testGlideProjectRoot)
	if err != nil {
		t.Fatal(err)
	}

	if len(manifest.Ignored) != 1 {
		t.Fatalf("Expected the manifest to contain 1 ignored project but got %d", len(manifest.Ignored))
	}
	i := manifest.Ignored[0]
	if i != "github.com/golang/notexist/samples" {
		t.Fatalf("Expected the manifest to ignore 'github.com/golang/notexist/samples' but got '%s'", i)
	}
}

func TestGlideConfig_Convert_ExcludeDir_IgnoresMismatchedPackageName(t *testing.T) {
	h := test.NewHelper(t)
	defer h.Cleanup()

	ctx := newTestContext(h)
	sm, err := ctx.SourceManager()
	h.Must(err)
	defer sm.Release()

	g := newGlideImporter(ctx.Err, true, sm)
	g.yaml = glideYaml{
		Name:        "github.com/golang/mismatched-package-name",
		ExcludeDirs: []string{"samples"},
	}

	manifest, _, err := g.convert(testGlideProjectRoot)
	if err != nil {
		t.Fatal(err)
	}

	if len(manifest.Ignored) != 1 {
		t.Fatalf("Expected the manifest to contain 1 ignored project but got %d", len(manifest.Ignored))
	}
	i := manifest.Ignored[0]
	if i != "github.com/golang/notexist/samples" {
		t.Fatalf("Expected the manifest to ignore 'github.com/golang/notexist/samples' but got '%s'", i)
	}
}

func TestGlideConfig_Convert_WarnsForUnusedFields(t *testing.T) {
	testCases := map[string]glidePackage{
		"specified an os":   {OS: "windows"},
		"specified an arch": {Arch: "i686"},
	}

	for wantWarning, pkg := range testCases {
		t.Run(wantWarning, func(t *testing.T) {
			h := test.NewHelper(t)
			defer h.Cleanup()

			pkg.Name = "github.com/sdboyer/deptest"
			pkg.Reference = "v1.0.0"

			ctx := newTestContext(h)
			sm, err := ctx.SourceManager()
			h.Must(err)
			defer sm.Release()

			// Capture stderr so we can verify warnings
			verboseOutput := &bytes.Buffer{}
			ctx.Err = log.New(verboseOutput, "", 0)

			g := newGlideImporter(ctx.Err, true, sm)
			g.yaml = glideYaml{
				Imports: []glidePackage{pkg},
			}

			_, _, err = g.convert(testGlideProjectRoot)
			if err != nil {
				t.Fatal(err)
			}

			warnings := verboseOutput.String()
			if !strings.Contains(warnings, wantWarning) {
				t.Errorf("Expected the output to include the warning '%s'", wantWarning)
			}
		})
	}
}

func TestGlideConfig_Convert_BadInput_EmptyPackageName(t *testing.T) {
	h := test.NewHelper(t)
	defer h.Cleanup()

	ctx := newTestContext(h)
	sm, err := ctx.SourceManager()
	h.Must(err)
	defer sm.Release()

	g := newGlideImporter(ctx.Err, true, sm)
	g.yaml = glideYaml{
		Imports: []glidePackage{{Name: ""}},
	}

	_, _, err = g.convert(testGlideProjectRoot)
	if err == nil {
		t.Fatal("Expected conversion to fail because the package name is empty")
	}
}
<|endoftext|>