<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"fmt"
	"time"

	federation_release_1_3 "k8s.io/kubernetes/federation/client/clientset_generated/federation_release_1_3"
	"k8s.io/kubernetes/pkg/api/errors"
	v1 "k8s.io/kubernetes/pkg/api/v1"
	cache "k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/controller"

	"github.com/golang/glog"
	"reflect"
	"sort"
)

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (sc *ServiceController) clusterServiceWorker() {
	fedClient := sc.federationClient
	for clusterName, cache := range sc.clusterCache.clientMap {
		go func(cache *clusterCache, clusterName string) {
			for {
				func() {
					key, quit := cache.serviceQueue.Get()
					defer cache.serviceQueue.Done(key)
					if quit {
						return
					}
					err := sc.clusterCache.syncService(key.(string), clusterName, cache, sc.serviceCache, fedClient, sc)
					if err != nil {
						glog.Errorf("Failed to sync service: %+v", err)
					}
				}()
			}
		}(cache, clusterName)
	}
}

// Whenever there is change on service, the federation service should be updated
func (cc *clusterClientCache) syncService(key, clusterName string, clusterCache *clusterCache, serviceCache *serviceCache, fedClient federation_release_1_3.Interface, sc *ServiceController) error {
	// obj holds the latest service info from apiserver, return if there is no federation cache for the service
	cachedService, ok := serviceCache.get(key)
	if !ok {
		// if serviceCache does not exists, that means the service is not created by federation, we should skip it
		return nil
	}
	serviceInterface, exists, err := clusterCache.serviceStore.GetByKey(key)
	if err != nil {
		glog.Infof("Did not successfully get %v from store: %v, will retry later", key, err)
		clusterCache.serviceQueue.Add(key)
		return err
	}
	var needUpdate bool
	if exists {
		service, ok := serviceInterface.(*v1.Service)
		if ok {
			glog.V(4).Infof("Found service for federation service %s/%s from cluster %s", service.Namespace, service.Name, clusterName)
			needUpdate = cc.processServiceUpdate(cachedService, service, clusterName)
		} else {
			_, ok := serviceInterface.(cache.DeletedFinalStateUnknown)
			if !ok {
				return fmt.Errorf("Object contained wasn't a service or a deleted key: %+v", serviceInterface)
			}
			glog.Infof("Found tombstone for %v", key)
			needUpdate = cc.processServiceDeletion(cachedService, clusterName)
		}
	} else {
		glog.Infof("Can not get service %v for cluster %s from serviceStore", key, clusterName)
		needUpdate = cc.processServiceDeletion(cachedService, clusterName)
	}

	if needUpdate {
		for i := 0; i < clientRetryCount; i++ {
			if err := sc.ensureDnsRecords(clusterName, cachedService); err == nil {
				break
			}
			glog.V(4).Infof("Error ensuring DNS Records for service %s on cluster %s: %v", key, clusterName, err)
			time.Sleep(cachedService.nextDNSUpdateDelay())
			clusterCache.serviceQueue.Add(key)
			// did not retry here as we still want to persist federation apiserver even ensure dns records fails
		}
		err := cc.persistFedServiceUpdate(cachedService, fedClient)
		if err == nil {
			cachedService.appliedState = cachedService.lastState
			cachedService.resetFedUpdateDelay()
		} else {
			if err != nil {
				glog.Errorf("Failed to sync service: %+v, put back to service queue", err)
				clusterCache.serviceQueue.Add(key)
			}
		}
	}
	return nil
}

// processServiceDeletion is triggered when a service is delete from underlying k8s cluster
// the deletion function will wip out the cached ingress info of the service from federation service ingress
// the function returns a bool to indicate if actual update happened on federation service cache
// and if the federation service cache is updated, the updated info should be post to federation apiserver
func (cc *clusterClientCache) processServiceDeletion(cachedService *cachedService, clusterName string) bool {
	cachedService.rwlock.Lock()
	defer cachedService.rwlock.Unlock()
	cachedStatus, ok := cachedService.serviceStatusMap[clusterName]
	// cached status found, remove ingress info from federation service cache
	if ok {
		cachedFedServiceStatus := cachedService.lastState.Status.LoadBalancer
		removeIndexes := []int{}
		for i, fed := range cachedFedServiceStatus.Ingress {
			for _, new := range cachedStatus.Ingress {
				// remove if same ingress record found
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					removeIndexes = append(removeIndexes, i)
				}
			}
		}
		sort.Ints(removeIndexes)
		for i := len(removeIndexes) - 1; i >= 0; i-- {
			cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress[:removeIndexes[i]], cachedFedServiceStatus.Ingress[removeIndexes[i]+1:]...)
			glog.V(4).Infof("Remove old ingress %d for service %s/%s", removeIndexes[i], cachedService.lastState.Namespace, cachedService.lastState.Name)
		}
		delete(cachedService.serviceStatusMap, clusterName)
		delete(cachedService.endpointMap, clusterName)
		cachedService.lastState.Status.LoadBalancer = cachedFedServiceStatus
		return true
	} else {
		glog.V(4).Infof("Service removal %s/%s from cluster %s observed.", cachedService.lastState.Namespace, cachedService.lastState.Name, clusterName)
	}
	return false
}

// processServiceUpdate Update ingress info when service updated
// the function returns a bool to indicate if actual update happened on federation service cache
// and if the federation service cache is updated, the updated info should be post to federation apiserver
func (cc *clusterClientCache) processServiceUpdate(cachedService *cachedService, service *v1.Service, clusterName string) bool {
	glog.V(4).Infof("Processing service update for %s/%s, cluster %s", service.Namespace, service.Name, clusterName)
	cachedService.rwlock.Lock()
	defer cachedService.rwlock.Unlock()
	var needUpdate bool
	newServiceLB := service.Status.LoadBalancer
	cachedFedServiceStatus := cachedService.lastState.Status.LoadBalancer
	if len(newServiceLB.Ingress) == 0 {
		// not yet get LB IP
		return false
	}

	cachedStatus, ok := cachedService.serviceStatusMap[clusterName]
	if ok {
		if reflect.DeepEqual(cachedStatus, newServiceLB) {
			glog.V(4).Infof("Same ingress info observed for service %s/%s: %+v ", service.Namespace, service.Name, cachedStatus.Ingress)
		} else {
			glog.V(4).Infof("Ingress info was changed for service %s/%s: cache: %+v, new: %+v ",
				service.Namespace, service.Name, cachedStatus.Ingress, newServiceLB)
			needUpdate = true
		}
	} else {
		glog.V(4).Infof("Cached service status was not found for %s/%s, cluster %s, building one", service.Namespace, service.Name, clusterName)

		// cache is not always reliable(cache will be cleaned when service controller restart)
		// two cases will run into this branch:
		// 1. new service loadbalancer info received -> no info in cache, and no in federation service
		// 2. service controller being restarted -> no info in cache, but it is in federation service

		// check if the lb info is already in federation service

		cachedService.serviceStatusMap[clusterName] = newServiceLB
		needUpdate = false
		// iterate service ingress info
		for _, new := range newServiceLB.Ingress {
			var found bool
			// if it is known by federation service
			for _, fed := range cachedFedServiceStatus.Ingress {
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					found = true
				}
			}
			if !found {
				needUpdate = true
				break
			}
		}
	}

	if needUpdate {
		// new status = cached federation status - cached status + new status from k8s cluster

		removeIndexes := []int{}
		for i, fed := range cachedFedServiceStatus.Ingress {
			for _, new := range cachedStatus.Ingress {
				// remove if same ingress record found
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					removeIndexes = append(removeIndexes, i)
				}
			}
		}
		sort.Ints(removeIndexes)
		for i := len(removeIndexes) - 1; i >= 0; i-- {
			cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress[:removeIndexes[i]], cachedFedServiceStatus.Ingress[removeIndexes[i]+1:]...)
		}
		cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress, service.Status.LoadBalancer.Ingress...)
		cachedService.lastState.Status.LoadBalancer = cachedFedServiceStatus
		glog.V(4).Infof("Add new ingress info %+v for service %s/%s", service.Status.LoadBalancer, service.Namespace, service.Name)
	} else {
		glog.V(4).Infof("Same ingress info found for %s/%s, cluster %s", service.Namespace, service.Name, clusterName)
	}
	return needUpdate
}

func (cc *clusterClientCache) persistFedServiceUpdate(cachedService *cachedService, fedClient federation_release_1_3.Interface) error {
	service := cachedService.lastState
	glog.V(5).Infof("Persist federation service status %s/%s", service.Namespace, service.Name)
	var err error
	for i := 0; i < clientRetryCount; i++ {
		_, err := fedClient.Core().Services(service.Namespace).Get(service.Name)
		if errors.IsNotFound(err) {
			glog.Infof("Not persisting update to service '%s/%s' that no longer exists: %v",
				service.Namespace, service.Name, err)
			return nil
		}
		_, err = fedClient.Core().Services(service.Namespace).UpdateStatus(service)
		if err == nil {
			glog.V(2).Infof("Successfully update service %s/%s to federation apiserver", service.Namespace, service.Name)
			return nil
		}
		if errors.IsNotFound(err) {
			glog.Infof("Not persisting update to service '%s/%s' that no longer exists: %v",
				service.Namespace, service.Name, err)
			return nil
		}
		if errors.IsConflict(err) {
			glog.V(4).Infof("Not persisting update to service '%s/%s' that has been changed since we received it: %v",
				service.Namespace, service.Name, err)
			return err
		}
		time.Sleep(cachedService.nextFedUpdateDelay())
	}
	return err
}

// obj could be an *api.Service, or a DeletionFinalStateUnknown marker item.
func (cc *clusterClientCache) enqueueService(obj interface{}, clusterName string) {
	key, err := controller.KeyFunc(obj)
	if err != nil {
		glog.Errorf("Couldn't get key for object %+v: %v", obj, err)
		return
	}
	_, ok := cc.clientMap[clusterName]
	if ok {
		cc.clientMap[clusterName].serviceQueue.Add(key)
	}
}
<commit_msg>Fix block-local err bug in service_helper.go<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"fmt"
	"time"

	federation_release_1_3 "k8s.io/kubernetes/federation/client/clientset_generated/federation_release_1_3"
	"k8s.io/kubernetes/pkg/api/errors"
	v1 "k8s.io/kubernetes/pkg/api/v1"
	cache "k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/controller"

	"github.com/golang/glog"
	"reflect"
	"sort"
)

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (sc *ServiceController) clusterServiceWorker() {
	fedClient := sc.federationClient
	for clusterName, cache := range sc.clusterCache.clientMap {
		go func(cache *clusterCache, clusterName string) {
			for {
				func() {
					key, quit := cache.serviceQueue.Get()
					defer cache.serviceQueue.Done(key)
					if quit {
						return
					}
					err := sc.clusterCache.syncService(key.(string), clusterName, cache, sc.serviceCache, fedClient, sc)
					if err != nil {
						glog.Errorf("Failed to sync service: %+v", err)
					}
				}()
			}
		}(cache, clusterName)
	}
}

// Whenever there is change on service, the federation service should be updated
func (cc *clusterClientCache) syncService(key, clusterName string, clusterCache *clusterCache, serviceCache *serviceCache, fedClient federation_release_1_3.Interface, sc *ServiceController) error {
	// obj holds the latest service info from apiserver, return if there is no federation cache for the service
	cachedService, ok := serviceCache.get(key)
	if !ok {
		// if serviceCache does not exists, that means the service is not created by federation, we should skip it
		return nil
	}
	serviceInterface, exists, err := clusterCache.serviceStore.GetByKey(key)
	if err != nil {
		glog.Infof("Did not successfully get %v from store: %v, will retry later", key, err)
		clusterCache.serviceQueue.Add(key)
		return err
	}
	var needUpdate bool
	if exists {
		service, ok := serviceInterface.(*v1.Service)
		if ok {
			glog.V(4).Infof("Found service for federation service %s/%s from cluster %s", service.Namespace, service.Name, clusterName)
			needUpdate = cc.processServiceUpdate(cachedService, service, clusterName)
		} else {
			_, ok := serviceInterface.(cache.DeletedFinalStateUnknown)
			if !ok {
				return fmt.Errorf("Object contained wasn't a service or a deleted key: %+v", serviceInterface)
			}
			glog.Infof("Found tombstone for %v", key)
			needUpdate = cc.processServiceDeletion(cachedService, clusterName)
		}
	} else {
		glog.Infof("Can not get service %v for cluster %s from serviceStore", key, clusterName)
		needUpdate = cc.processServiceDeletion(cachedService, clusterName)
	}

	if needUpdate {
		for i := 0; i < clientRetryCount; i++ {
			err := sc.ensureDnsRecords(clusterName, cachedService)
			if err == nil {
				break
			}
			glog.V(4).Infof("Error ensuring DNS Records for service %s on cluster %s: %v", key, clusterName, err)
			time.Sleep(cachedService.nextDNSUpdateDelay())
			clusterCache.serviceQueue.Add(key)
			// did not retry here as we still want to persist federation apiserver even ensure dns records fails
		}
		err := cc.persistFedServiceUpdate(cachedService, fedClient)
		if err == nil {
			cachedService.appliedState = cachedService.lastState
			cachedService.resetFedUpdateDelay()
		} else {
			if err != nil {
				glog.Errorf("Failed to sync service: %+v, put back to service queue", err)
				clusterCache.serviceQueue.Add(key)
			}
		}
	}
	return nil
}

// processServiceDeletion is triggered when a service is delete from underlying k8s cluster
// the deletion function will wip out the cached ingress info of the service from federation service ingress
// the function returns a bool to indicate if actual update happened on federation service cache
// and if the federation service cache is updated, the updated info should be post to federation apiserver
func (cc *clusterClientCache) processServiceDeletion(cachedService *cachedService, clusterName string) bool {
	cachedService.rwlock.Lock()
	defer cachedService.rwlock.Unlock()
	cachedStatus, ok := cachedService.serviceStatusMap[clusterName]
	// cached status found, remove ingress info from federation service cache
	if ok {
		cachedFedServiceStatus := cachedService.lastState.Status.LoadBalancer
		removeIndexes := []int{}
		for i, fed := range cachedFedServiceStatus.Ingress {
			for _, new := range cachedStatus.Ingress {
				// remove if same ingress record found
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					removeIndexes = append(removeIndexes, i)
				}
			}
		}
		sort.Ints(removeIndexes)
		for i := len(removeIndexes) - 1; i >= 0; i-- {
			cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress[:removeIndexes[i]], cachedFedServiceStatus.Ingress[removeIndexes[i]+1:]...)
			glog.V(4).Infof("Remove old ingress %d for service %s/%s", removeIndexes[i], cachedService.lastState.Namespace, cachedService.lastState.Name)
		}
		delete(cachedService.serviceStatusMap, clusterName)
		delete(cachedService.endpointMap, clusterName)
		cachedService.lastState.Status.LoadBalancer = cachedFedServiceStatus
		return true
	} else {
		glog.V(4).Infof("Service removal %s/%s from cluster %s observed.", cachedService.lastState.Namespace, cachedService.lastState.Name, clusterName)
	}
	return false
}

// processServiceUpdate Update ingress info when service updated
// the function returns a bool to indicate if actual update happened on federation service cache
// and if the federation service cache is updated, the updated info should be post to federation apiserver
func (cc *clusterClientCache) processServiceUpdate(cachedService *cachedService, service *v1.Service, clusterName string) bool {
	glog.V(4).Infof("Processing service update for %s/%s, cluster %s", service.Namespace, service.Name, clusterName)
	cachedService.rwlock.Lock()
	defer cachedService.rwlock.Unlock()
	var needUpdate bool
	newServiceLB := service.Status.LoadBalancer
	cachedFedServiceStatus := cachedService.lastState.Status.LoadBalancer
	if len(newServiceLB.Ingress) == 0 {
		// not yet get LB IP
		return false
	}

	cachedStatus, ok := cachedService.serviceStatusMap[clusterName]
	if ok {
		if reflect.DeepEqual(cachedStatus, newServiceLB) {
			glog.V(4).Infof("Same ingress info observed for service %s/%s: %+v ", service.Namespace, service.Name, cachedStatus.Ingress)
		} else {
			glog.V(4).Infof("Ingress info was changed for service %s/%s: cache: %+v, new: %+v ",
				service.Namespace, service.Name, cachedStatus.Ingress, newServiceLB)
			needUpdate = true
		}
	} else {
		glog.V(4).Infof("Cached service status was not found for %s/%s, cluster %s, building one", service.Namespace, service.Name, clusterName)

		// cache is not always reliable(cache will be cleaned when service controller restart)
		// two cases will run into this branch:
		// 1. new service loadbalancer info received -> no info in cache, and no in federation service
		// 2. service controller being restarted -> no info in cache, but it is in federation service

		// check if the lb info is already in federation service

		cachedService.serviceStatusMap[clusterName] = newServiceLB
		needUpdate = false
		// iterate service ingress info
		for _, new := range newServiceLB.Ingress {
			var found bool
			// if it is known by federation service
			for _, fed := range cachedFedServiceStatus.Ingress {
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					found = true
				}
			}
			if !found {
				needUpdate = true
				break
			}
		}
	}

	if needUpdate {
		// new status = cached federation status - cached status + new status from k8s cluster

		removeIndexes := []int{}
		for i, fed := range cachedFedServiceStatus.Ingress {
			for _, new := range cachedStatus.Ingress {
				// remove if same ingress record found
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					removeIndexes = append(removeIndexes, i)
				}
			}
		}
		sort.Ints(removeIndexes)
		for i := len(removeIndexes) - 1; i >= 0; i-- {
			cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress[:removeIndexes[i]], cachedFedServiceStatus.Ingress[removeIndexes[i]+1:]...)
		}
		cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress, service.Status.LoadBalancer.Ingress...)
		cachedService.lastState.Status.LoadBalancer = cachedFedServiceStatus
		glog.V(4).Infof("Add new ingress info %+v for service %s/%s", service.Status.LoadBalancer, service.Namespace, service.Name)
	} else {
		glog.V(4).Infof("Same ingress info found for %s/%s, cluster %s", service.Namespace, service.Name, clusterName)
	}
	return needUpdate
}

func (cc *clusterClientCache) persistFedServiceUpdate(cachedService *cachedService, fedClient federation_release_1_3.Interface) error {
	service := cachedService.lastState
	glog.V(5).Infof("Persist federation service status %s/%s", service.Namespace, service.Name)
	var err error
	for i := 0; i < clientRetryCount; i++ {
		_, err := fedClient.Core().Services(service.Namespace).Get(service.Name)
		if errors.IsNotFound(err) {
			glog.Infof("Not persisting update to service '%s/%s' that no longer exists: %v",
				service.Namespace, service.Name, err)
			return nil
		}
		_, err = fedClient.Core().Services(service.Namespace).UpdateStatus(service)
		if err == nil {
			glog.V(2).Infof("Successfully update service %s/%s to federation apiserver", service.Namespace, service.Name)
			return nil
		}
		if errors.IsNotFound(err) {
			glog.Infof("Not persisting update to service '%s/%s' that no longer exists: %v",
				service.Namespace, service.Name, err)
			return nil
		}
		if errors.IsConflict(err) {
			glog.V(4).Infof("Not persisting update to service '%s/%s' that has been changed since we received it: %v",
				service.Namespace, service.Name, err)
			return err
		}
		time.Sleep(cachedService.nextFedUpdateDelay())
	}
	return err
}

// obj could be an *api.Service, or a DeletionFinalStateUnknown marker item.
func (cc *clusterClientCache) enqueueService(obj interface{}, clusterName string) {
	key, err := controller.KeyFunc(obj)
	if err != nil {
		glog.Errorf("Couldn't get key for object %+v: %v", obj, err)
		return
	}
	_, ok := cc.clientMap[clusterName]
	if ok {
		cc.clientMap[clusterName].serviceQueue.Add(key)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 - 2017 Huawei Technologies Co., Ltd. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package module

import (
	"fmt"
	"io"
	"time"

	"github.com/Huawei/containerops/common"
	"github.com/Huawei/containerops/singular/model"
	"github.com/Huawei/containerops/singular/module/infras"
	"github.com/Huawei/containerops/singular/module/objects"
	"github.com/Huawei/containerops/singular/module/service"
	"github.com/Huawei/containerops/singular/module/tools"
)

const (
	//InfraEtcd is Etcd flag
	InfraEtcd = "etcd"
	//InfraFlannel is Flannel flg
	InfraFlannel = "flannel"
	//InfraDocker is Docker flag
	InfraDocker = "docker"
	//InfraKubernetes is Kubernetes flag
	InfraKubernetes = "kubernetes"
)

//DeployInfraStacks deploy all infras in the cluster.
//Deploy Sequence:
//   Preparing SSH Key files ->
//   Preparing VM ->
//   Preparing SSL root Key files ->
//   Deploy Etcd ->
//   Deploy flannel ->
//   Deploy k8s Master ->
//   Deploy k8s node ->
//Parameters:
//   db [bool] Save Deploy model and log in the database.
func DeployInfraStacks(d *objects.Deployment, db bool, stdout io.Writer, timestamp bool) error {
	var err error

	//Open Database Connection and migrate table struct.
	if db == true {
		objects.WriteLog("Open database connection and migrate the table struct", stdout, timestamp, d)
		if err := model.OpenDatabase(&common.Database); err != nil {
			return err
		}

		if err := model.Migrate(); err != nil {
			return err
		}
	}

	//Preparing SSH Keys
	if d.Tools.SSH.Private != "" {
		objects.WriteLog(
			fmt.Sprintf("Calculate the public key and fingerprint using the private key file [%s]", d.Tools.SSH.Private),
			stdout, timestamp, d)
		if d.Tools.SSH.Public, d.Tools.SSH.Private, d.Tools.SSH.Fingerprint, err = tools.OpenSSHKeyFiles(d.Tools.SSH.Public, d.Tools.SSH.Private); err != nil {
			return err
		}
	} else {
		if d.Tools.SSH.Public, d.Tools.SSH.Private, d.Tools.SSH.Fingerprint, err = tools.GenerateSSHKeyFiles(d.Config); err != nil {
			return err
		}
		objects.WriteLog(fmt.Sprintf("Generate the SSH keys file [%s] for deploy cloud native stack", d.Tools.SSH.Private),
			stdout, timestamp, d)
	}

	//Preparing VM from service provider like DigitalOcean, GCE, AWS, Azure ...
	if d.Service.Provider != "" {
		switch d.Service.Provider {
		case "digitalocean":
			do := service.DigitalOcean{}
			do.Token = d.Service.Token
			do.Region, do.Size, do.Image = d.Service.Region, d.Service.Size, d.Service.Image
			//TODO parse do.Image get distro
			//Init DigitalOcean API client.
			objects.WriteLog("Create DigitalOcean droplets", stdout, timestamp, d, &do)

			objects.WriteLog("Init DigitalOcean client with token", stdout, timestamp, d, &do)
			do.InitClient()

			objects.WriteLog(
				fmt.Sprintf("Upload ssh public key file [%s] to DigitalOcean", d.Tools.SSH.Public),
				stdout, timestamp, d, &do)
			//Upload ssh public key
			if err := do.UploadSSHKey(d.Tools.SSH.Public); err != nil {
				return err
			}

			//Prepare droplet prefix name and tags
			namespace, repository, _, _ := d.URIs()
			tags := []string{namespace, repository, d.Tag}
			objects.WriteLog(fmt.Sprintf("Droplets tag is %v", tags), stdout, timestamp, d, &do)

			//Create DigitalOcean Droplets
			if err := do.CreateDroplets(d.Service.Nodes, d.Tools.SSH.Fingerprint, fmt.Sprintf("%s-%s", namespace, repository), tags); err != nil {
				return err
			}

			//Export droplets IP
			for id, droplet := range do.Droplets {
				node := objects.Node{
					ID:      id,
					IP:      droplet["public"],
					Private: droplet["private"],
					User:    service.DORootUser,
					Distro:  tools.DistroUbuntu,
				}

				objects.WriteLog(fmt.Sprintf("Droplet [%d] ip is [%s/%s]", id, droplet["public"], droplet["private"]), stdout, timestamp, d, &do)
				d.Nodes = append(d.Nodes, &node)
			}

			objects.WriteLog("sleep 60 second for preparing Droplets", stdout, timestamp, d, &do)
			time.Sleep(60 * time.Second)

			d.Service.Logs = do.Logs
		default:
			return fmt.Errorf("unsupport service provide: %s", d.Service.Provider)
		}
	}

	if len(d.Nodes) > 0 {
		//Set Out Node Parameters
		i := 0
		for _, node := range d.Nodes {
			d.Output(fmt.Sprintf("NODE_%d", i), node.IP)
			i += 1

			//Export node ip to stdout
			objects.WriteLog(fmt.Sprintf("[COUT] NODE_%d = %s", i, node.IP), stdout, timestamp, d)
		}

		//Initialization node environment
		for _, node := range d.Nodes {
			objects.WriteLog(fmt.Sprintf("Initializate the %s node environment", node.IP), stdout, timestamp, d, node)
			if commands, err := tools.InitializationEnvironment(d.Tools.SSH.Private, node.IP, node.User, node.Distro, stdout); err != nil {
				return err
			} else {
				objects.WriteLog(fmt.Sprintf("execute commands %v in the %d node", commands, node.ID), stdout, timestamp, d, node)
			}
		}

		//Generate CA CARootTemplate files
		if roots, err := tools.GenerateCARootFiles(d.Config); err != nil {
			return err
		} else {
			objects.WriteLog(fmt.Sprintf("Generate cluster root CA files."), stdout, timestamp, d)
			for key, value := range roots {
				d.Output(key, value)
				objects.WriteLog(fmt.Sprintf("[COUT] %s = %s", key, value), stdout, timestamp, d)
			}

			//Upload CA root files to the nodes.
			for _, node := range d.Nodes {
				objects.WriteLog(fmt.Sprintf("Upload root CA files %v to %s node", roots, node.IP), stdout, timestamp, d, node)
				if err := tools.UploadCARootFiles(d.Tools.SSH.Private, roots, node.IP, node.User, stdout); err != nil {
					return err
				}
			}
		}

		//Deploy infras
		for _, infra := range d.Infras {
			switch infra.Name {
			case InfraEtcd:
				objects.WriteLog(fmt.Sprintf("Deploy etcd in cluster"), stdout, timestamp, d, infra)
				if err := infras.DeployEtcdInCluster(d, infra, stdout, timestamp); err != nil {
					return err
				}
			case InfraFlannel:
				objects.WriteLog(fmt.Sprintf("Deploy flannel in cluster"), stdout, timestamp, d, infra)
				if err := infras.DeployFlannelInCluster(d, infra, stdout, timestamp); err != nil {
					return err
				}
			case InfraDocker:
				objects.WriteLog(fmt.Sprintf("Deploy docker in cluster"), stdout, timestamp, d, infra)
				if err := infras.DeployDockerInCluster(d, infra, stdout, timestamp); err != nil {
					return err
				}
			case InfraKubernetes:
				objects.WriteLog(fmt.Sprintf("Deploy k8s in cluster"), stdout, timestamp, d, infra)
				if err := infras.DeployKubernetesInCluster(d, infra, stdout, timestamp); err != nil {
					return err
				}
			default:
				return fmt.Errorf("unsupport infrastruction software: %s", infra)
			}
		}

	}

	//Save data
	if db == true {
		if err := d.Save(); err != nil {
			return err
		}
	}

	return err
}

//DeleteInfraStacks delete the droplets of stack.
func DeleteInfraStacks(d *objects.Deployment, stdout io.Writer, timestamp bool) error {
	if d.Service.Provider != "" {
		switch d.Service.Provider {
		case "digitalocean":
			do := service.DigitalOcean{}
			do.Token = d.Service.Token
			do.InitClient()

			for _, node := range d.Nodes {
				if err := do.DeleteDroplet(node.ID); err != nil {
					return err
				}
				objects.WriteLog(fmt.Sprintf("Delete %d droplet in DigitalOcaen", node.ID), stdout, timestamp, d)
			}

		default:
			return fmt.Errorf("unsupport service provide: %s", d.Service.Provider)
		}
	}

	return nil
}
<commit_msg>Test digital ocean droplets' SSH port to get the ready status.<commit_after>/*
Copyright 2016 - 2017 Huawei Technologies Co., Ltd. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package module

import (
	"fmt"
	"io"

	"github.com/Huawei/containerops/common"
	"github.com/Huawei/containerops/common/utils"
	"github.com/Huawei/containerops/singular/model"
	"github.com/Huawei/containerops/singular/module/infras"
	"github.com/Huawei/containerops/singular/module/objects"
	"github.com/Huawei/containerops/singular/module/service"
	"github.com/Huawei/containerops/singular/module/tools"
)

const (
	//InfraEtcd is Etcd flag
	InfraEtcd = "etcd"
	//InfraFlannel is Flannel flg
	InfraFlannel = "flannel"
	//InfraDocker is Docker flag
	InfraDocker = "docker"
	//InfraKubernetes is Kubernetes flag
	InfraKubernetes = "kubernetes"
)

//DeployInfraStacks deploy all infras in the cluster.
//Deploy Sequence:
//   Preparing SSH Key files ->
//   Preparing VM ->
//   Preparing SSL root Key files ->
//   Deploy Etcd ->
//   Deploy flannel ->
//   Deploy k8s Master ->
//   Deploy k8s node ->
//Parameters:
//   db [bool] Save Deploy model and log in the database.
func DeployInfraStacks(d *objects.Deployment, db bool, stdout io.Writer, timestamp bool) error {
	var err error

	//Open Database Connection and migrate table struct.
	if db == true {
		objects.WriteLog("Open database connection and migrate the table struct", stdout, timestamp, d)
		if err := model.OpenDatabase(&common.Database); err != nil {
			return err
		}

		if err := model.Migrate(); err != nil {
			return err
		}
	}

	//Preparing SSH Keys
	if d.Tools.SSH.Private != "" {
		objects.WriteLog(
			fmt.Sprintf("Calculate the public key and fingerprint using the private key file [%s]", d.Tools.SSH.Private),
			stdout, timestamp, d)
		if d.Tools.SSH.Public, d.Tools.SSH.Private, d.Tools.SSH.Fingerprint, err = tools.OpenSSHKeyFiles(d.Tools.SSH.Public, d.Tools.SSH.Private); err != nil {
			return err
		}
	} else {
		if d.Tools.SSH.Public, d.Tools.SSH.Private, d.Tools.SSH.Fingerprint, err = tools.GenerateSSHKeyFiles(d.Config); err != nil {
			return err
		}
		objects.WriteLog(fmt.Sprintf("Generate the SSH keys file [%s] for deploy cloud native stack", d.Tools.SSH.Private),
			stdout, timestamp, d)
	}

	//Preparing VM from service provider like DigitalOcean, GCE, AWS, Azure ...
	if d.Service.Provider != "" {
		switch d.Service.Provider {
		case "digitalocean":
			do := service.DigitalOcean{}
			do.Token = d.Service.Token
			do.Region, do.Size, do.Image = d.Service.Region, d.Service.Size, d.Service.Image
			//TODO parse do.Image get distro
			//Init DigitalOcean API client.
			objects.WriteLog("Create DigitalOcean droplets", stdout, timestamp, d, &do)

			objects.WriteLog("Init DigitalOcean client with token", stdout, timestamp, d, &do)
			do.InitClient()

			objects.WriteLog(
				fmt.Sprintf("Upload ssh public key file [%s] to DigitalOcean", d.Tools.SSH.Public),
				stdout, timestamp, d, &do)
			//Upload ssh public key
			if err := do.UploadSSHKey(d.Tools.SSH.Public); err != nil {
				return err
			}

			//Prepare droplet prefix name and tags
			namespace, repository, _, _ := d.URIs()
			tags := []string{namespace, repository, d.Tag}
			objects.WriteLog(fmt.Sprintf("Droplets tag is %v", tags), stdout, timestamp, d, &do)

			//Create DigitalOcean Droplets
			if err := do.CreateDroplets(d.Service.Nodes, d.Tools.SSH.Fingerprint, fmt.Sprintf("%s-%s", namespace, repository), tags); err != nil {
				return err
			}

			//Export droplets IP
			for id, droplet := range do.Droplets {
				node := objects.Node{
					ID:      id,
					IP:      droplet["public"],
					Private: droplet["private"],
					User:    service.DORootUser,
					Distro:  tools.DistroUbuntu,
				}

				objects.WriteLog(fmt.Sprintf("Droplet [%d] ip is [%s/%s]", id, droplet["public"], droplet["private"]), stdout, timestamp, d, &do)
				d.Nodes = append(d.Nodes, &node)
			}

			objects.WriteLog("Wait droplets to be ready(22 port on)", stdout, timestamp, d, &do)

			waitChannel := make(chan error)
			for _, node := range d.Nodes {
				go func(node *objects.Node) {
					// Try to dial the default ssh port on
					waitChannel <- utils.WaitForHostPort(node.IP, tools.DefaultSSHPort, 10, 10)
				}(node)
			}
			for i := 0; i < len(d.Nodes); i++ {
				if err := <-waitChannel; err != nil {
					return err
				}
			}

			d.Service.Logs = do.Logs
		default:
			return fmt.Errorf("unsupport service provide: %s", d.Service.Provider)
		}
	}

	if len(d.Nodes) > 0 {
		//Set Out Node Parameters
		i := 0
		for _, node := range d.Nodes {
			d.Output(fmt.Sprintf("NODE_%d", i), node.IP)
			i += 1

			//Export node ip to stdout
			objects.WriteLog(fmt.Sprintf("[COUT] NODE_%d = %s", i, node.IP), stdout, timestamp, d)
		}

		//Initialization node environment
		for _, node := range d.Nodes {
			objects.WriteLog(fmt.Sprintf("Initializate the %s node environment", node.IP), stdout, timestamp, d, node)
			if commands, err := tools.InitializationEnvironment(d.Tools.SSH.Private, node.IP, node.User, node.Distro, stdout); err != nil {
				return err
			} else {
				objects.WriteLog(fmt.Sprintf("execute commands %v in the %d node", commands, node.ID), stdout, timestamp, d, node)
			}
		}

		//Generate CA CARootTemplate files
		if roots, err := tools.GenerateCARootFiles(d.Config); err != nil {
			return err
		} else {
			objects.WriteLog(fmt.Sprintf("Generate cluster root CA files."), stdout, timestamp, d)
			for key, value := range roots {
				d.Output(key, value)
				objects.WriteLog(fmt.Sprintf("[COUT] %s = %s", key, value), stdout, timestamp, d)
			}

			//Upload CA root files to the nodes.
			for _, node := range d.Nodes {
				objects.WriteLog(fmt.Sprintf("Upload root CA files %v to %s node", roots, node.IP), stdout, timestamp, d, node)
				if err := tools.UploadCARootFiles(d.Tools.SSH.Private, roots, node.IP, node.User, stdout); err != nil {
					return err
				}
			}
		}

		//Deploy infras
		for _, infra := range d.Infras {
			switch infra.Name {
			case InfraEtcd:
				objects.WriteLog(fmt.Sprintf("Deploy etcd in cluster"), stdout, timestamp, d, infra)
				if err := infras.DeployEtcdInCluster(d, infra, stdout, timestamp); err != nil {
					return err
				}
			case InfraFlannel:
				objects.WriteLog(fmt.Sprintf("Deploy flannel in cluster"), stdout, timestamp, d, infra)
				if err := infras.DeployFlannelInCluster(d, infra, stdout, timestamp); err != nil {
					return err
				}
			case InfraDocker:
				objects.WriteLog(fmt.Sprintf("Deploy docker in cluster"), stdout, timestamp, d, infra)
				if err := infras.DeployDockerInCluster(d, infra, stdout, timestamp); err != nil {
					return err
				}
			case InfraKubernetes:
				objects.WriteLog(fmt.Sprintf("Deploy k8s in cluster"), stdout, timestamp, d, infra)
				if err := infras.DeployKubernetesInCluster(d, infra, stdout, timestamp); err != nil {
					return err
				}
			default:
				return fmt.Errorf("unsupport infrastruction software: %s", infra.Name)
			}
		}

	}

	//Save data
	if db == true {
		if err := d.Save(); err != nil {
			return err
		}
	}

	return err
}

//DeleteInfraStacks delete the droplets of stack.
func DeleteInfraStacks(d *objects.Deployment, stdout io.Writer, timestamp bool) error {
	if d.Service.Provider != "" {
		switch d.Service.Provider {
		case "digitalocean":
			do := service.DigitalOcean{}
			do.Token = d.Service.Token
			do.InitClient()

			for _, node := range d.Nodes {
				if err := do.DeleteDroplet(node.ID); err != nil {
					return err
				}
				objects.WriteLog(fmt.Sprintf("Delete %d droplet in DigitalOcaen", node.ID), stdout, timestamp, d)
			}

		default:
			return fmt.Errorf("unsupport service provide: %s", d.Service.Provider)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Bra(Brilliant Ridiculous Assistant) is a command line utility tool for Unknown.
package main

import (
	"os"
	"runtime"

	"github.com/codegangsta/cli"

	"github.com/Unknwon/bra/cmd"
)

const APP_VER = "0.1.1.1205"

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	cmd.AppVer = APP_VER
}

func main() {
	app := cli.NewApp()
	app.Name = "bra"
	app.Usage = "Brilliant Ridiculous Assistant"
	app.Version = APP_VER
	app.Commands = []cli.Command{
		cmd.CmdRun,
		cmd.CmdSync,
		// cmd.CmdHttp,
	}
	app.Flags = append(app.Flags, []cli.Flag{}...)
	app.Run(os.Args)
}
<commit_msg>mirror desc change<commit_after>// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Bra(Brilliant Ridiculous Assistant) is a command line utility tool for Unknown.
package main

import (
	"os"
	"runtime"

	"github.com/codegangsta/cli"

	"github.com/Unknwon/bra/cmd"
)

const APP_VER = "0.1.1.1205"

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	cmd.AppVer = APP_VER
}

func main() {
	app := cli.NewApp()
	app.Name = "Bra"
	app.Usage = "Brilliant Ridiculous Assistant is a command line utility tool for Unknwon."
	app.Version = APP_VER
	app.Commands = []cli.Command{
		cmd.CmdRun,
		cmd.CmdSync,
		// cmd.CmdHttp,
	}
	app.Flags = append(app.Flags, []cli.Flag{}...)
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package services

import (
	"encoding/json"
	"fmt"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/config"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/logger"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/models"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/restclient"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/retry"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/utils"
	"github.com/spf13/cobra"
	"net/http"
	"regexp"
	"strconv"
	"strings"
)

//CommandService holds the method signatures for all common Command object invocations
type CommandService interface {
	Get(uri string) (string, error)
	List(uri string) (string, error)
	Post(uri string, payload string) (string, error)
	Patch(uri string, payload string) (string, error)
	Put(uri string, payload string) (string, error)
	Delete(uri string) (string, error)
	DetermineAction(httpMethod string, operationId string, uri string, originalURI string) func(retryConfiguration *retry.RetryConfiguration) (string, error)
}

type commandService struct {
	cmd *cobra.Command
}

// The following functions are added as variables to allow reassignment to mock functions in unit tests
var (
	configGetConfig         = config.GetConfig
	restclientNewRESTClient = restclient.NewRESTClient
	// This will be set if the application silently re-authenticates for a 401 error
	hasReAuthenticated bool
)

//NewCommandService initializes a new command Service object
func NewCommandService(cmd *cobra.Command) *commandService {
	return &commandService{cmd: cmd}
}

func (c *commandService) Get(uri string) (string, error) {
	profileName, _ := c.cmd.Root().Flags().GetString("profile")
	config, err := configGetConfig(profileName)
	if err != nil {
		return "", err
	}

	restClient := restclientNewRESTClient(config)
	response, err := restClient.Get(uri)
	if err == nil {
		return response, nil
	}

	err = reAuthenticateIfNecessary(config, err)
	if err != nil {
		return "", err
	}

	return c.Get(uri)
}

func (c *commandService) List(uri string) (string, error) {
	profileName, _ := c.cmd.Root().Flags().GetString("profile")
	config, err := configGetConfig(profileName)
	if err != nil {
		return "", err
	}

	restClient := restclientNewRESTClient(config)

	//Looks up first page
	data, err := restClient.Get(uri)
	if err != nil {
		err = reAuthenticateIfNecessary(config, err)
		if err != nil {
			return "", err
		}
		return c.List(uri)
	}

	firstPage := &models.Entities{}
	json.Unmarshal([]byte(data), firstPage)

	//Allocate the total results based on the page count
	totalResults := make([]string, 0)

	//Appends the individual records from Entities into the array
	for _, val := range firstPage.Entities {
		totalResults = append(totalResults, string(val))
	}

    //Looks up the rest of the pages
	if firstPage.PageCount > 1 {
		pagedURI := uri
		for x := 2; x <= firstPage.PageCount; x++ {
			pagedURI = updatePageNumber(pagedURI, x)
			logger.Info("Paginating with URI: ", pagedURI)
			retryFunc := retry.Retry(pagedURI, restClient.Get)
			data, err = retryFunc(&retry.RetryConfiguration{
				MaxRetryTimeSec: 20,
				MaxRetriesBeforeQuitting: 10,
			})
			if err != nil {
				return "", err
			}

			pageData := &models.Entities{}
			json.Unmarshal([]byte(data), pageData)

			for _, val := range pageData.Entities {
				totalResults = append(totalResults, string(val))
			}
		}
	}

	//Convert the data into one big string
	var finalJSONString string
	switch len(totalResults) {
	case 0:
		finalJSONString = "[]"
	case 1:
		finalJSONString = fmt.Sprintf("[%s]", totalResults[0])
	default:
		finalJSONString = fmt.Sprintf("[%s]", strings.Join(totalResults, ","))
	}

	return finalJSONString, nil
}

func updatePageNumber(pagedURI string, index int) string {
	if strings.Contains(pagedURI,"pageNumber=") {
		re := regexp.MustCompile("pageNumber=([0-9]+)")
		result := re.FindStringSubmatch(pagedURI)
		pageNumber, _ := strconv.Atoi(result[1])
		pageNumber++
		pagedURI = strings.Replace(pagedURI, result[0], fmt.Sprintf("pageNumber=%v", pageNumber), 1)
	} else {
		if strings.Contains(pagedURI, "?") {
			pagedURI = fmt.Sprintf("%s&pageNumber=%d", pagedURI, index)
		} else {
			pagedURI = fmt.Sprintf("%s?pageNumber=%d", pagedURI, index)
		}
	}

	return pagedURI
}

func (c *commandService) Post(uri string, payload string) (string, error) {
	return c.upsert(http.MethodPost, uri, payload)
}

func (c *commandService) Patch(uri string, payload string) (string, error) {
	return c.upsert(http.MethodPatch, uri, payload)
}

func (c *commandService) Put(uri string, payload string) (string, error) {
	return c.upsert(http.MethodPut, uri, payload)
}

func (c *commandService) Delete(uri string) (string, error) {
	return c.upsert(http.MethodDelete, uri, "")
}

func (c *commandService) upsert(method string, uri string, payload string) (string, error) {
	profileName, _ := c.cmd.Root().Flags().GetString("profile")
	config, err := configGetConfig(profileName)
	if err != nil {
		return "", err
	}

	restClient := restclientNewRESTClient(config)

	var response string

	switch method {
	case http.MethodPost:
		response, err = restClient.Post(uri, payload)
	case http.MethodPut:
		response, err = restClient.Put(uri, payload)
	case http.MethodPatch:
		response, err = restClient.Patch(uri, payload)
	case http.MethodDelete:
		response, err = restClient.Delete(uri)
	default:
		logger.Fatalf("Unable to resolve the http verb: %v", method)
	}

	if err == nil {
		return response, nil
	}

	err = reAuthenticateIfNecessary(config, err)
	if err != nil {
		return "", err
	}

	return c.upsert(method, uri, payload)
}

func reAuthenticateIfNecessary(config config.Configuration, err error) error {
	if hasReAuthenticated {
		logger.Warn("Have already re-authenticated. Will not authenticate again")
		return err
	}

	if e, ok := err.(models.HttpStatusError); ok && e.StatusCode == http.StatusUnauthorized {
		logger.Info("Received HTTP 401 error, re-authenticating")
		_, err = restclient.ReAuthenticate(config)
		if err != nil {
			return err
		}
	} else {
		return err
	}

	hasReAuthenticated = true

	return nil
}

func (c *commandService) DetermineAction(httpMethod string, operationId string, uri string, originalURI string) func(retryConfiguration *retry.RetryConfiguration) (string, error) {
	switch httpMethod {
	case http.MethodGet:
		listOverrides := make(map[string]int)
		// Overrides for resources with custom operationIds requiring pagination
		{{=it.listOverrides}}

		_, ok := listOverrides[originalURI]
		if operationId == "list" || ok {
			return retry.Retry(uri, c.List)
		} else {
			return retry.Retry(uri, c.Get)
		}
	case http.MethodPatch:
		return retry.RetryWithData(uri, utils.ResolveInputData(c.cmd), c.Patch)
	case http.MethodPost:
		return retry.RetryWithData(uri, utils.ResolveInputData(c.cmd), c.Post)
	case http.MethodPut:
		return retry.RetryWithData(uri, utils.ResolveInputData(c.cmd), c.Put)
	case http.MethodDelete:
		return retry.Retry(uri, c.Delete)
	}
	return nil
}
<commit_msg>Large retry parameters when paginating to avoid rate-limits<commit_after>package services

import (
	"encoding/json"
	"fmt"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/config"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/logger"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/models"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/restclient"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/retry"
	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/utils"
	"github.com/spf13/cobra"
	"net/http"
	"regexp"
	"strconv"
	"strings"
)

//CommandService holds the method signatures for all common Command object invocations
type CommandService interface {
	Get(uri string) (string, error)
	List(uri string) (string, error)
	Post(uri string, payload string) (string, error)
	Patch(uri string, payload string) (string, error)
	Put(uri string, payload string) (string, error)
	Delete(uri string) (string, error)
	DetermineAction(httpMethod string, operationId string, uri string, originalURI string) func(retryConfiguration *retry.RetryConfiguration) (string, error)
}

type commandService struct {
	cmd *cobra.Command
}

// The following functions are added as variables to allow reassignment to mock functions in unit tests
var (
	configGetConfig         = config.GetConfig
	restclientNewRESTClient = restclient.NewRESTClient
	// This will be set if the application silently re-authenticates for a 401 error
	hasReAuthenticated bool
)

//NewCommandService initializes a new command Service object
func NewCommandService(cmd *cobra.Command) *commandService {
	return &commandService{cmd: cmd}
}

func (c *commandService) Get(uri string) (string, error) {
	profileName, _ := c.cmd.Root().Flags().GetString("profile")
	config, err := configGetConfig(profileName)
	if err != nil {
		return "", err
	}

	restClient := restclientNewRESTClient(config)
	response, err := restClient.Get(uri)
	if err == nil {
		return response, nil
	}

	err = reAuthenticateIfNecessary(config, err)
	if err != nil {
		return "", err
	}

	return c.Get(uri)
}

func (c *commandService) List(uri string) (string, error) {
	profileName, _ := c.cmd.Root().Flags().GetString("profile")
	config, err := configGetConfig(profileName)
	if err != nil {
		return "", err
	}

	restClient := restclientNewRESTClient(config)

	//Looks up first page
	data, err := restClient.Get(uri)
	if err != nil {
		err = reAuthenticateIfNecessary(config, err)
		if err != nil {
			return "", err
		}
		return c.List(uri)
	}

	firstPage := &models.Entities{}
	json.Unmarshal([]byte(data), firstPage)

	//Allocate the total results based on the page count
	totalResults := make([]string, 0)

	//Appends the individual records from Entities into the array
	for _, val := range firstPage.Entities {
		totalResults = append(totalResults, string(val))
	}

    //Looks up the rest of the pages
	if firstPage.PageCount > 1 {
		pagedURI := uri
		for x := 2; x <= firstPage.PageCount; x++ {
			pagedURI = updatePageNumber(pagedURI, x)
			logger.Info("Paginating with URI: ", pagedURI)
			retryFunc := retry.Retry(pagedURI, restClient.Get)
			data, err = retryFunc(&retry.RetryConfiguration{
				MaxRetryTimeSec: 1000,
				MaxRetriesBeforeQuitting: 100,
			})
			if err != nil {
				return "", err
			}

			pageData := &models.Entities{}
			json.Unmarshal([]byte(data), pageData)

			for _, val := range pageData.Entities {
				totalResults = append(totalResults, string(val))
			}
		}
	}

	//Convert the data into one big string
	var finalJSONString string
	switch len(totalResults) {
	case 0:
		finalJSONString = "[]"
	case 1:
		finalJSONString = fmt.Sprintf("[%s]", totalResults[0])
	default:
		finalJSONString = fmt.Sprintf("[%s]", strings.Join(totalResults, ","))
	}

	return finalJSONString, nil
}

func updatePageNumber(pagedURI string, index int) string {
	if strings.Contains(pagedURI,"pageNumber=") {
		re := regexp.MustCompile("pageNumber=([0-9]+)")
		result := re.FindStringSubmatch(pagedURI)
		pageNumber, _ := strconv.Atoi(result[1])
		pageNumber++
		pagedURI = strings.Replace(pagedURI, result[0], fmt.Sprintf("pageNumber=%v", pageNumber), 1)
	} else {
		if strings.Contains(pagedURI, "?") {
			pagedURI = fmt.Sprintf("%s&pageNumber=%d", pagedURI, index)
		} else {
			pagedURI = fmt.Sprintf("%s?pageNumber=%d", pagedURI, index)
		}
	}

	return pagedURI
}

func (c *commandService) Post(uri string, payload string) (string, error) {
	return c.upsert(http.MethodPost, uri, payload)
}

func (c *commandService) Patch(uri string, payload string) (string, error) {
	return c.upsert(http.MethodPatch, uri, payload)
}

func (c *commandService) Put(uri string, payload string) (string, error) {
	return c.upsert(http.MethodPut, uri, payload)
}

func (c *commandService) Delete(uri string) (string, error) {
	return c.upsert(http.MethodDelete, uri, "")
}

func (c *commandService) upsert(method string, uri string, payload string) (string, error) {
	profileName, _ := c.cmd.Root().Flags().GetString("profile")
	config, err := configGetConfig(profileName)
	if err != nil {
		return "", err
	}

	restClient := restclientNewRESTClient(config)

	var response string

	switch method {
	case http.MethodPost:
		response, err = restClient.Post(uri, payload)
	case http.MethodPut:
		response, err = restClient.Put(uri, payload)
	case http.MethodPatch:
		response, err = restClient.Patch(uri, payload)
	case http.MethodDelete:
		response, err = restClient.Delete(uri)
	default:
		logger.Fatalf("Unable to resolve the http verb: %v", method)
	}

	if err == nil {
		return response, nil
	}

	err = reAuthenticateIfNecessary(config, err)
	if err != nil {
		return "", err
	}

	return c.upsert(method, uri, payload)
}

func reAuthenticateIfNecessary(config config.Configuration, err error) error {
	if hasReAuthenticated {
		logger.Warn("Have already re-authenticated. Will not authenticate again")
		return err
	}

	if e, ok := err.(models.HttpStatusError); ok && e.StatusCode == http.StatusUnauthorized {
		logger.Info("Received HTTP 401 error, re-authenticating")
		_, err = restclient.ReAuthenticate(config)
		if err != nil {
			return err
		}
	} else {
		return err
	}

	hasReAuthenticated = true

	return nil
}

func (c *commandService) DetermineAction(httpMethod string, operationId string, uri string, originalURI string) func(retryConfiguration *retry.RetryConfiguration) (string, error) {
	switch httpMethod {
	case http.MethodGet:
		listOverrides := make(map[string]int)
		// Overrides for resources with custom operationIds requiring pagination
		{{=it.listOverrides}}

		_, ok := listOverrides[originalURI]
		if operationId == "list" || ok {
			return retry.Retry(uri, c.List)
		} else {
			return retry.Retry(uri, c.Get)
		}
	case http.MethodPatch:
		return retry.RetryWithData(uri, utils.ResolveInputData(c.cmd), c.Patch)
	case http.MethodPost:
		return retry.RetryWithData(uri, utils.ResolveInputData(c.cmd), c.Post)
	case http.MethodPut:
		return retry.RetryWithData(uri, utils.ResolveInputData(c.cmd), c.Put)
	case http.MethodDelete:
		return retry.Retry(uri, c.Delete)
	}
	return nil
}
<|endoftext|>


<commit_before>package ccg

//go:generate myccg -u AstDecls.Filter -o utils.go slice ast.Decl AstDecls
//go:generate myccg -u AstSpecs.Filter -o utils.go slice ast.Spec AstSpecs
//go:generate myccg -u ObjectSet.Add,ObjectSet.In,NewObjectSet -o utils.go set types.Object ObjectSet NewObjectSet

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/format"
	"go/token"
	"io"
	"strings"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
	"golang.org/x/tools/imports"
)

var (
	pt = fmt.Printf
)

type Config struct {
	From    string
	Params  map[string]string
	Renames map[string]string
	Writer  io.Writer
	Package string
	Decls   []ast.Decl
	FileSet *token.FileSet
	Uses    []string
}

func Copy(config Config) error {
	// load package
	loadConf := loader.Config{
		Fset: config.FileSet,
	}
	loadConf.Import(config.From)
	program, err := loadConf.Load()
	if err != nil {
		return fmt.Errorf("ccg: load package %v", err)
	}
	info := program.Imported[config.From]

	// remove param declarations
	for _, f := range info.Files {
		f.Decls = filterDecls(f.Decls, func(node interface{}) bool {
			switch node := node.(type) {
			case *ast.TypeSpec:
				name := node.Name.Name
				_, exists := config.Params[name]
				return !exists
			case valueInfo:
				_, exists := config.Params[node.Name.Name]
				return !exists
			}
			return true
		})
	}

	// collect objects to rename
	renamed := map[string]string{}
	objects := make(map[types.Object]string)
	collectObjects := func(mapping map[string]string) error {
		for from, to := range mapping {
			obj := info.Pkg.Scope().Lookup(from)
			if obj == nil {
				return fmt.Errorf("ccg: name not found %s", from)
			}
			objects[obj] = to
			renamed[to] = from
		}
		return nil
	}
	if err := collectObjects(config.Params); err != nil {
		return err
	}
	if err := collectObjects(config.Renames); err != nil {
		return err
	}

	// rename
	rename := func(defs map[*ast.Ident]types.Object) {
		for id, obj := range defs {
			if to, ok := objects[obj]; ok {
				id.Name = to
			}
		}
	}
	rename(info.Defs)
	rename(info.Uses)

	// collect existing decls
	existingDecls := make(map[string]func(interface{}))
	var decls []ast.Decl
	for i, decl := range config.Decls {
		decls = append(decls, decl)
		switch decl := decl.(type) {
		case *ast.GenDecl:
			switch decl.Tok {
			case token.VAR, token.CONST:
				for _, spec := range decl.Specs {
					spec := spec.(*ast.ValueSpec)
					for i, name := range spec.Names {
						i := i
						spec := spec
						existingDecls[name.Name] = func(expr interface{}) {
							spec.Values[i] = expr.(ast.Expr)
						}
					}
				}
			case token.TYPE:
				for i, spec := range decl.Specs {
					spec := spec.(*ast.TypeSpec)
					i := i
					decl := decl
					existingDecls[spec.Name.Name] = func(expr interface{}) {
						decl.Specs[i].(*ast.TypeSpec).Type = expr.(ast.Expr)
					}
				}
			case token.IMPORT:
				for i, spec := range decl.Specs {
					spec := spec.(*ast.ImportSpec)
					i := i
					decl := decl
					var name string
					if spec.Name == nil {
						name = spec.Path.Value
					} else {
						name = spec.Name.Name
					}
					existingDecls[name] = func(path interface{}) {
						decl.Specs[i].(*ast.ImportSpec).Path = path.(*ast.BasicLit)
					}
				}
			}
		case *ast.FuncDecl:
			name := decl.Name.Name
			if decl.Recv != nil {
				name = decl.Recv.List[0].Type.(*ast.Ident).Name + "." + name
			}
			i := i
			existingDecls[name] = func(fndecl interface{}) {
				decls[i] = fndecl.(*ast.FuncDecl)
			}
		}
	}

	// collect output declarations
	var newDecls []ast.Decl
	for _, f := range info.Files {
		for _, decl := range f.Decls {
			switch decl := decl.(type) {
			case *ast.GenDecl:
				switch decl.Tok {
				case token.VAR, token.CONST:
					newDecl := &ast.GenDecl{
						Tok: decl.Tok,
					}
					for _, spec := range decl.Specs {
						spec := spec.(*ast.ValueSpec)
						for i, name := range spec.Names {
							if mutator, ok := existingDecls[name.Name]; ok {
								mutator(spec.Values[i])
							} else {
								newDecl.Specs = append(newDecl.Specs, spec)
							}
						}
					}
					if len(newDecl.Specs) > 0 {
						newDecls = append(newDecls, newDecl)
					}
				case token.TYPE:
					newDecl := &ast.GenDecl{
						Tok: token.TYPE,
					}
					for _, spec := range decl.Specs {
						spec := spec.(*ast.TypeSpec)
						name := spec.Name.Name
						if mutator, ok := existingDecls[name]; ok {
							mutator(spec.Type)
						} else {
							newDecl.Specs = append(newDecl.Specs, spec)
						}
					}
					if len(newDecl.Specs) > 0 {
						newDecls = append(newDecls, newDecl)
					}
				case token.IMPORT:
					newDecl := &ast.GenDecl{
						Tok: token.IMPORT,
					}
					for _, spec := range decl.Specs {
						spec := spec.(*ast.ImportSpec)
						var name string
						if spec.Name == nil {
							name = spec.Path.Value
						} else {
							name = spec.Name.Name
						}
						if mutator, ok := existingDecls[name]; ok {
							mutator(spec.Path)
						} else {
							newDecl.Specs = append(newDecl.Specs, spec)
						}
					}
					if len(newDecl.Specs) > 0 {
						newDecls = append(newDecls, newDecl)
					}
				}
			case *ast.FuncDecl:
				name := decl.Name.Name
				if decl.Recv != nil {
					name = decl.Recv.List[0].Type.(*ast.Ident).Name + "." + name
				}
				if mutator, ok := existingDecls[name]; ok {
					mutator(decl)
				} else {
					newDecls = append(newDecls, decl)
				}
			}
		}
	}

	// get function dependencies
	deps := make(map[types.Object]ObjectSet)
	collectDeps := func(decls []ast.Decl) {
		for _, decl := range decls {
			switch decl := decl.(type) {
			case *ast.FuncDecl:
				obj := info.ObjectOf(decl.Name)
				set := NewObjectSet()
				var visitor astVisitor
				visitor = func(node ast.Node) astVisitor {
					switch node := node.(type) {
					case *ast.Ident:
						dep := info.ObjectOf(node)
						set.Add(dep)
					}
					return visitor
				}
				ast.Walk(visitor, decl)
				deps[obj] = set
			}
		}
	}
	collectDeps(newDecls)
	collectDeps(decls)

	// get uses objects
	uses := NewObjectSet()
	for _, use := range config.Uses {
		parts := strings.Split(use, ".")
		switch len(parts) {
		case 2: // method
			var ty types.Object
			if from, ok := renamed[parts[0]]; ok { // renamed type, use original type name
				ty = info.Pkg.Scope().Lookup(from)
			} else {
				ty = info.Pkg.Scope().Lookup(parts[0])
			}
			typeName, ok := ty.(*types.TypeName)
			if !ok {
				return fmt.Errorf("%s is not a type", parts[0])
			}
			obj, _, _ := types.LookupFieldOrMethod(typeName.Type(), true, info.Pkg, parts[1])
			uses.Add(obj)
		case 1: // non-method
			var obj types.Object
			if from, ok := renamed[parts[0]]; ok { // renamed function
				obj = info.Pkg.Scope().Lookup(from)
			} else {
				obj = info.Pkg.Scope().Lookup(parts[0])
			}
			uses.Add(obj)
		default:
			return fmt.Errorf("invalid use spec: %s", use)
		}
	}

	// filter by uses
	if len(uses) > 0 {
		// calculate uses closure
		for {
			l := len(uses)
			for use := range uses {
				if deps, ok := deps[use]; ok {
					for dep := range deps {
						uses.Add(dep)
					}
				}
			}
			if len(uses) == l {
				break
			}
		}
		// filter
		newDecls = filterDecls(newDecls, func(node interface{}) bool {
			switch node := node.(type) {
			case *ast.FuncDecl:
				return uses.In(info.ObjectOf(node.Name))
			case *ast.TypeSpec:
				return uses.In(info.ObjectOf(node.Name))
			case valueInfo:
				return uses.In(info.ObjectOf(node.Name))
			}
			return true
		})
	}

	// merge new and existing decls
	decls = append(decls, newDecls...)

	// decls tidy ups
	newDecls = newDecls[0:0]
	var importDecls []ast.Decl
	for _, decl := range decls {
		// ensure linebreak between decls
		switch decl := decl.(type) {
		case *ast.FuncDecl:
			if decl.Doc == nil {
				decl.Doc = new(ast.CommentGroup)
			}
		case *ast.GenDecl:
			if decl.Doc == nil {
				decl.Doc = new(ast.CommentGroup)
			}
		}
		// move import decls to beginning
		if decl, ok := decl.(*ast.GenDecl); ok && decl.Tok == token.IMPORT {
			importDecls = append(importDecls, decl)
			continue
		}
		newDecls = append(newDecls, decl)
	}
	decls = append(importDecls, newDecls...)

	// output
	if config.Writer != nil {
		if config.Package != "" { // output complete file
			file := &ast.File{
				Name:  ast.NewIdent(config.Package),
				Decls: decls,
			}
			buf := new(bytes.Buffer)
			err = format.Node(buf, program.Fset, file)
			if err != nil { //NOCOVER
				return fmt.Errorf("ccg: format output %v", err)
			}
			bs, err := imports.Process("", buf.Bytes(), nil)
			if err != nil { //NOCOVER
				return fmt.Errorf("ccg: format output %v", err)
			}
			config.Writer.Write(bs)
		} else { // output decls only
			err = format.Node(config.Writer, program.Fset, decls)
			if err != nil { //NOCOVER
				return fmt.Errorf("ccg: format output %v", err)
			}
		}
	}

	return nil
}

type astVisitor func(ast.Node) astVisitor

func (v astVisitor) Visit(node ast.Node) ast.Visitor {
	return v(node)
}

type valueInfo struct {
	Name  *ast.Ident
	Value ast.Expr
	Type  ast.Expr
}

func filterDecls(decls []ast.Decl, fn func(interface{}) bool) []ast.Decl {
	decls = AstDecls(decls).Filter(func(decl ast.Decl) (ret bool) {
		switch decl := decl.(type) {
		case *ast.FuncDecl:
			ret = fn(decl)
		case *ast.GenDecl:
			switch decl.Tok {
			case token.VAR, token.CONST:
				decl.Specs = AstSpecs(decl.Specs).Filter(func(sp ast.Spec) bool {
					spec := sp.(*ast.ValueSpec)
					names := []*ast.Ident{}
					values := []ast.Expr{}
					for i, name := range spec.Names {
						var value ast.Expr
						if i < len(spec.Values) {
							value = spec.Values[i]
						}
						if fn(valueInfo{name, value, spec.Type}) {
							names = append(names, name)
							if value != nil {
								values = append(values, value)
							}
						}
					}
					spec.Names = names
					if len(values) == 0 {
						spec.Values = nil
					} else {
						spec.Values = values
					}
					return len(spec.Names) > 0
				})
				ret = len(decl.Specs) > 0
			case token.TYPE, token.IMPORT:
				decl.Specs = AstSpecs(decl.Specs).Filter(func(spec ast.Spec) bool {
					return fn(spec)
				})
				ret = len(decl.Specs) > 0
			}
		}
		return
	})
	return decls
}
<commit_msg>refactors<commit_after>package ccg

//go:generate myccg -u AstDecls.Filter -o utils.go slice ast.Decl AstDecls
//go:generate myccg -u AstSpecs.Filter -o utils.go slice ast.Spec AstSpecs
//go:generate myccg -u ObjectSet.Add,ObjectSet.In,NewObjectSet -o utils.go set types.Object ObjectSet NewObjectSet

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/format"
	"go/token"
	"io"
	"strings"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
	"golang.org/x/tools/imports"
)

var (
	pt = fmt.Printf
)

type Config struct {
	From    string
	Params  map[string]string
	Renames map[string]string
	Writer  io.Writer
	Package string
	Decls   []ast.Decl
	FileSet *token.FileSet
	Uses    []string
}

func Copy(config Config) error {
	// load package
	loadConf := loader.Config{
		Fset: config.FileSet,
	}
	loadConf.Import(config.From)
	program, err := loadConf.Load()
	if err != nil {
		return fmt.Errorf("ccg: load package %v", err)
	}
	info := program.Imported[config.From]

	// remove param declarations
	for _, f := range info.Files {
		f.Decls = filterDecls(f.Decls, func(node interface{}) bool {
			switch node := node.(type) {
			case *ast.TypeSpec:
				name := node.Name.Name
				_, exists := config.Params[name]
				return !exists
			case valueInfo:
				_, exists := config.Params[node.Name.Name]
				return !exists
			}
			return true
		})
	}

	// collect objects to rename
	renamed := map[string]string{}
	objects := make(map[types.Object]string)
	collectObjects := func(mapping map[string]string) error {
		for from, to := range mapping {
			obj := info.Pkg.Scope().Lookup(from)
			if obj == nil {
				return fmt.Errorf("ccg: name not found %s", from)
			}
			objects[obj] = to
			renamed[to] = from
		}
		return nil
	}
	if err := collectObjects(config.Params); err != nil {
		return err
	}
	if err := collectObjects(config.Renames); err != nil {
		return err
	}

	// rename
	rename := func(defs map[*ast.Ident]types.Object) {
		for id, obj := range defs {
			if to, ok := objects[obj]; ok {
				id.Name = to
			}
		}
	}
	rename(info.Defs)
	rename(info.Uses)

	// collect existing decls
	existingDecls := make(map[string]func(interface{}))
	decls := []ast.Decl{}
	used := NewObjectSet()
	for i, decl := range config.Decls {
		decls = append(decls, decl)
		switch decl := decl.(type) {
		case *ast.GenDecl:
			switch decl.Tok {
			case token.VAR, token.CONST:
				for _, spec := range decl.Specs {
					spec := spec.(*ast.ValueSpec)
					for i, name := range spec.Names {
						i := i
						spec := spec
						existingDecls[name.Name] = func(expr interface{}) {
							spec.Values[i] = expr.(ast.Expr)
						}
						used.Add(info.ObjectOf(name))
					}
				}
			case token.TYPE:
				for i, spec := range decl.Specs {
					spec := spec.(*ast.TypeSpec)
					i := i
					decl := decl
					existingDecls[spec.Name.Name] = func(expr interface{}) {
						decl.Specs[i].(*ast.TypeSpec).Type = expr.(ast.Expr)
					}
					used.Add(info.ObjectOf(spec.Name))
				}
			case token.IMPORT:
				for i, spec := range decl.Specs {
					spec := spec.(*ast.ImportSpec)
					i := i
					decl := decl
					var name string
					if spec.Name == nil {
						name = spec.Path.Value
					} else {
						name = spec.Name.Name
					}
					existingDecls[name] = func(path interface{}) {
						decl.Specs[i].(*ast.ImportSpec).Path = path.(*ast.BasicLit)
					}
				}
			}
		case *ast.FuncDecl:
			name := decl.Name.Name
			if decl.Recv != nil {
				name = decl.Recv.List[0].Type.(*ast.Ident).Name + "." + name
			}
			i := i
			existingDecls[name] = func(fndecl interface{}) {
				decl := fndecl.(*ast.FuncDecl)
				decls[i] = decl
				used.Add(info.ObjectOf(decl.Name))
			}
			used.Add(info.ObjectOf(decl.Name))
		}
	}

	// collect output declarations
	for _, f := range info.Files {
		for _, decl := range f.Decls {
			switch decl := decl.(type) {
			case *ast.GenDecl:
				switch decl.Tok {
				case token.VAR, token.CONST:
					newDecl := &ast.GenDecl{
						Tok: decl.Tok,
					}
					for _, spec := range decl.Specs {
						spec := spec.(*ast.ValueSpec)
						for i, name := range spec.Names {
							if mutator, ok := existingDecls[name.Name]; ok {
								mutator(spec.Values[i])
							} else {
								newDecl.Specs = append(newDecl.Specs, spec)
							}
						}
					}
					if len(newDecl.Specs) > 0 {
						decls = append(decls, newDecl)
					}
				case token.TYPE:
					newDecl := &ast.GenDecl{
						Tok: token.TYPE,
					}
					for _, spec := range decl.Specs {
						spec := spec.(*ast.TypeSpec)
						name := spec.Name.Name
						if mutator, ok := existingDecls[name]; ok {
							mutator(spec.Type)
						} else {
							newDecl.Specs = append(newDecl.Specs, spec)
						}
					}
					if len(newDecl.Specs) > 0 {
						decls = append(decls, newDecl)
					}
				case token.IMPORT:
					newDecl := &ast.GenDecl{
						Tok: token.IMPORT,
					}
					for _, spec := range decl.Specs {
						spec := spec.(*ast.ImportSpec)
						var name string
						if spec.Name == nil {
							name = spec.Path.Value
						} else {
							name = spec.Name.Name
						}
						if mutator, ok := existingDecls[name]; ok {
							mutator(spec.Path)
						} else {
							newDecl.Specs = append(newDecl.Specs, spec)
						}
					}
					if len(newDecl.Specs) > 0 {
						decls = append(decls, newDecl)
					}
				}
			case *ast.FuncDecl:
				name := decl.Name.Name
				if decl.Recv != nil {
					name = decl.Recv.List[0].Type.(*ast.Ident).Name + "." + name
				}
				if mutator, ok := existingDecls[name]; ok {
					mutator(decl)
				} else {
					decls = append(decls, decl)
				}
			}
		}
	}

	// get function dependencies
	deps := make(map[types.Object]ObjectSet)
	for _, decl := range decls {
		switch decl := decl.(type) {
		case *ast.FuncDecl:
			obj := info.ObjectOf(decl.Name)
			set := NewObjectSet()
			var visitor astVisitor
			visitor = func(node ast.Node) astVisitor {
				switch node := node.(type) {
				case *ast.Ident:
					dep := info.ObjectOf(node)
					set.Add(dep)
				}
				return visitor
			}
			ast.Walk(visitor, decl)
			deps[obj] = set
		}
	}

	// get objects being used
	for _, use := range config.Uses {
		parts := strings.Split(use, ".")
		switch len(parts) {
		case 2: // method
			var ty types.Object
			if from, ok := renamed[parts[0]]; ok { // renamed type, use original type name
				ty = info.Pkg.Scope().Lookup(from)
			} else {
				ty = info.Pkg.Scope().Lookup(parts[0])
			}
			typeName, ok := ty.(*types.TypeName)
			if !ok {
				return fmt.Errorf("%s is not a type", parts[0])
			}
			obj, _, _ := types.LookupFieldOrMethod(typeName.Type(), true, info.Pkg, parts[1])
			used.Add(obj)
		case 1: // non-method
			var obj types.Object
			if from, ok := renamed[parts[0]]; ok { // renamed function
				obj = info.Pkg.Scope().Lookup(from)
			} else {
				obj = info.Pkg.Scope().Lookup(parts[0])
			}
			used.Add(obj)
		default:
			return fmt.Errorf("invalid use spec: %s", use)
		}
	}

	// filter
	if len(config.Uses) > 0 {
		// calculate uses closure
		for {
			l := len(used)
			for use := range used {
				if deps, ok := deps[use]; ok {
					for dep := range deps {
						used.Add(dep)
					}
				}
			}
			if len(used) == l {
				break
			}
		}
		// filter
		decls = filterDecls(decls, func(node interface{}) bool {
			switch node := node.(type) {
			case *ast.FuncDecl:
				return used.In(info.ObjectOf(node.Name))
			case *ast.TypeSpec:
				return used.In(info.ObjectOf(node.Name))
			case valueInfo:
				return used.In(info.ObjectOf(node.Name))
			}
			return true
		})
	}

	// decls tidy ups
	newDecls := []ast.Decl{}
	var importDecls []ast.Decl
	for _, decl := range decls {
		// ensure linebreak between decls
		switch decl := decl.(type) {
		case *ast.FuncDecl:
			if decl.Doc == nil {
				decl.Doc = new(ast.CommentGroup)
			}
		case *ast.GenDecl:
			if decl.Doc == nil {
				decl.Doc = new(ast.CommentGroup)
			}
		}
		// move import decls to the beginning
		if decl, ok := decl.(*ast.GenDecl); ok && decl.Tok == token.IMPORT {
			importDecls = append(importDecls, decl)
			continue
		}
		newDecls = append(newDecls, decl)
	}
	decls = append(importDecls, newDecls...)

	// output
	if config.Writer != nil {
		if config.Package != "" { // output complete file
			file := &ast.File{
				Name:  ast.NewIdent(config.Package),
				Decls: decls,
			}
			buf := new(bytes.Buffer)
			err = format.Node(buf, program.Fset, file)
			if err != nil { //NOCOVER
				return fmt.Errorf("ccg: format output %v", err)
			}
			bs, err := imports.Process("", buf.Bytes(), nil)
			if err != nil { //NOCOVER
				return fmt.Errorf("ccg: format output %v", err)
			}
			config.Writer.Write(bs)
		} else { // output decls only
			err = format.Node(config.Writer, program.Fset, decls)
			if err != nil { //NOCOVER
				return fmt.Errorf("ccg: format output %v", err)
			}
		}
	}

	return nil
}

type astVisitor func(ast.Node) astVisitor

func (v astVisitor) Visit(node ast.Node) ast.Visitor {
	return v(node)
}

type valueInfo struct {
	Name  *ast.Ident
	Value ast.Expr
	Type  ast.Expr
}

func filterDecls(decls []ast.Decl, fn func(interface{}) bool) []ast.Decl {
	decls = AstDecls(decls).Filter(func(decl ast.Decl) (ret bool) {
		switch decl := decl.(type) {
		case *ast.FuncDecl:
			ret = fn(decl)
		case *ast.GenDecl:
			switch decl.Tok {
			case token.VAR, token.CONST:
				decl.Specs = AstSpecs(decl.Specs).Filter(func(sp ast.Spec) bool {
					spec := sp.(*ast.ValueSpec)
					names := []*ast.Ident{}
					values := []ast.Expr{}
					for i, name := range spec.Names {
						var value ast.Expr
						if i < len(spec.Values) {
							value = spec.Values[i]
						}
						if fn(valueInfo{name, value, spec.Type}) {
							names = append(names, name)
							if value != nil {
								values = append(values, value)
							}
						}
					}
					spec.Names = names
					if len(values) == 0 {
						spec.Values = nil
					} else {
						spec.Values = values
					}
					return len(spec.Names) > 0
				})
				ret = len(decl.Specs) > 0
			case token.TYPE, token.IMPORT:
				decl.Specs = AstSpecs(decl.Specs).Filter(func(spec ast.Spec) bool {
					return fn(spec)
				})
				ret = len(decl.Specs) > 0
			}
		}
		return
	})
	return decls
}
<|endoftext|>


<commit_before>package flagconfig

import (
	"flag"
	"fmt"
	"os"
	"strconv"
)

var fullConfig = map[string]interface{}{}

// GetInt64 looks for a configuration parameter of the given name and
// returns its value (assuming the parameter is an integer)
func GetInt64(name string) int64 {
	val, ok := fullConfig[name]
	if !ok {
		panic("attempted to access non-int-parameter " + name)
	}
	return val.(int64)
}

// GetStr looks for a configuration parameter of the given name and
// returns its value (assuming the parameter is a string)
func GetStr(name string) string {
	val, ok := fullConfig[name]
	if !ok {
		panic("attempted to access non-str-parameter " + name)
	}
	return val.(string)
}

// Pars loads flagconfig's runtime configuration, using both command-line arguments
// and a possible configuration file
func Parse(projname string) {

	//Load cli into its own set of config maps
	cliConfig := map[string]interface{}{}
	for name, param := range params {
		if param.Type == INT {
			dummy := int64(0)
			cliConfig[name] = &dummy
			flag.Int64Var(cliConfig[name].(*int64), name, param.Default.(int64), param.Description)
		} else {
			dummy := ""
			cliConfig[name] = &dummy
			flag.StringVar(cliConfig[name].(*string), name, param.Default.(string), param.Description)
		}
	}

	//Some extra cli args
	dumpExample := flag.Bool("example", false, "Dump example configuration to stdout and exit")
	configFile := flag.String("config", "", "Configuration file to load, empty means don't load any file and only use command-line args")

	flag.Parse()

	//If the flag to dump example config is set to true, do that
	if *dumpExample {
		fmt.Print(dumpExampleConfig(projname))
		os.Exit(0)
	}

	//If config file is specified, load the string map from it and load the values into
	//global config
	if *configFile != "" {
		configFileMap, err := readConfig(*configFile)
		if err != nil {
			panic(err)
		}

		for name, val := range configFileMap {
			if param, ok := params[name]; ok {
				if param.Type == INT {
					valint, err := strconv.Atoi(val)
					if err != nil {
						panic("field " + name + " in " + *configFile + " cannot be read as a number")
					}
					fullConfig[name] = valint
				} else {
					fullConfig[name] = val
				}
			}
		}
	}

	//Now we look through each param. If it's set on the command-line (not to the default)
	//we set that in the global config maps. If it's also not set in the conf we set it
	//to the param's default. If it is set in the conf then it's already set in the global
	//configs by the previous section
	for name, param := range params {
		if param.Type == INT {
			cliVal := *cliConfig[name].(*int64)
			_, confSet := fullConfig[name]
			if cliVal != param.Default {
				fullConfig[name] = cliVal
			} else if !confSet {
				fullConfig[name] = param.Default.(int64)
			}
		} else {
			cliVal := *cliConfig[name].(*string)
			_, confSet := fullConfig[name]
			if cliVal != param.Default {
				fullConfig[name] = cliVal
			} else if !confSet {
				fullConfig[name] = param.Default.(string)
			}
		}
	}

}
<commit_msg>another int64 fix<commit_after>package flagconfig

import (
	"flag"
	"fmt"
	"os"
	"strconv"
)

var fullConfig = map[string]interface{}{}

// GetInt64 looks for a configuration parameter of the given name and
// returns its value (assuming the parameter is an integer)
func GetInt64(name string) int64 {
	val, ok := fullConfig[name]
	if !ok {
		panic("attempted to access non-int-parameter " + name)
	}
	return val.(int64)
}

// GetStr looks for a configuration parameter of the given name and
// returns its value (assuming the parameter is a string)
func GetStr(name string) string {
	val, ok := fullConfig[name]
	if !ok {
		panic("attempted to access non-str-parameter " + name)
	}
	return val.(string)
}

// Pars loads flagconfig's runtime configuration, using both command-line arguments
// and a possible configuration file
func Parse(projname string) {

	//Load cli into its own set of config maps
	cliConfig := map[string]interface{}{}
	for name, param := range params {
		if param.Type == INT {
			dummy := int64(0)
			cliConfig[name] = &dummy
			flag.Int64Var(cliConfig[name].(*int64), name, param.Default.(int64), param.Description)
		} else {
			dummy := ""
			cliConfig[name] = &dummy
			flag.StringVar(cliConfig[name].(*string), name, param.Default.(string), param.Description)
		}
	}

	//Some extra cli args
	dumpExample := flag.Bool("example", false, "Dump example configuration to stdout and exit")
	configFile := flag.String("config", "", "Configuration file to load, empty means don't load any file and only use command-line args")

	flag.Parse()

	//If the flag to dump example config is set to true, do that
	if *dumpExample {
		fmt.Print(dumpExampleConfig(projname))
		os.Exit(0)
	}

	//If config file is specified, load the string map from it and load the values into
	//global config
	if *configFile != "" {
		configFileMap, err := readConfig(*configFile)
		if err != nil {
			panic(err)
		}

		for name, val := range configFileMap {
			if param, ok := params[name]; ok {
				if param.Type == INT {
					valint, err := strconv.ParseInt(val,10,64)
					if err != nil {
						panic("field " + name + " in " + *configFile + " cannot be read as a number")
					}
					fullConfig[name] = valint
				} else {
					fullConfig[name] = val
				}
			}
		}
	}

	//Now we look through each param. If it's set on the command-line (not to the default)
	//we set that in the global config maps. If it's also not set in the conf we set it
	//to the param's default. If it is set in the conf then it's already set in the global
	//configs by the previous section
	for name, param := range params {
		if param.Type == INT {
			cliVal := *cliConfig[name].(*int64)
			_, confSet := fullConfig[name]
			if cliVal != param.Default {
				fullConfig[name] = cliVal
			} else if !confSet {
				fullConfig[name] = param.Default.(int64)
			}
		} else {
			cliVal := *cliConfig[name].(*string)
			_, confSet := fullConfig[name]
			if cliVal != param.Default {
				fullConfig[name] = cliVal
			} else if !confSet {
				fullConfig[name] = param.Default.(string)
			}
		}
	}

}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: MIT

package web

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"os"
	"os/signal"
	"time"

	"golang.org/x/text/message/catalog"
	"gopkg.in/yaml.v2"

	"github.com/issue9/web/content"
	"github.com/issue9/web/serialization"
	"github.com/issue9/web/server"
)

// Command 提供一种简单的命令行处理方式
type Command struct {
	// 程序名称
	Name string

	// 程序版本
	Version string

	// 当作服务运行的标签名
	ServeTag string

	// 退出的信号
	//
	// 为 nil 表示没有，如果是 []，则表示所有信息。
	Signals []os.Signal

	CmdVersion string
	CmdTag     string
	CmdFS      string

	// 信息的输出通道
	//
	// 可以 os.Stdout 和 os.Stderr 选择，默认为 os.Stdout。
	Out io.Writer

	// 以下是初始 Server 对象的参数

	ResultBuilder content.BuildResultFunc // 默认为 content.DefaultBuilder
	Locale        *serialization.Locale   // 默认情况下，能正常解析 xml、yaml 和 json
	LogsFilename  string                  // 默认为 logs.xml
	WebFilename   string                  // 默认为 web.yaml
}

// Exec 执行命令行操作
func (cmd *Command) Exec() {
	if cmd.Out == nil {
		cmd.Out = os.Stdout
	}

	if cmd.Locale == nil {
		l := serialization.NewLocale(catalog.NewBuilder(), serialization.NewFiles(5))

		if err := l.Files().Add(json.Marshal, json.Unmarshal, ".json"); err != nil {
			panic(err)
		}

		if err := l.Files().Add(xml.Marshal, xml.Unmarshal, ".xml"); err != nil {
			panic(err)
		}

		if err := l.Files().Add(yaml.Marshal, yaml.Unmarshal, ".yaml", ".yml"); err != nil {
			panic(err)
		}
	}

	if cmd.LogsFilename == "" {
		cmd.LogsFilename = "logs.xml"
	}
	if cmd.WebFilename == "" {
		cmd.WebFilename = "web.yaml"
	}

	if cmd.CmdFS == "" {
		cmd.CmdFS = "fs"
	}
	if cmd.CmdTag == "" {
		cmd.CmdTag = "tag"
	}
	if cmd.CmdVersion == "" {
		cmd.CmdVersion = "v"
	}

	if err := cmd.exec(); err != nil {
		panic(err)
	}
}

func (cmd *Command) exec() error {
	v := flag.Bool(cmd.CmdVersion, false, "显示版本号")
	tag := flag.String(cmd.CmdTag, "", "执行的标签")
	f := flag.String(cmd.CmdFS, "./", "可读取的目录")
	dir := os.DirFS(*f)

	if *v {
		fmt.Fprintln(cmd.Out, cmd.Name, cmd.Version)
		return nil
	}

	srv, err := LoadServer(cmd.Name, cmd.Version, cmd.ResultBuilder, cmd.Locale, dir, cmd.LogsFilename, cmd.WebFilename)
	if err != nil {
		return err
	}

	if err := srv.InitModules(*tag); err != nil {
		return err
	}
	if *tag == cmd.ServeTag {
		if cmd.Signals != nil { // nil 和 [] 处理方式不同，[] 表示所有信息

			cmd.grace(srv, cmd.Signals...)
		}
		return srv.Serve()
	}
	return nil
}

func (cmd *Command) grace(s *server.Server, sig ...os.Signal) {
	go func() {
		signalChannel := make(chan os.Signal, 1)
		signal.Notify(signalChannel, sig...)

		<-signalChannel
		signal.Stop(signalChannel)
		close(signalChannel)

		if err := s.Close(3 * time.Second); err != nil {
			s.Logs().Error(err)
		}
		s.Logs().Flush() // 保证内容会被正常输出到日志。
	}()
}
<commit_msg>refactor: 拆分函数<commit_after>// SPDX-License-Identifier: MIT

package web

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"os"
	"os/signal"
	"time"

	"golang.org/x/text/message/catalog"
	"gopkg.in/yaml.v2"

	"github.com/issue9/web/content"
	"github.com/issue9/web/serialization"
	"github.com/issue9/web/server"
)

// Command 提供一种简单的命令行处理方式
//
// 由 Command 生成的命令行带以下三个参数：
//  - tag 运行的标签；
//  - v 显示版本号；
//  - fs 指定当前程序可读取的文件目录；
// 以上三个参数的参数名称，可在配置内容中修改。
type Command struct {
	// 程序名称
	Name string

	// 程序版本
	Version string

	// 当作服务运行的标签名
	ServeTag string

	// 退出的信号
	//
	// 为 nil 表示没有，如果是 []，则表示所有信息。
	Signals []os.Signal

	CmdVersion string
	CmdTag     string
	CmdFS      string

	// 信息的输出通道
	//
	// 可以 os.Stdout 和 os.Stderr 选择，默认为 os.Stdout。
	Out io.Writer

	// 以下是初始 Server 对象的参数

	ResultBuilder content.BuildResultFunc // 默认为 content.DefaultBuilder
	Locale        *serialization.Locale   // 默认情况下，能正常解析 xml、yaml 和 json
	LogsFilename  string                  // 默认为 logs.xml
	WebFilename   string                  // 默认为 web.yaml
}

// Exec 执行命令行操作
func (cmd *Command) Exec() {
	if err := cmd.sanitize(); err != nil {
		panic(err)
	}

	if err := cmd.exec(); err != nil {
		panic(err)
	}
}

func (cmd *Command) sanitize() error {
	if cmd.Out == nil {
		cmd.Out = os.Stdout
	}

	if cmd.Locale == nil {
		l := serialization.NewLocale(catalog.NewBuilder(), serialization.NewFiles(5))

		if err := l.Files().Add(json.Marshal, json.Unmarshal, ".json"); err != nil {
			return err
		}

		if err := l.Files().Add(xml.Marshal, xml.Unmarshal, ".xml"); err != nil {
			return err
		}

		if err := l.Files().Add(yaml.Marshal, yaml.Unmarshal, ".yaml", ".yml"); err != nil {
			return err
		}
	}

	if cmd.LogsFilename == "" {
		cmd.LogsFilename = "logs.xml"
	}
	if cmd.WebFilename == "" {
		cmd.WebFilename = "web.yaml"
	}

	if cmd.CmdFS == "" {
		cmd.CmdFS = "fs"
	}
	if cmd.CmdTag == "" {
		cmd.CmdTag = "tag"
	}
	if cmd.CmdVersion == "" {
		cmd.CmdVersion = "v"
	}

	return nil
}

func (cmd *Command) exec() error {
	v := flag.Bool(cmd.CmdVersion, false, "显示版本号")
	tag := flag.String(cmd.CmdTag, "", "执行的标签")
	f := flag.String(cmd.CmdFS, "./", "可读取的目录")
	dir := os.DirFS(*f)

	if *v {
		fmt.Fprintln(cmd.Out, cmd.Name, cmd.Version)
		return nil
	}

	srv, err := LoadServer(cmd.Name, cmd.Version, cmd.ResultBuilder, cmd.Locale, dir, cmd.LogsFilename, cmd.WebFilename)
	if err != nil {
		return err
	}

	if err := srv.InitModules(*tag); err != nil {
		return err
	}
	if *tag == cmd.ServeTag {
		if cmd.Signals != nil { // nil 和 [] 处理方式不同，[] 表示所有信息

			cmd.grace(srv, cmd.Signals...)
		}
		return srv.Serve()
	}
	return nil
}

func (cmd *Command) grace(s *server.Server, sig ...os.Signal) {
	go func() {
		signalChannel := make(chan os.Signal, 1)
		signal.Notify(signalChannel, sig...)

		<-signalChannel
		signal.Stop(signalChannel)
		close(signalChannel)

		if err := s.Close(3 * time.Second); err != nil {
			s.Logs().Error(err)
		}
		s.Logs().Flush() // 保证内容会被正常输出到日志。
	}()
}
<|endoftext|>