<commit_before>// Package forms provides support for HTML forms and its input elements.
package forms

import (
	"net/http"
	"strconv"
	"strings"

	"github.com/ChristianSiegert/go-packages/html/elements"
	"github.com/ChristianSiegert/go-packages/validation"
)

type Form struct {
	request *http.Request

	// ValidationFields is a map of field names and their corresponding
	// validation.Item. Used to get information about the items’ validation
	// rules.
	ValidationFields validation.Items

	// ValidationMessages contains a validation error message for each form
	// field that has an invalid value.
	ValidationMessages validation.Messages
}

func New(request *http.Request) (*Form, error) {
	return &Form{
		request: request,
	}, nil
}

// Error returns a
 element that contains the
// validation error message as text.
func (f *Form) Error(fieldName string) *elements.Element {
	validationMessage, ok := f.ValidationMessages[fieldName]
	if !ok || validationMessage == "" {
		return nil
	}

	element := &elements.Element{
		Attributes: map[string]string{
			"class": "validation-error",
		},
		HasEndTag: true,
		TagName:   "div",
		Text:      validationMessage,
	}
	return element
}

// HasError returns whether the field’s value is invalid.
func (f *Form) HasError(fieldName string) bool {
	_, ok := f.ValidationMessages[fieldName]
	return ok
}

// Input returns an <input> element.
func (f *Form) Input(fieldName, placeholder string, attributes ...string) *elements.Element {
	element := &elements.Element{
		Attributes: map[string]string{
			"id":   fieldName,
			"name": fieldName,
		},
		TagName: "input",
	}

	if f.HasError(fieldName) {
		element.Attributes["class"] = "error"
	}

	if placeholder != "" {
		element.Attributes["placeholder"] = placeholder
	}

	if value := f.request.FormValue(fieldName); value != "" {
		element.Attributes["value"] = strings.TrimSpace(value)
	}

	for i, length := 0, len(attributes); i < length; i += 2 {
		if i+1 < length {
			element.AddAttributeValue(attributes[i], attributes[i+1])
		} else {
			element.AddAttributeValue(attributes[i], "")
		}
	}

	if f.ValidationFields == nil {
		return element
	}

	if field, ok := f.ValidationFields[fieldName]; ok {
		for _, rule := range field.Rules {
			if rule.Type == validation.RuleTypeRequired {
				element.Attributes["required"] = ""
			} else if rule.Type == validation.RuleTypeMaxLength {
				if maxLength, ok := rule.Args[0].(int); ok && maxLength > 0 {
					element.Attributes["maxlength"] = strconv.FormatUint(uint64(maxLength), 10)
				}
			} else if rule.Type == validation.RuleTypeMinLength {
				if minLength, ok := rule.Args[0].(int); ok && minLength > 0 {
					element.Attributes["minlength"] = strconv.FormatUint(uint64(minLength), 10)
				}
			}
		}
	}

	return element
}

// Checkbox returns an <input type="checkbox"> element.
func (f *Form) Checkbox(fieldName, value string) *elements.Element {
	element := f.Input(fieldName, "")
	element.Attributes["id"] = fieldName + "-" + value
	element.Attributes["type"] = "checkbox"
	element.Attributes["value"] = value

	if values, ok := f.request.Form[fieldName]; ok {
		for _, v := range values {
			if v == value {
				element.Attributes["checked"] = ""
				break
			}
		}
	}

	return element
}

// Email returns an <input type="email"> element.
func (f *Form) Email(fieldName, placeholder string) *elements.Element {
	element := f.Input(fieldName, placeholder)
	element.Attributes["type"] = "email"

	if _, ok := element.Attributes["maxlength"]; !ok {
		element.Attributes["maxlength"] = "254"
	}
	return element
}

// Number returns an <input type="number"> element.
func (f *Form) Number(fieldName, placeholder string, attributes ...string) *elements.Element {
	element := f.Input(fieldName, placeholder, attributes...)
	element.Attributes["type"] = "number"
	return element
}

// Radio returns an <input type="radio"> element.
func (f *Form) Radio(fieldName, value string) *elements.Element {
	element := f.Input(fieldName, "")
	element.Attributes["id"] = fieldName + "-" + value
	element.Attributes["type"] = "radio"
	element.Attributes["value"] = value

	if f.request.FormValue(fieldName) == value {
		element.Attributes["checked"] = ""
	}
	return element
}

// Password returns an <input type="password"> element.
func (f *Form) Password(fieldName, placeholder string) *elements.Element {
	element := f.Input(fieldName, placeholder)
	element.Attributes["type"] = "password"
	return element
}

// Search returns an <input type="search"> element.
func (f *Form) Search(fieldName, placeholder string, attributes ...string) *elements.Element {
	element := f.Input(fieldName, placeholder, attributes...)
	element.Attributes["type"] = "search"
	return element
}

// Text returns an <input type="text"> element.
func (f *Form) Text(fieldName, placeholder string, attributes ...string) *elements.Element {
	element := f.Input(fieldName, placeholder, attributes...)
	element.Attributes["type"] = "text"
	return element
}

// Select returns a <select> element.
func (f *Form) Select(fieldName string, options []*Option) *elements.Element {
	element := &elements.Element{
		Attributes: map[string]string{
			"id":   fieldName,
			"name": fieldName,
		},
		HasEndTag: true,
		TagName:   "select",
	}

	if len(options) > 0 {
		element.Children = make([]*elements.Element, 0, len(options))

		for _, option := range options {
			optionElement := f.Option(option.Value, option.Label)
			element.Children = append(element.Children, optionElement)

			if postedValue := f.request.FormValue(fieldName); postedValue == option.Value {
				optionElement.SetAttributeValue("selected", "")
			}
		}
	}

	if f.HasError(fieldName) {
		element.Attributes["class"] = "error"
	}

	return element
}

// Option returns an <option> element.
func (f *Form) Option(value, label string) *elements.Element {
	return &elements.Element{
		Attributes: map[string]string{
			"value": value,
		},
		HasEndTag: true,
		TagName:   "option",
		Text:      label,
	}
}

// Textarea returns a <textarea> element.
func (f *Form) Textarea(fieldName, placeholder string) *elements.Element {
	element := &elements.Element{
		Attributes: map[string]string{
			"id":   fieldName,
			"name": fieldName,
		},
		HasEndTag: true,
		TagName:   "textarea",
	}

	if f.HasError(fieldName) {
		element.Attributes["class"] = "error"
	}

	if placeholder != "" {
		element.Attributes["placeholder"] = placeholder
	}

	if value := f.request.FormValue(fieldName); value != "" {
		element.Text = strings.TrimSpace(value)
	}

	if f.ValidationFields == nil {
		return element
	}

	if field, ok := f.ValidationFields[fieldName]; ok {
		for _, rule := range field.Rules {
			if rule.Type == validation.RuleTypeRequired {
				element.Attributes["required"] = ""
			} else if rule.Type == validation.RuleTypeMaxLength {
				if maxLength, ok := rule.Args[0].(int); ok && maxLength > 0 {
					element.Attributes["maxlength"] = strconv.FormatUint(uint64(maxLength), 10)
				}
			} else if rule.Type == validation.RuleTypeMinLength {
				if minLength, ok := rule.Args[0].(int); ok && minLength > 0 {
					element.Attributes["minlength"] = strconv.FormatUint(uint64(minLength), 10)
				}
			}
		}
	}

	return element
}
<commit_msg>Fixed not initializing Form.ValidationMessages.<commit_after>// Package forms provides support for HTML forms and its input elements.
package forms

import (
	"net/http"
	"strconv"
	"strings"

	"github.com/ChristianSiegert/go-packages/html/elements"
	"github.com/ChristianSiegert/go-packages/validation"
)

// Form represents an HTML form.
type Form struct {
	request *http.Request

	// ValidationFields is a map of field names and their corresponding
	// validation.Item. Used to get information about the items’ validation
	// rules.
	ValidationFields validation.Items

	// ValidationMessages contains a validation error message for each form
	// field that has an invalid value.
	ValidationMessages validation.Messages
}

// New returns a new instance of Form.
func New(request *http.Request) (*Form, error) {
	return &Form{
		request:            request,
		ValidationMessages: validation.Messages{},
	}, nil
}

// Error returns a
 element that contains the
// validation error message as text.
func (f *Form) Error(fieldName string) *elements.Element {
	validationMessage, ok := f.ValidationMessages[fieldName]
	if !ok || validationMessage == "" {
		return nil
	}

	element := &elements.Element{
		Attributes: map[string]string{
			"class": "validation-error",
		},
		HasEndTag: true,
		TagName:   "div",
		Text:      validationMessage,
	}
	return element
}

// HasError returns whether the field’s value is invalid.
func (f *Form) HasError(fieldName string) bool {
	_, ok := f.ValidationMessages[fieldName]
	return ok
}

// Input returns an <input> element.
func (f *Form) Input(fieldName, placeholder string, attributes ...string) *elements.Element {
	element := &elements.Element{
		Attributes: map[string]string{
			"id":   fieldName,
			"name": fieldName,
		},
		TagName: "input",
	}

	if f.HasError(fieldName) {
		element.Attributes["class"] = "error"
	}

	if placeholder != "" {
		element.Attributes["placeholder"] = placeholder
	}

	if value := f.request.FormValue(fieldName); value != "" {
		element.Attributes["value"] = strings.TrimSpace(value)
	}

	for i, length := 0, len(attributes); i < length; i += 2 {
		if i+1 < length {
			element.AddAttributeValue(attributes[i], attributes[i+1])
		} else {
			element.AddAttributeValue(attributes[i], "")
		}
	}

	if f.ValidationFields == nil {
		return element
	}

	if field, ok := f.ValidationFields[fieldName]; ok {
		for _, rule := range field.Rules {
			if rule.Type == validation.RuleTypeRequired {
				element.Attributes["required"] = ""
			} else if rule.Type == validation.RuleTypeMaxLength {
				if maxLength, ok := rule.Args[0].(int); ok && maxLength > 0 {
					element.Attributes["maxlength"] = strconv.FormatUint(uint64(maxLength), 10)
				}
			} else if rule.Type == validation.RuleTypeMinLength {
				if minLength, ok := rule.Args[0].(int); ok && minLength > 0 {
					element.Attributes["minlength"] = strconv.FormatUint(uint64(minLength), 10)
				}
			}
		}
	}

	return element
}

// Checkbox returns an <input type="checkbox"> element.
func (f *Form) Checkbox(fieldName, value string) *elements.Element {
	element := f.Input(fieldName, "")
	element.Attributes["id"] = fieldName + "-" + value
	element.Attributes["type"] = "checkbox"
	element.Attributes["value"] = value

	if values, ok := f.request.Form[fieldName]; ok {
		for _, v := range values {
			if v == value {
				element.Attributes["checked"] = ""
				break
			}
		}
	}

	return element
}

// Email returns an <input type="email"> element.
func (f *Form) Email(fieldName, placeholder string) *elements.Element {
	element := f.Input(fieldName, placeholder)
	element.Attributes["type"] = "email"

	if _, ok := element.Attributes["maxlength"]; !ok {
		element.Attributes["maxlength"] = "254"
	}
	return element
}

// Number returns an <input type="number"> element.
func (f *Form) Number(fieldName, placeholder string, attributes ...string) *elements.Element {
	element := f.Input(fieldName, placeholder, attributes...)
	element.Attributes["type"] = "number"
	return element
}

// Radio returns an <input type="radio"> element.
func (f *Form) Radio(fieldName, value string) *elements.Element {
	element := f.Input(fieldName, "")
	element.Attributes["id"] = fieldName + "-" + value
	element.Attributes["type"] = "radio"
	element.Attributes["value"] = value

	if f.request.FormValue(fieldName) == value {
		element.Attributes["checked"] = ""
	}
	return element
}

// Password returns an <input type="password"> element.
func (f *Form) Password(fieldName, placeholder string) *elements.Element {
	element := f.Input(fieldName, placeholder)
	element.Attributes["type"] = "password"
	return element
}

// Search returns an <input type="search"> element.
func (f *Form) Search(fieldName, placeholder string, attributes ...string) *elements.Element {
	element := f.Input(fieldName, placeholder, attributes...)
	element.Attributes["type"] = "search"
	return element
}

// Text returns an <input type="text"> element.
func (f *Form) Text(fieldName, placeholder string, attributes ...string) *elements.Element {
	element := f.Input(fieldName, placeholder, attributes...)
	element.Attributes["type"] = "text"
	return element
}

// Select returns a <select> element.
func (f *Form) Select(fieldName string, options []*Option) *elements.Element {
	element := &elements.Element{
		Attributes: map[string]string{
			"id":   fieldName,
			"name": fieldName,
		},
		HasEndTag: true,
		TagName:   "select",
	}

	if len(options) > 0 {
		element.Children = make([]*elements.Element, 0, len(options))

		for _, option := range options {
			optionElement := f.Option(option.Value, option.Label)
			element.Children = append(element.Children, optionElement)

			if postedValue := f.request.FormValue(fieldName); postedValue == option.Value {
				optionElement.SetAttributeValue("selected", "")
			}
		}
	}

	if f.HasError(fieldName) {
		element.Attributes["class"] = "error"
	}

	return element
}

// Option returns an <option> element.
func (f *Form) Option(value, label string) *elements.Element {
	return &elements.Element{
		Attributes: map[string]string{
			"value": value,
		},
		HasEndTag: true,
		TagName:   "option",
		Text:      label,
	}
}

// Textarea returns a <textarea> element.
func (f *Form) Textarea(fieldName, placeholder string) *elements.Element {
	element := &elements.Element{
		Attributes: map[string]string{
			"id":   fieldName,
			"name": fieldName,
		},
		HasEndTag: true,
		TagName:   "textarea",
	}

	if f.HasError(fieldName) {
		element.Attributes["class"] = "error"
	}

	if placeholder != "" {
		element.Attributes["placeholder"] = placeholder
	}

	if value := f.request.FormValue(fieldName); value != "" {
		element.Text = strings.TrimSpace(value)
	}

	if f.ValidationFields == nil {
		return element
	}

	if field, ok := f.ValidationFields[fieldName]; ok {
		for _, rule := range field.Rules {
			if rule.Type == validation.RuleTypeRequired {
				element.Attributes["required"] = ""
			} else if rule.Type == validation.RuleTypeMaxLength {
				if maxLength, ok := rule.Args[0].(int); ok && maxLength > 0 {
					element.Attributes["maxlength"] = strconv.FormatUint(uint64(maxLength), 10)
				}
			} else if rule.Type == validation.RuleTypeMinLength {
				if minLength, ok := rule.Args[0].(int); ok && minLength > 0 {
					element.Attributes["minlength"] = strconv.FormatUint(uint64(minLength), 10)
				}
			}
		}
	}

	return element
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package routines

import (
	"context"
	"flag"
	"time"

	"github.com/golang/glog"
	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta1"
	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
	vpa_api "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta1"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/checkpoint"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/logic"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model"
	metrics_recommender "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/recommender"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
	"k8s.io/client-go/rest"
)

const (
	// AggregateContainerStateGCInterval defines how often expired AggregateContainerStates are garbage collected.
	AggregateContainerStateGCInterval = 1 * time.Hour
)

var (
	checkpointsWriteTimeout = flag.Duration("checkpoints-timeout", time.Minute, `Timeout for writing checkpoints since the start of the recommender's main loop`)
	minCheckpointsPerRun    = flag.Int("min-checkpoints", 10, "Minimum number of checkpoints to write per recommender's main loop")
)

// Recommender recommend resources for certain containers, based on utilization periodically got from metrics api.
type Recommender interface {
	// RunOnce performs one iteration of recommender duties followed by update of recommendations in VPA objects.
	RunOnce()
	// GetClusterState returns ClusterState used by Recommender
	GetClusterState() *model.ClusterState
	// GetClusterStateFeeder returns ClusterStateFeeder used by Recommender
	GetClusterStateFeeder() input.ClusterStateFeeder
	// UpdateVPAs computes recommendations and sends VPAs status updates to API Server
	UpdateVPAs()
	// MaintainCheckpoints stores current checkpoints in API Server and garbage collect old ones
	// MaintainCheckpoints writes at least minCheckpoints if there are more checkponits to write.
	// Checkpoints are written until ctx permits or all checkpoints are written.
	MaintainCheckpoints(ctx context.Context, minCheckpoints int)
	// GarbageCollect removes old AggregateCollectionStates
	GarbageCollect()
}

type recommender struct {
	clusterState                  *model.ClusterState
	clusterStateFeeder            input.ClusterStateFeeder
	checkpointWriter              checkpoint.CheckpointWriter
	checkpointsGCInterval         time.Duration
	lastCheckpointGC              time.Time
	vpaClient                     vpa_api.VerticalPodAutoscalersGetter
	podResourceRecommender        logic.PodResourceRecommender
	useCheckpoints                bool
	lastAggregateContainerStateGC time.Time
}

func (r *recommender) GetClusterState() *model.ClusterState {
	return r.clusterState
}

func (r *recommender) GetClusterStateFeeder() input.ClusterStateFeeder {
	return r.clusterStateFeeder
}

// Updates VPA CRD objects' statuses.
func (r *recommender) UpdateVPAs() {
	cnt := metrics_recommender.NewObjectCounter()
	defer cnt.Observe()

	for _, observedVpa := range r.clusterState.ObservedVpas {
		key := model.VpaID{
			Namespace: observedVpa.Namespace,
			VpaName:   observedVpa.Name}

		vpa, found := r.clusterState.Vpas[key]
		if !found {
			continue
		}
		resources := r.podResourceRecommender.GetRecommendedPodResources(GetContainerNameToAggregateStateMap(vpa))
		containerResources := make([]vpa_types.RecommendedContainerResources, 0, len(resources))
		for containerName, res := range resources {
			containerResources = append(containerResources, vpa_types.RecommendedContainerResources{
				ContainerName: containerName,
				Target:        model.ResourcesAsResourceList(res.Target),
				LowerBound:    model.ResourcesAsResourceList(res.LowerBound),
				UpperBound:    model.ResourcesAsResourceList(res.UpperBound),
			})

		}
		had := vpa.HasRecommendation()
		vpa.Recommendation = &vpa_types.RecommendedPodResources{containerResources}
		// Set RecommendationProvided if recommendation not empty.
		if len(vpa.Recommendation.ContainerRecommendations) > 0 {
			vpa.Conditions.Set(vpa_types.RecommendationProvided, true, "", "")
			if !had {
				metrics_recommender.ObserveRecommendationLatency(vpa.Created)
			}
		}
		cnt.Add(vpa)

		_, err := vpa_api_util.UpdateVpaStatusIfNeeded(
			r.vpaClient.VerticalPodAutoscalers(vpa.ID.Namespace), vpa, &observedVpa.Status)
		if err != nil {
			glog.Errorf(
				"Cannot update VPA %v object. Reason: %+v", vpa.ID.VpaName, err)
		}
	}
}

func (r *recommender) MaintainCheckpoints(ctx context.Context, minCheckpointsPerRun int) {
	now := time.Now()
	if r.useCheckpoints {
		r.checkpointWriter.StoreCheckpoints(ctx, now, minCheckpointsPerRun)
		if time.Now().Sub(r.lastCheckpointGC) > r.checkpointsGCInterval {
			r.lastCheckpointGC = now
			r.clusterStateFeeder.GarbageCollectCheckpoints()
		}
	}

}

func (r *recommender) GarbageCollect() {
	gcTime := time.Now()
	if gcTime.Sub(r.lastAggregateContainerStateGC) > AggregateContainerStateGCInterval {
		r.clusterState.GarbageCollectAggregateCollectionStates(gcTime)
		r.lastAggregateContainerStateGC = gcTime
	}
}

func (r *recommender) RunOnce() {
	timer := metrics_recommender.NewExecutionTimer()
	defer timer.ObserveTotal()

	ctx := context.Background()
	ctx, cancelFunc := context.WithDeadline(ctx, time.Now().Add(*checkpointsWriteTimeout))
	defer cancelFunc()

	glog.V(3).Infof("Recommender Run")
	r.clusterStateFeeder.LoadVPAs()
	timer.ObserveStep("LoadVPAs")
	r.clusterStateFeeder.LoadPods()
	timer.ObserveStep("LoadPods")
	r.clusterStateFeeder.LoadRealTimeMetrics()
	timer.ObserveStep("LoadMetrics")
	glog.V(3).Infof("ClusterState is tracking %v PodStates and %v VPAs", len(r.clusterState.Pods), len(r.clusterState.Vpas))

	r.UpdateVPAs()
	timer.ObserveStep("UpdateVPAs")

	r.MaintainCheckpoints(ctx, *minCheckpointsPerRun)
	timer.ObserveStep("MaintainCheckpoints")

	r.GarbageCollect()
	timer.ObserveStep("GarbageCollect")
}

// NewRecommender creates a new recommender instance,
// which can be run in order to provide continuous resource recommendations for containers.
// It requires cluster configuration object and duration between recommender intervals.
func NewRecommender(config *rest.Config, checkpointsGCInterval time.Duration, useCheckpoints bool) Recommender {
	clusterState := model.NewClusterState()
	recommender := &recommender{
		clusterState:                  clusterState,
		clusterStateFeeder:            input.NewClusterStateFeeder(config, clusterState),
		checkpointWriter:              checkpoint.NewCheckpointWriter(clusterState, vpa_clientset.NewForConfigOrDie(config).AutoscalingV1beta1()),
		checkpointsGCInterval:         checkpointsGCInterval,
		lastCheckpointGC:              time.Now(),
		vpaClient:                     vpa_clientset.NewForConfigOrDie(config).AutoscalingV1beta1(),
		podResourceRecommender:        logic.CreatePodResourceRecommender(),
		useCheckpoints:                useCheckpoints,
		lastAggregateContainerStateGC: time.Now(),
	}
	glog.V(3).Infof("New Recommender created %+v", recommender)
	return recommender
}
<commit_msg>Add factory for Recommender<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package routines

import (
	"context"
	"flag"
	"time"

	"github.com/golang/glog"
	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta1"
	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
	vpa_api "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1beta1"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/checkpoint"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/logic"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model"
	metrics_recommender "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/recommender"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
	"k8s.io/client-go/rest"
)

const (
	// AggregateContainerStateGCInterval defines how often expired AggregateContainerStates are garbage collected.
	AggregateContainerStateGCInterval = 1 * time.Hour
)

var (
	checkpointsWriteTimeout = flag.Duration("checkpoints-timeout", time.Minute, `Timeout for writing checkpoints since the start of the recommender's main loop`)
	minCheckpointsPerRun    = flag.Int("min-checkpoints", 10, "Minimum number of checkpoints to write per recommender's main loop")
)

// Recommender recommend resources for certain containers, based on utilization periodically got from metrics api.
type Recommender interface {
	// RunOnce performs one iteration of recommender duties followed by update of recommendations in VPA objects.
	RunOnce()
	// GetClusterState returns ClusterState used by Recommender
	GetClusterState() *model.ClusterState
	// GetClusterStateFeeder returns ClusterStateFeeder used by Recommender
	GetClusterStateFeeder() input.ClusterStateFeeder
	// UpdateVPAs computes recommendations and sends VPAs status updates to API Server
	UpdateVPAs()
	// MaintainCheckpoints stores current checkpoints in API Server and garbage collect old ones
	// MaintainCheckpoints writes at least minCheckpoints if there are more checkponits to write.
	// Checkpoints are written until ctx permits or all checkpoints are written.
	MaintainCheckpoints(ctx context.Context, minCheckpoints int)
	// GarbageCollect removes old AggregateCollectionStates
	GarbageCollect()
}

type recommender struct {
	clusterState                  *model.ClusterState
	clusterStateFeeder            input.ClusterStateFeeder
	checkpointWriter              checkpoint.CheckpointWriter
	checkpointsGCInterval         time.Duration
	lastCheckpointGC              time.Time
	vpaClient                     vpa_api.VerticalPodAutoscalersGetter
	podResourceRecommender        logic.PodResourceRecommender
	useCheckpoints                bool
	lastAggregateContainerStateGC time.Time
}

func (r *recommender) GetClusterState() *model.ClusterState {
	return r.clusterState
}

func (r *recommender) GetClusterStateFeeder() input.ClusterStateFeeder {
	return r.clusterStateFeeder
}

// Updates VPA CRD objects' statuses.
func (r *recommender) UpdateVPAs() {
	cnt := metrics_recommender.NewObjectCounter()
	defer cnt.Observe()

	for _, observedVpa := range r.clusterState.ObservedVpas {
		key := model.VpaID{
			Namespace: observedVpa.Namespace,
			VpaName:   observedVpa.Name}

		vpa, found := r.clusterState.Vpas[key]
		if !found {
			continue
		}
		resources := r.podResourceRecommender.GetRecommendedPodResources(GetContainerNameToAggregateStateMap(vpa))
		containerResources := make([]vpa_types.RecommendedContainerResources, 0, len(resources))
		for containerName, res := range resources {
			containerResources = append(containerResources, vpa_types.RecommendedContainerResources{
				ContainerName: containerName,
				Target:        model.ResourcesAsResourceList(res.Target),
				LowerBound:    model.ResourcesAsResourceList(res.LowerBound),
				UpperBound:    model.ResourcesAsResourceList(res.UpperBound),
			})

		}
		had := vpa.HasRecommendation()
		vpa.Recommendation = &vpa_types.RecommendedPodResources{containerResources}
		// Set RecommendationProvided if recommendation not empty.
		if len(vpa.Recommendation.ContainerRecommendations) > 0 {
			vpa.Conditions.Set(vpa_types.RecommendationProvided, true, "", "")
			if !had {
				metrics_recommender.ObserveRecommendationLatency(vpa.Created)
			}
		}
		cnt.Add(vpa)

		_, err := vpa_api_util.UpdateVpaStatusIfNeeded(
			r.vpaClient.VerticalPodAutoscalers(vpa.ID.Namespace), vpa, &observedVpa.Status)
		if err != nil {
			glog.Errorf(
				"Cannot update VPA %v object. Reason: %+v", vpa.ID.VpaName, err)
		}
	}
}

func (r *recommender) MaintainCheckpoints(ctx context.Context, minCheckpointsPerRun int) {
	now := time.Now()
	if r.useCheckpoints {
		r.checkpointWriter.StoreCheckpoints(ctx, now, minCheckpointsPerRun)
		if time.Now().Sub(r.lastCheckpointGC) > r.checkpointsGCInterval {
			r.lastCheckpointGC = now
			r.clusterStateFeeder.GarbageCollectCheckpoints()
		}
	}

}

func (r *recommender) GarbageCollect() {
	gcTime := time.Now()
	if gcTime.Sub(r.lastAggregateContainerStateGC) > AggregateContainerStateGCInterval {
		r.clusterState.GarbageCollectAggregateCollectionStates(gcTime)
		r.lastAggregateContainerStateGC = gcTime
	}
}

func (r *recommender) RunOnce() {
	timer := metrics_recommender.NewExecutionTimer()
	defer timer.ObserveTotal()

	ctx := context.Background()
	ctx, cancelFunc := context.WithDeadline(ctx, time.Now().Add(*checkpointsWriteTimeout))
	defer cancelFunc()

	glog.V(3).Infof("Recommender Run")

	r.clusterStateFeeder.LoadVPAs()
	timer.ObserveStep("LoadVPAs")

	r.clusterStateFeeder.LoadPods()
	timer.ObserveStep("LoadPods")

	r.clusterStateFeeder.LoadRealTimeMetrics()
	timer.ObserveStep("LoadMetrics")
	glog.V(3).Infof("ClusterState is tracking %v PodStates and %v VPAs", len(r.clusterState.Pods), len(r.clusterState.Vpas))

	r.UpdateVPAs()
	timer.ObserveStep("UpdateVPAs")

	r.MaintainCheckpoints(ctx, *minCheckpointsPerRun)
	timer.ObserveStep("MaintainCheckpoints")

	r.GarbageCollect()
	timer.ObserveStep("GarbageCollect")
}

// RecommenderFactory makes instances of Recommender.
type RecommenderFactory struct {
	ClusterState *model.ClusterState

	ClusterStateFeeder     input.ClusterStateFeeder
	CheckpointWriter       checkpoint.CheckpointWriter
	PodResourceRecommender logic.PodResourceRecommender
	VpaClient              vpa_api.VerticalPodAutoscalersGetter

	CheckpointsGCInterval time.Duration
	UseCheckpoints        bool
}

// Make creates a new recommender instance,
// which can be run in order to provide continuous resource recommendations for containers.
func (c RecommenderFactory) Make() Recommender {
	recommender := &recommender{
		clusterState:                  c.ClusterState,
		clusterStateFeeder:            c.ClusterStateFeeder,
		checkpointWriter:              c.CheckpointWriter,
		checkpointsGCInterval:         c.CheckpointsGCInterval,
		useCheckpoints:                c.UseCheckpoints,
		vpaClient:                     c.VpaClient,
		podResourceRecommender:        c.PodResourceRecommender,
		lastAggregateContainerStateGC: time.Now(),
		lastCheckpointGC:              time.Now(),
	}
	glog.V(3).Infof("New Recommender created %+v", recommender)
	return recommender
}

// NewRecommender creates a new recommender instance.
// Dependencies are created automatically.
// Deprecated; use RecommenderFactory instead.
func NewRecommender(config *rest.Config, checkpointsGCInterval time.Duration, useCheckpoints bool) Recommender {
	clusterState := model.NewClusterState()
	return RecommenderFactory{
		ClusterState:           clusterState,
		ClusterStateFeeder:     input.NewClusterStateFeeder(config, clusterState),
		CheckpointWriter:       checkpoint.NewCheckpointWriter(clusterState, vpa_clientset.NewForConfigOrDie(config).AutoscalingV1beta1()),
		VpaClient:              vpa_clientset.NewForConfigOrDie(config).AutoscalingV1beta1(),
		PodResourceRecommender: logic.CreatePodResourceRecommender(),
		CheckpointsGCInterval:  checkpointsGCInterval,
		UseCheckpoints:         useCheckpoints,
	}.Make()
}
<|endoftext|>


<commit_before>package main

func main() {

}
<commit_msg>star main flow<commit_after>package main

import (
	"net/http"

	"github.com/LeoCBS/garden/parameter"
	"github.com/LeoCBS/garden/storage"
)

func main() {
	// TODO get param from cli ou service discovery
	s := storage.NewStorer("mongodb://localhost:27017", "garden", "parameters")
	p := parameter.NewParameter(s)
	mux := server.NewSever(p)
	http.ListenAndServe(":3000", mux)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/tylerb/graceful"
	"github.com/yosisa/sigm"
	"github.com/yosisa/webutil"
)

var (
	dataDir         = flag.String("data-dir", "./data", "Data directory")
	listen          = flag.String("listen", ":8000", "Listen address")
	gracefulTimeout = flag.Duration("graceful-timeout", 10*time.Second, "Wait until force shutdown")
	gcInterval      = flag.Duration("gc-interval", time.Hour, "GC interval for cleaning deleted files")
	accessLog       = flag.String("access-log", "-", "Path to access log file")
)

var (
	accessLogWriter = new(webutil.ConsoleLogWriter)
	middlewares     []*middleware
)

const tombstone = ".restfs-deleted"

type middleware struct {
	priority int
	wrap     func(h http.Handler) http.Handler
}

type byPriority []*middleware

func (x byPriority) Len() int           { return len(x) }
func (x byPriority) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
func (x byPriority) Less(i, j int) bool { return x[i].priority < x[j].priority }

func registerMiddleware(priority int, wrap func(http.Handler) http.Handler) {
	middlewares = append(middlewares, &middleware{priority: priority, wrap: wrap})
}

type restfs struct {
	dir string
}

func (c *restfs) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fullpath := path.Join(c.dir, r.URL.Path)
	var (
		fi  os.FileInfo
		err error
	)
	switch r.Method {
	case "GET":
		s := stat(fullpath)
		if s == nil {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}
		if s.IsDir() {
			serveFileList(w, fullpath)
		} else {
			http.ServeFile(w, r, fullpath)
		}
		return
	case "PUT":
		fi, err = os.Stat(fullpath)
		if err == nil && fi.IsDir() {
			http.Error(w, "Cannot overwrite directory", http.StatusBadRequest)
			return
		}
		if err == nil || os.IsNotExist(err) {
			err = c.saveFile(fullpath, r.Body)
			r.Body.Close()
		}
	case "DELETE":
		fi, err = os.Stat(fullpath)
		if err == nil {
			if fi.IsDir() {
				recursive, _ := strconv.ParseBool(r.URL.Query().Get("recursive"))
				if recursive {
					err = c.removeAll(fullpath)
				} else {
					http.Error(w, "Cannot remove directory; forgot recursive=true?", http.StatusBadRequest)
					return
				}
			} else {
				err = c.remove(fullpath)
			}
		} else if os.IsNotExist(err) {
			return
		}
	default:
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
}

func (c *restfs) saveFile(fullpath string, r io.Reader) error {
	dir, _ := path.Split(fullpath)
	if err := os.MkdirAll(dir, 0777); err != nil {
		return err
	}
	f, err := os.OpenFile(fullpath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666)
	if err != nil {
		return err
	}
	if _, err := io.Copy(f, r); err != nil {
		return err
	}
	return nil
}

func (c *restfs) remove(fullpath string) error {
	f, err := os.Create(fullpath + tombstone)
	if err == nil {
		f.Close()
	}
	return err
}

func (c *restfs) removeAll(fullpath string) error {
	return filepath.Walk(fullpath, func(name string, stat os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if stat.IsDir() || strings.HasSuffix(name, tombstone) {
			return nil
		}
		return c.remove(name)
	})
}

type gc struct {
	dir    string
	invoke chan struct{}
}

func newGC(dir string) *gc {
	g := &gc{
		dir:    dir,
		invoke: make(chan struct{}, 1),
	}
	go g.loop()
	return g
}

func (g *gc) loop() {
	remove := func(s string) error {
		log.Printf("Remove %s", s)
		if err := os.Remove(s); err != nil && !os.IsNotExist(err) {
			return err
		}
		return nil
	}
	for range g.invoke {
		log.Print("GC started")
		start := time.Now()
		err := filepath.Walk(g.dir, func(name string, stat os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if stat.IsDir() || !strings.HasSuffix(name, tombstone) {
				return nil
			}
			fname := name[:len(name)-len(tombstone)]
			fstat, err := os.Stat(fname)
			if err == nil {
				if !fstat.ModTime().After(stat.ModTime()) {
					if err = remove(fname); err != nil {
						return err
					}
				}
				return remove(name)
			} else if os.IsNotExist(err) {
				return remove(name)
			}
			return err
		})
		took := time.Since(start)
		if err == nil {
			log.Printf("GC has finished in %v", took)
		} else {
			log.Printf("GC has aborted in %v with error: %v", took, err)
		}
	}
}

func (g *gc) Start() {
	select {
	case g.invoke <- struct{}{}:
	default:
	}
}

func stat(fullpath string) os.FileInfo {
	astat, err := os.Stat(fullpath)
	if err != nil {
		return nil
	}
	if astat.IsDir() {
		return astat
	}

	bstat, err := os.Stat(fullpath + tombstone)
	if err != nil {
		if os.IsNotExist(err) {
			return astat
		}
		log.Print(err)
		return nil
	}
	if astat.ModTime().After(bstat.ModTime()) {
		return astat
	}
	return nil
}

func serveFileList(w http.ResponseWriter, s string) {
	fis, err := ioutil.ReadDir(s)
	if err != nil {
		log.Print(err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	tombstones := make(map[string]os.FileInfo)
	for _, fi := range fis {
		name := fi.Name()
		if strings.HasSuffix(name, tombstone) {
			name = name[:len(name)-len(tombstone)]
			tombstones[name] = fi
		}
	}

	for _, fi := range fis {
		name := fi.Name()
		if strings.HasSuffix(name, tombstone) {
			continue
		}
		if fi.IsDir() {
			name += "/"
		} else if ts := tombstones[name]; ts != nil && !fi.ModTime().After(ts.ModTime()) {
			continue
		}
		fmt.Fprintf(w, "%s\n", name)
	}
}

func openAccessLog() {
	if *accessLog == "-" {
		accessLogWriter.Swap(os.Stdout)
		return
	}
	f, err := os.OpenFile(*accessLog, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		log.Print(err)
		return
	}
	if old := accessLogWriter.Swap(f); old != nil {
		if ic, ok := old.(io.Closer); ok {
			ic.Close()
		}
		log.Print("Reopen access log file")
	}
}

func main() {
	flag.Parse()

	log.Printf("Data directory: %s", *dataDir)
	var h http.Handler = &restfs{*dataDir}

	sort.Sort(sort.Reverse(byPriority(middlewares)))
	for _, m := range middlewares {
		h = m.wrap(h)
	}

	openAccessLog()
	h = webutil.Logger(h, accessLogWriter)
	sigm.Handle(syscall.SIGHUP, openAccessLog)

	g := newGC(*dataDir)
	g.Start()
	sigm.Handle(syscall.SIGUSR1, g.Start)
	if *gcInterval > 0 {
		log.Printf("GC runs every %s", *gcInterval)
		go func() {
			for range time.Tick(*gcInterval) {
				g.Start()
			}
		}()
	}

	log.Printf("Server started at %s", *listen)
	graceful.Run(*listen, *gracefulTimeout, webutil.Recoverer(h, os.Stderr))
}
<commit_msg>Add Etag support<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/tylerb/graceful"
	"github.com/yosisa/sigm"
	"github.com/yosisa/webutil"
)

var (
	dataDir         = flag.String("data-dir", "./data", "Data directory")
	listen          = flag.String("listen", ":8000", "Listen address")
	gracefulTimeout = flag.Duration("graceful-timeout", 10*time.Second, "Wait until force shutdown")
	gcInterval      = flag.Duration("gc-interval", time.Hour, "GC interval for cleaning deleted files")
	accessLog       = flag.String("access-log", "-", "Path to access log file")
)

var (
	accessLogWriter = new(webutil.ConsoleLogWriter)
	middlewares     []*middleware
)

const tombstone = ".restfs-deleted"

type middleware struct {
	priority int
	wrap     func(h http.Handler) http.Handler
}

type byPriority []*middleware

func (x byPriority) Len() int           { return len(x) }
func (x byPriority) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
func (x byPriority) Less(i, j int) bool { return x[i].priority < x[j].priority }

func registerMiddleware(priority int, wrap func(http.Handler) http.Handler) {
	middlewares = append(middlewares, &middleware{priority: priority, wrap: wrap})
}

type restfs struct {
	dir string
}

func (c *restfs) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fullpath := path.Join(c.dir, r.URL.Path)
	var (
		fi  os.FileInfo
		err error
	)
	switch r.Method {
	case "GET":
		s := stat(fullpath)
		if s == nil {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}
		if s.IsDir() {
			serveFileList(w, fullpath)
		} else {
			w.Header().Set("Etag", genEtag(s))
			http.ServeFile(w, r, fullpath)
		}
		return
	case "PUT":
		fi, err = os.Stat(fullpath)
		if err == nil && fi.IsDir() {
			http.Error(w, "Cannot overwrite directory", http.StatusBadRequest)
			return
		}
		if err == nil || os.IsNotExist(err) {
			err = c.saveFile(fullpath, r.Body)
			r.Body.Close()
		}
	case "DELETE":
		fi, err = os.Stat(fullpath)
		if err == nil {
			if fi.IsDir() {
				recursive, _ := strconv.ParseBool(r.URL.Query().Get("recursive"))
				if recursive {
					err = c.removeAll(fullpath)
				} else {
					http.Error(w, "Cannot remove directory; forgot recursive=true?", http.StatusBadRequest)
					return
				}
			} else {
				err = c.remove(fullpath)
			}
		} else if os.IsNotExist(err) {
			return
		}
	default:
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
}

func (c *restfs) saveFile(fullpath string, r io.Reader) error {
	dir, _ := path.Split(fullpath)
	if err := os.MkdirAll(dir, 0777); err != nil {
		return err
	}
	f, err := os.OpenFile(fullpath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666)
	if err != nil {
		return err
	}
	if _, err := io.Copy(f, r); err != nil {
		return err
	}
	return nil
}

func (c *restfs) remove(fullpath string) error {
	f, err := os.Create(fullpath + tombstone)
	if err == nil {
		f.Close()
	}
	return err
}

func (c *restfs) removeAll(fullpath string) error {
	return filepath.Walk(fullpath, func(name string, stat os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if stat.IsDir() || strings.HasSuffix(name, tombstone) {
			return nil
		}
		return c.remove(name)
	})
}

type gc struct {
	dir    string
	invoke chan struct{}
}

func newGC(dir string) *gc {
	g := &gc{
		dir:    dir,
		invoke: make(chan struct{}, 1),
	}
	go g.loop()
	return g
}

func (g *gc) loop() {
	remove := func(s string) error {
		log.Printf("Remove %s", s)
		if err := os.Remove(s); err != nil && !os.IsNotExist(err) {
			return err
		}
		return nil
	}
	for range g.invoke {
		log.Print("GC started")
		start := time.Now()
		err := filepath.Walk(g.dir, func(name string, stat os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if stat.IsDir() || !strings.HasSuffix(name, tombstone) {
				return nil
			}
			fname := name[:len(name)-len(tombstone)]
			fstat, err := os.Stat(fname)
			if err == nil {
				if !fstat.ModTime().After(stat.ModTime()) {
					if err = remove(fname); err != nil {
						return err
					}
				}
				return remove(name)
			} else if os.IsNotExist(err) {
				return remove(name)
			}
			return err
		})
		took := time.Since(start)
		if err == nil {
			log.Printf("GC has finished in %v", took)
		} else {
			log.Printf("GC has aborted in %v with error: %v", took, err)
		}
	}
}

func (g *gc) Start() {
	select {
	case g.invoke <- struct{}{}:
	default:
	}
}

func stat(fullpath string) os.FileInfo {
	astat, err := os.Stat(fullpath)
	if err != nil {
		return nil
	}
	if astat.IsDir() {
		return astat
	}

	bstat, err := os.Stat(fullpath + tombstone)
	if err != nil {
		if os.IsNotExist(err) {
			return astat
		}
		log.Print(err)
		return nil
	}
	if astat.ModTime().After(bstat.ModTime()) {
		return astat
	}
	return nil
}

func genEtag(s os.FileInfo) string {
	mtime := s.ModTime()
	t := mtime.Unix()*1000000 + mtime.UnixNano()/1000
	return fmt.Sprintf(`W/"%x-%x"`, s.Size(), t)
}

func serveFileList(w http.ResponseWriter, s string) {
	fis, err := ioutil.ReadDir(s)
	if err != nil {
		log.Print(err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	tombstones := make(map[string]os.FileInfo)
	for _, fi := range fis {
		name := fi.Name()
		if strings.HasSuffix(name, tombstone) {
			name = name[:len(name)-len(tombstone)]
			tombstones[name] = fi
		}
	}

	for _, fi := range fis {
		name := fi.Name()
		if strings.HasSuffix(name, tombstone) {
			continue
		}
		if fi.IsDir() {
			name += "/"
		} else if ts := tombstones[name]; ts != nil && !fi.ModTime().After(ts.ModTime()) {
			continue
		}
		fmt.Fprintf(w, "%s\n", name)
	}
}

func openAccessLog() {
	if *accessLog == "-" {
		accessLogWriter.Swap(os.Stdout)
		return
	}
	f, err := os.OpenFile(*accessLog, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		log.Print(err)
		return
	}
	if old := accessLogWriter.Swap(f); old != nil {
		if ic, ok := old.(io.Closer); ok {
			ic.Close()
		}
		log.Print("Reopen access log file")
	}
}

func main() {
	flag.Parse()

	log.Printf("Data directory: %s", *dataDir)
	var h http.Handler = &restfs{*dataDir}

	sort.Sort(sort.Reverse(byPriority(middlewares)))
	for _, m := range middlewares {
		h = m.wrap(h)
	}

	openAccessLog()
	h = webutil.Logger(h, accessLogWriter)
	sigm.Handle(syscall.SIGHUP, openAccessLog)

	g := newGC(*dataDir)
	g.Start()
	sigm.Handle(syscall.SIGUSR1, g.Start)
	if *gcInterval > 0 {
		log.Printf("GC runs every %s", *gcInterval)
		go func() {
			for range time.Tick(*gcInterval) {
				g.Start()
			}
		}()
	}

	log.Printf("Server started at %s", *listen)
	graceful.Run(*listen, *gracefulTimeout, webutil.Recoverer(h, os.Stderr))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

var shouldOverwriteTests = flag.Bool("destructive", false, "rewrite tests in-place")
var shouldCreateTestSuite = flag.Bool("create-suite", true, "creates a ginkgo suite file")

func main() {
	flag.Parse()

	if len(flag.Args()) != 1 {
		println(fmt.Sprintf("usage: %s /path/to/some/file_test.go --destructive=(true|false)", os.Args[0]))
		println("\n--destructive indicates that you want to update your tests in-place, and can possibly lead to data loss if your tests are not committed to version control (or otherwise backed up)")
		os.Exit(1)
	}

	testFiles, err := findTestsForPackage(flag.Args()[0])
	if err != nil {
		fmt.Printf("unexpected error reading package: '%s'\n%s\n", flag.Args()[0], err.Error())
		os.Exit(1)
	}

	for _, filename := range testFiles {
		err := findTestsInFile(filename)

		if err != nil {
			panic(err.Error())
		}
	}

	pkg, err := build.Default.Import(flag.Args()[0], ".", build.ImportMode(0))
	if err != nil {
		panic(err)
	}

	if *shouldCreateTestSuite {
		addGinkgoSuiteFile(pkg.Dir)
	}
}

func addGinkgoSuiteFile(pathToPackage string) {
	originalDir, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	err = os.Chdir(pathToPackage)
	if err != nil {
		panic(err)
	}

	// FIXME : don't do this if we already have a test suite file
	cmd := exec.Command("ginkgo", "bootstrap")
	err = cmd.Run()

	if err != nil {
		panic(err.Error())
	}

	err = os.Chdir(originalDir)
	if err != nil {
		panic(err)
	}
}

func findTestsForPackage(packageName string) (tests []string, err error) {
	pkg, err := build.Default.Import(packageName, ".", build.ImportMode(0))
	if err != nil {
		return
	}

	for _, file := range append(pkg.TestGoFiles, pkg.XTestGoFiles...) {
		tests = append(tests, filepath.Join(pkg.Dir, file))
	}

	dirFiles, err := ioutil.ReadDir(pkg.Dir)
	if err != nil {
		return
	}

	for _, file := range dirFiles {
		if !file.IsDir() {
			continue
		}

		moreTests, err := findTestsForPackage(filepath.Join(pkg.ImportPath, file.Name()))
		tests = append(tests, moreTests...)

		if err != nil {
			return tests, err
		}
	}

	return
}

func findTestsInFile(pathToFile string) (err error) {
	fileSet := token.NewFileSet()
	rootNode, err := parser.ParseFile(fileSet, pathToFile, nil, 0)
	if err != nil {
		return
	}

	addGinkgoImports(rootNode)
	removeTestingImport(rootNode)

	testsToRewrite := findTestFuncs(rootNode)
	topLevelInitFunc := createInitBlock()

	describeBlock := createDescribeBlock()
	topLevelInitFunc.Body.List = append(topLevelInitFunc.Body.List, describeBlock)

	for _, testFunc := range testsToRewrite {
		rewriteTestInGinkgo(testFunc, rootNode, describeBlock)
	}

	rootNode.Decls = append(rootNode.Decls, topLevelInitFunc)
	rewriteOtherFuncsToUseMrT(rootNode.Decls)

	var buffer bytes.Buffer
	if err = format.Node(&buffer, fileSet, rootNode); err != nil {
		println(err.Error())
		return
	}

	var fileToWrite string
	if *shouldOverwriteTests {
		fileToWrite = pathToFile
	} else {
		fileToWrite = strings.Replace(pathToFile, "_test.go", "_ginkgo_test.go", 1)
	}

	// TODO: write this out with the same file permissions
	ioutil.WriteFile(fileToWrite, buffer.Bytes(), 0666)
	return
}

func createInitBlock() *ast.FuncDecl {
	blockStatement := &ast.BlockStmt{List: []ast.Stmt{}}
	fieldList := &ast.FieldList{}
	funcType := &ast.FuncType{Params: fieldList}
	ident := &ast.Ident{Name: "init"}

	return &ast.FuncDecl{Name: ident, Type: funcType, Body: blockStatement}
}

func createDescribeBlock() *ast.ExprStmt {
	blockStatement := &ast.BlockStmt{List: []ast.Stmt{}}

	fieldList := &ast.FieldList{}
	funcType := &ast.FuncType{Params: fieldList}
	funcLit := &ast.FuncLit{Type: funcType, Body: blockStatement}
	basicLit := &ast.BasicLit{Kind: 9, Value: "\"Testing with ginkgo\""}
	describeIdent := &ast.Ident{Name: "Describe"}
	callExpr := &ast.CallExpr{Fun: describeIdent, Args: []ast.Expr{basicLit, funcLit}}

	return &ast.ExprStmt{X: callExpr}
}

func findTestFuncs(rootNode *ast.File) (testsToRewrite []*ast.FuncDecl) {
	testNameRegexp := regexp.MustCompile("^Test[A-Z].+")

	ast.Inspect(rootNode, func(node ast.Node) bool {
		if node == nil {
			return false
		}

		switch node := node.(type) {
		case *ast.FuncDecl:
			matches := testNameRegexp.MatchString(node.Name.Name)

			if matches && receivesTestingT(node) {
				testsToRewrite = append(testsToRewrite, node)
			}
		}

		return true
	})

	return
}

// name is on field.Name.Name
// type is on field.Type.X.X.Name + "." + field.Type.X.Sel.Name
func receivesTestingT(node *ast.FuncDecl) bool {
	if len(node.Type.Params.List) != 1 {
		return false
	}

	// (node.Type.Params.List[0].Names[0].Name == "t") => the name of the testingT arg
	base, ok := node.Type.Params.List[0].Type.(*ast.StarExpr)
	if !ok {
		return false
	}

	intermediate := base.X.(*ast.SelectorExpr)
	isTestingPackage := intermediate.X.(*ast.Ident).Name == "testing"
	isTestingT := intermediate.Sel.Name == "T"

	return isTestingPackage && isTestingT
}

func namedTestingTArg(node *ast.FuncDecl) string {
	return node.Type.Params.List[0].Names[0].Name // *exhale*
}

func replaceTestingTsInsideFuncParams(selectorExpr *ast.SelectorExpr, testingT string) {
	funcTargetIdent, ok := selectorExpr.X.(*ast.Ident)
	if !ok {
		return
	}

	// fix t.Fail() or any other *testing.T method calls
	// by replacing with T().Fail()
	if funcTargetIdent.Name == testingT {
		selectorExpr.X = &ast.CallExpr{
  		Lparen: funcTargetIdent.NamePos + 1,
			Rparen: funcTargetIdent.NamePos + 2,
			Fun: &ast.Ident{Name: "T"},
		}
		return
	}
}

func replaceTestingTsWithMrT(statementsBlock *ast.BlockStmt, testingT string) {
	ast.Inspect(statementsBlock, func(node ast.Node) bool {
		if node == nil {
			return false
		}

		callExpr, ok := node.(*ast.CallExpr)
		if !ok {
			return true
		}

		funCall, ok := callExpr.Fun.(*ast.SelectorExpr)
		if ok {
			replaceTestingTsInsideFuncParams(funCall, testingT)
			return true
		}

		for index, arg := range callExpr.Args {
			identArg, ok := arg.(*ast.Ident)
			if !ok {
				return true
			}

			if identArg.Name == testingT {
				callExpr.Args[index] = &ast.CallExpr{
					Args: []ast.Expr{},
					Fun:  &ast.Ident{Name: "T", Obj: nil},
				}
			}
		}

		return true
	})
}

func rewriteTestInGinkgo(testFunc *ast.FuncDecl, rootNode *ast.File, describe *ast.ExprStmt) {
	var funcIndex int = -1
	for index, child := range rootNode.Decls {
		if child == testFunc {
			funcIndex = index
			break
		}
	}

	if funcIndex < 0 {
		println("Assert Error: Error finding index for test node %s\n", testFunc.Name.Name)
		os.Exit(1)
	}

	// create a new node
	blockStatement := &ast.BlockStmt{List: testFunc.Body.List}
	fieldList := &ast.FieldList{}
	funcType := &ast.FuncType{Params: fieldList}
	funcLit := &ast.FuncLit{Type: funcType, Body: blockStatement}
	basicLit := &ast.BasicLit{Kind: 9, Value: fmt.Sprintf("\"%s\"", testFunc.Name.Name)}
	itBlockIdent := &ast.Ident{Name: "It"}
	callExpr := &ast.CallExpr{Fun: itBlockIdent, Args: []ast.Expr{basicLit, funcLit}}
	expressionStatement := &ast.ExprStmt{X: callExpr}

	var block *ast.BlockStmt = blockStatementFromDescribe(describe)
	block.List = append(block.List, expressionStatement)
	replaceTestingTsWithMrT(block, namedTestingTArg(testFunc))

	// remove the old test func from the root node
	rootNode.Decls = append(rootNode.Decls[:funcIndex], rootNode.Decls[funcIndex+1:]...)

	return
}

func blockStatementFromDescribe(desc *ast.ExprStmt) *ast.BlockStmt {
	var funcLit *ast.FuncLit

	for _, node := range desc.X.(*ast.CallExpr).Args {
		switch node := node.(type) {
		case *ast.FuncLit:
			funcLit = node
			break
		}
	}

	return funcLit.Body
}

func rewriteOtherFuncsToUseMrT(declarations []ast.Decl) {
	for _, decl := range declarations {
		decl, ok := decl.(*ast.FuncDecl)
		if !ok {
			continue
		}

		for _, param := range decl.Type.Params.List {
			starExpr, ok := param.Type.(*ast.StarExpr)
			if !ok {
				continue
			}

			selectorExpr, ok := starExpr.X.(*ast.SelectorExpr)
			if !ok {
				continue
			}

			xIdent, ok := selectorExpr.X.(*ast.Ident)
			if !ok || xIdent.Name != "testing" {
				continue
			}

			if selectorExpr.Sel.Name != "T" {
				continue
			}

			param.Type = &ast.Ident{Name: "TestingT"}
		}
	}
}
<commit_msg>R.E.S.P.E.C.T. file permissions<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

var shouldOverwriteTests = flag.Bool("destructive", false, "rewrite tests in-place")
var shouldCreateTestSuite = flag.Bool("create-suite", true, "creates a ginkgo suite file")

func main() {
	flag.Parse()

	if len(flag.Args()) != 1 {
		println(fmt.Sprintf("usage: %s /path/to/some/file_test.go --destructive=(true|false)", os.Args[0]))
		println("\n--destructive indicates that you want to update your tests in-place, and can possibly lead to data loss if your tests are not committed to version control (or otherwise backed up)")
		os.Exit(1)
	}

	testFiles, err := findTestsForPackage(flag.Args()[0])
	if err != nil {
		fmt.Printf("unexpected error reading package: '%s'\n%s\n", flag.Args()[0], err.Error())
		os.Exit(1)
	}

	for _, filename := range testFiles {
		err := findTestsInFile(filename)

		if err != nil {
			panic(err.Error())
		}
	}

	pkg, err := build.Default.Import(flag.Args()[0], ".", build.ImportMode(0))
	if err != nil {
		panic(err)
	}

	if *shouldCreateTestSuite {
		addGinkgoSuiteFile(pkg.Dir)
	}
}

func addGinkgoSuiteFile(pathToPackage string) {
	originalDir, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	err = os.Chdir(pathToPackage)
	if err != nil {
		panic(err)
	}

	// FIXME : don't do this if we already have a test suite file
	cmd := exec.Command("ginkgo", "bootstrap")
	err = cmd.Run()

	if err != nil {
		panic(err.Error())
	}

	err = os.Chdir(originalDir)
	if err != nil {
		panic(err)
	}
}

func findTestsForPackage(packageName string) (tests []string, err error) {
	pkg, err := build.Default.Import(packageName, ".", build.ImportMode(0))
	if err != nil {
		return
	}

	for _, file := range append(pkg.TestGoFiles, pkg.XTestGoFiles...) {
		tests = append(tests, filepath.Join(pkg.Dir, file))
	}

	dirFiles, err := ioutil.ReadDir(pkg.Dir)
	if err != nil {
		return
	}

	for _, file := range dirFiles {
		if !file.IsDir() {
			continue
		}

		moreTests, err := findTestsForPackage(filepath.Join(pkg.ImportPath, file.Name()))
		tests = append(tests, moreTests...)

		if err != nil {
			return tests, err
		}
	}

	return
}

func findTestsInFile(pathToFile string) (err error) {
	fileSet := token.NewFileSet()
	rootNode, err := parser.ParseFile(fileSet, pathToFile, nil, 0)
	if err != nil {
		return
	}

	addGinkgoImports(rootNode)
	removeTestingImport(rootNode)

	testsToRewrite := findTestFuncs(rootNode)
	topLevelInitFunc := createInitBlock()

	describeBlock := createDescribeBlock()
	topLevelInitFunc.Body.List = append(topLevelInitFunc.Body.List, describeBlock)

	for _, testFunc := range testsToRewrite {
		rewriteTestInGinkgo(testFunc, rootNode, describeBlock)
	}

	rootNode.Decls = append(rootNode.Decls, topLevelInitFunc)
	rewriteOtherFuncsToUseMrT(rootNode.Decls)

	var buffer bytes.Buffer
	if err = format.Node(&buffer, fileSet, rootNode); err != nil {
		println(err.Error())
		return
	}

	var fileToWrite string
	var mode os.FileMode
	if *shouldOverwriteTests {
		fileToWrite = pathToFile

		var fileInfo os.FileInfo
		fileInfo, err = os.Stat(pathToFile)
		if err != nil {
			return
		}

		mode = fileInfo.Mode()
	} else {
		fileToWrite = strings.Replace(pathToFile, "_test.go", "_ginkgo_test.go", 1)
		mode = 0644
	}

	ioutil.WriteFile(fileToWrite, buffer.Bytes(), mode)
	return
}

func createInitBlock() *ast.FuncDecl {
	blockStatement := &ast.BlockStmt{List: []ast.Stmt{}}
	fieldList := &ast.FieldList{}
	funcType := &ast.FuncType{Params: fieldList}
	ident := &ast.Ident{Name: "init"}

	return &ast.FuncDecl{Name: ident, Type: funcType, Body: blockStatement}
}

func createDescribeBlock() *ast.ExprStmt {
	blockStatement := &ast.BlockStmt{List: []ast.Stmt{}}

	fieldList := &ast.FieldList{}
	funcType := &ast.FuncType{Params: fieldList}
	funcLit := &ast.FuncLit{Type: funcType, Body: blockStatement}
	basicLit := &ast.BasicLit{Kind: 9, Value: "\"Testing with ginkgo\""}
	describeIdent := &ast.Ident{Name: "Describe"}
	callExpr := &ast.CallExpr{Fun: describeIdent, Args: []ast.Expr{basicLit, funcLit}}

	return &ast.ExprStmt{X: callExpr}
}

func findTestFuncs(rootNode *ast.File) (testsToRewrite []*ast.FuncDecl) {
	testNameRegexp := regexp.MustCompile("^Test[A-Z].+")

	ast.Inspect(rootNode, func(node ast.Node) bool {
		if node == nil {
			return false
		}

		switch node := node.(type) {
		case *ast.FuncDecl:
			matches := testNameRegexp.MatchString(node.Name.Name)

			if matches && receivesTestingT(node) {
				testsToRewrite = append(testsToRewrite, node)
			}
		}

		return true
	})

	return
}

// name is on field.Name.Name
// type is on field.Type.X.X.Name + "." + field.Type.X.Sel.Name
func receivesTestingT(node *ast.FuncDecl) bool {
	if len(node.Type.Params.List) != 1 {
		return false
	}

	// (node.Type.Params.List[0].Names[0].Name == "t") => the name of the testingT arg
	base, ok := node.Type.Params.List[0].Type.(*ast.StarExpr)
	if !ok {
		return false
	}

	intermediate := base.X.(*ast.SelectorExpr)
	isTestingPackage := intermediate.X.(*ast.Ident).Name == "testing"
	isTestingT := intermediate.Sel.Name == "T"

	return isTestingPackage && isTestingT
}

func namedTestingTArg(node *ast.FuncDecl) string {
	return node.Type.Params.List[0].Names[0].Name // *exhale*
}

func replaceTestingTsInsideFuncParams(selectorExpr *ast.SelectorExpr, testingT string) {
	funcTargetIdent, ok := selectorExpr.X.(*ast.Ident)
	if !ok {
		return
	}

	// fix t.Fail() or any other *testing.T method calls
	// by replacing with T().Fail()
	if funcTargetIdent.Name == testingT {
		selectorExpr.X = &ast.CallExpr{
  		Lparen: funcTargetIdent.NamePos + 1,
			Rparen: funcTargetIdent.NamePos + 2,
			Fun: &ast.Ident{Name: "T"},
		}
		return
	}
}

func replaceTestingTsWithMrT(statementsBlock *ast.BlockStmt, testingT string) {
	ast.Inspect(statementsBlock, func(node ast.Node) bool {
		if node == nil {
			return false
		}

		callExpr, ok := node.(*ast.CallExpr)
		if !ok {
			return true
		}

		funCall, ok := callExpr.Fun.(*ast.SelectorExpr)
		if ok {
			replaceTestingTsInsideFuncParams(funCall, testingT)
			return true
		}

		for index, arg := range callExpr.Args {
			identArg, ok := arg.(*ast.Ident)
			if !ok {
				return true
			}

			if identArg.Name == testingT {
				callExpr.Args[index] = &ast.CallExpr{
					Args: []ast.Expr{},
					Fun:  &ast.Ident{Name: "T", Obj: nil},
				}
			}
		}

		return true
	})
}

func rewriteTestInGinkgo(testFunc *ast.FuncDecl, rootNode *ast.File, describe *ast.ExprStmt) {
	var funcIndex int = -1
	for index, child := range rootNode.Decls {
		if child == testFunc {
			funcIndex = index
			break
		}
	}

	if funcIndex < 0 {
		println("Assert Error: Error finding index for test node %s\n", testFunc.Name.Name)
		os.Exit(1)
	}

	// create a new node
	blockStatement := &ast.BlockStmt{List: testFunc.Body.List}
	fieldList := &ast.FieldList{}
	funcType := &ast.FuncType{Params: fieldList}
	funcLit := &ast.FuncLit{Type: funcType, Body: blockStatement}
	basicLit := &ast.BasicLit{Kind: 9, Value: fmt.Sprintf("\"%s\"", testFunc.Name.Name)}
	itBlockIdent := &ast.Ident{Name: "It"}
	callExpr := &ast.CallExpr{Fun: itBlockIdent, Args: []ast.Expr{basicLit, funcLit}}
	expressionStatement := &ast.ExprStmt{X: callExpr}

	var block *ast.BlockStmt = blockStatementFromDescribe(describe)
	block.List = append(block.List, expressionStatement)
	replaceTestingTsWithMrT(block, namedTestingTArg(testFunc))

	// remove the old test func from the root node
	rootNode.Decls = append(rootNode.Decls[:funcIndex], rootNode.Decls[funcIndex+1:]...)

	return
}

func blockStatementFromDescribe(desc *ast.ExprStmt) *ast.BlockStmt {
	var funcLit *ast.FuncLit

	for _, node := range desc.X.(*ast.CallExpr).Args {
		switch node := node.(type) {
		case *ast.FuncLit:
			funcLit = node
			break
		}
	}

	return funcLit.Body
}

func rewriteOtherFuncsToUseMrT(declarations []ast.Decl) {
	for _, decl := range declarations {
		decl, ok := decl.(*ast.FuncDecl)
		if !ok {
			continue
		}

		for _, param := range decl.Type.Params.List {
			starExpr, ok := param.Type.(*ast.StarExpr)
			if !ok {
				continue
			}

			selectorExpr, ok := starExpr.X.(*ast.SelectorExpr)
			if !ok {
				continue
			}

			xIdent, ok := selectorExpr.X.(*ast.Ident)
			if !ok || xIdent.Name != "testing" {
				continue
			}

			if selectorExpr.Sel.Name != "T" {
				continue
			}

			param.Type = &ast.Ident{Name: "TestingT"}
		}
	}
}
<|endoftext|>


<commit_before>/*
 Copyright 2018 Padduck, LLC
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 	http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/pufferpanel/pufferpanel/database"
	"github.com/pufferpanel/pufferpanel/web"
)

const Hash = "none"
const Version = "2.0.0-DEV"

func main() {
	logging.Init()
	logging.SetLevel(logging.DEBUG)

	err := database.Load()

	if err != nil {
		logging.Error("Error connecting to database", err)
		return
	}

	defer database.Close()

	r := gin.Default()
	web.RegisterRoutes(r)

	err = r.Run() // listen and serve on 0.0.0.0:8080
	if err != nil {
		logging.Error("Error running web service", err)
	}
}<commit_msg>Add -r/--run to actually run service, and --addUser to handle proxy creation for now<commit_after>/*
 Copyright 2018 Padduck, LLC
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 	http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/pufferpanel/pufferpanel/database"
	"github.com/pufferpanel/pufferpanel/models"
	"github.com/pufferpanel/pufferpanel/services"
	"github.com/pufferpanel/pufferpanel/web"
	"os"
)

const Hash = "none"
const Version = "2.0.0-DEV"

func main() {
	logging.Init()
	logging.SetLevel(logging.DEBUG)

	err := database.Load()

	if err != nil {
		logging.Error("Error connecting to database", err)
		return
	}

	defer database.Close()

	args := os.Args[1:]

	run := false

	if len(args) > 0 {
		counter := 0
		for counter < len(args) {
			arg := args[counter]
			switch arg {
			case "--addUser":
				{
					if counter+3 >= len(args) {
						logging.Errorf("not enough arguments to create user")
						return
					}
					username := args[counter+1]
					email := args[counter+2]
					password := args[counter+3]
					counter += 3

					us, err := services.GetUserService()
					if err != nil {
						logging.Error("could not load user service", err)
						return
					}
					user := &models.User{Email: email, Username: username}
					err = user.SetPassword(password)

					if err != nil {
						logging.Error("could not create user", err)
						return
					}

					err = us.Create(user)
					if err != nil {
						logging.Error("could not create user", err)
						return
					}
				}
			case "--run", "-r":
				{
					run = true
				}
			default:
				{
					logging.Warnf("unknown argument: %s", arg)
				}
			}
			counter++
		}
	}

	if !run {
		return
	}

	r := gin.Default()
	web.RegisterRoutes(r)

	err = r.Run() // listen and serve on 0.0.0.0:8080
	if err != nil {
		logging.Error("Error running web service", err)
	}
}

func createUser(username, email, password string) {

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	cb "github.com/clearblade/Go-SDK"
	"github.com/clearblade/cblib"
	"strings"
)

func init() {
	flag.StringVar(&cblib.URL, "url", "", "Set the URL of the platform you want to use")
	flag.BoolVar(&cblib.ShouldImportCollectionRows, "importrows", false, "If supplied the import command will transfer collection rows from the old system to the new system")
	flag.IntVar(&cblib.ImportPageSize, "pagesize", 100, "If supplied the import command will migrate the specified number of rows per request")
}

func main() {
	flag.Parse()
	if cblib.URL != "" {
		cb.CB_ADDR = cblib.URL
	} else {
		cblib.URL = cb.CB_ADDR
	}
	cmd := strings.ToLower(flag.Arg(0))
	var err error
	var sysKey, dir string

	switch cmd {
	case "init":
		if err := cblib.Init_cmd(); err != nil {
			fmt.Printf("Error initializing: %v\n", err)
		}
	case "pull":
		if flag.NArg() != 2 {
			fmt.Printf("pull requires the systemKey as an argument\n")
		}
		if err := cblib.Pull_cmd(flag.Arg(1)); err != nil {
			fmt.Printf("Error pulling data: %v\n", err)
		}
	case "push":
		if flag.NArg() != 2 {
			sysKey, err = cblib.Sys_for_dir()
			if err != nil {
				fmt.Printf("%v\n", err)
				return
			}
			dir = "."
		} else {
			dir = ""
			sysKey = flag.Arg(1)
		}
		if err := cblib.Push_cmd(sysKey, dir); err != nil {
			fmt.Printf("Error pushing: %v\n", err)
		}
	case "export":
		if flag.NArg() != 2 {
			fmt.Printf("export requires the systemKey as an argument\n")
		}
		if err := cblib.Export_cmd(flag.Arg(1), flag.Arg(2)); err != nil {
			fmt.Printf("Error export data: %v\n", err)
		}
	case "import":
		var sysKey, dir string
		var err error
		sysKey, err = cblib.Sys_for_dir()
		if err != nil {
			fmt.Printf("%v\n", err)
			fmt.Printf("%v\n", sysKey)
			return
		}
		dir = "."

		if err := cblib.Import_cmd(dir, flag.Arg(2)); err != nil {
			fmt.Printf("Error import data: %v\n", err)
		}
	case "promote":
		fmt.Printf("promote command not yet implemented\n")
	case "diff":
		fmt.Printf("diff command not yet implemented\n")
	case "status":
		fmt.Printf("status command not yet implemented\n")
	case "test":
		fmt.Printf("test command not yet implemented\n")
	case "generate":
		fmt.Printf("generate command not yet implemented\n")
	default:
		fmt.Printf("Commands: 'init', 'pull', 'push', 'export', 'import', 'promote', 'diff', 'status', 'test', 'generate'\n")
	}
}
<commit_msg>Import/Export working<commit_after>package main

import (
	"flag"
	"fmt"
	cb "github.com/clearblade/Go-SDK"
	"github.com/clearblade/cblib"
	"strings"
)

func init() {
	flag.StringVar(&cblib.URL, "url", "", "Set the URL of the platform you want to use")
	flag.BoolVar(&cblib.ShouldImportCollectionRows, "importrows", false, "If supplied the import command will transfer collection rows from the old system to the new system")
	flag.IntVar(&cblib.ImportPageSize, "pagesize", 100, "If supplied the import command will migrate the specified number of rows per request")
}

func main() {
	flag.Parse()
	if cblib.URL != "" {
		cb.CB_ADDR = cblib.URL
	} else {
		cblib.URL = cb.CB_ADDR
	}
	cmd := strings.ToLower(flag.Arg(0))
	var err error
	var sysKey, dir string

	switch cmd {
	case "init":
		if err := cblib.Init_cmd(); err != nil {
			fmt.Printf("Error initializing: %v\n", err)
		}
	case "pull":
		if flag.NArg() != 2 {
			fmt.Printf("pull requires the systemKey as an argument\n")
		}
		if err := cblib.Pull_cmd(flag.Arg(1)); err != nil {
			fmt.Printf("Error pulling data: %v\n", err)
		}
	case "push":
		if flag.NArg() != 2 {
			sysKey, err = cblib.Sys_for_dir()
			if err != nil {
				fmt.Printf("%v\n", err)
				return
			}
			dir = "."
		} else {
			dir = ""
			sysKey = flag.Arg(1)
		}
		if err := cblib.Push_cmd(sysKey, dir); err != nil {
			fmt.Printf("Error pushing: %v\n", err)
		}
	case "export":
		if flag.NArg() != 2 {
			fmt.Printf("export requires the systemKey as an argument\n")
		}
		if err := cblib.Export_cmd(flag.Arg(1), flag.Arg(2)); err != nil {
			fmt.Printf("Error export data: %v\n", err)
		}
	case "import":
		/*
			var sysKey, dir string
			var err error
			sysKey, err = cblib.Sys_for_dir()
			if err != nil {
				fmt.Printf("%v\n", err)
				fmt.Printf("%v\n", sysKey)
				return
			}
			dir = "."

			if err := cblib.Import_cmd(dir, flag.Arg(2)); err != nil {
				fmt.Printf("Error import data: %v\n", err)
			}
		*/
		if err := cblib.Import_cmd(); err != nil {
			fmt.Printf("Error import data: %s\n", err.Error())
		}
	case "promote":
		fmt.Printf("promote command not yet implemented\n")
	case "diff":
		fmt.Printf("diff command not yet implemented\n")
	case "status":
		fmt.Printf("status command not yet implemented\n")
	case "test":
		fmt.Printf("test command not yet implemented\n")
	case "generate":
		fmt.Printf("generate command not yet implemented\n")
	default:
		fmt.Printf("Commands: 'init', 'pull', 'push', 'export', 'import', 'promote', 'diff', 'status', 'test', 'generate'\n")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"golang.org/x/net/context"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"

	skicka "github.com/pellaeon/skicka"
	"github.com/pellaeon/skicka/gdrive"
)

///////////////////////////////////////////////////////////////////////////
// Global Variables

var progName = filepath.Base(os.Args[0])

type debugging bool

var (
	gd *gdrive.GDrive

	// The key is only set if encryption is needed (i.e. if -encrypt is
	// provided for an upload, or if an encrypted file is encountered
	// during 'download' or 'cat').
	key []byte

	debug   debugging
	verbose debugging
	quiet   bool

	// Configuration read in from the skicka config file.
	config struct {
		Google struct {
			ClientId     string
			ClientSecret string
			// If set, is appended to all http requests via ?key=XXX.
			ApiKey string
		}
		Encryption struct {
			Salt             string
			Passphrase_hash  string
			Encrypted_key    string
			Encrypted_key_iv string
		}
		Upload struct {
			Ignored_Regexp         []string
			Bytes_per_second_limit int
		}
		Download struct {
			Bytes_per_second_limit int
		}
	}

	// Various statistics gathered along the way. These all should be
	// updated using atomic operations since we often have multiple threads
	// working concurrently for uploads and downloads.
	stats struct {
		DiskReadBytes     int64
		DiskWriteBytes    int64
		UploadBytes       int64
		DownloadBytes     int64
		LocalFilesUpdated int64
		DriveFilesUpdated int64
	}

	// Smaller files will be handled with multiple threads going at once;
	// doing so improves bandwidth utilization since round-trips to the
	// Drive APIs take a while.  (However, we don't want too have too many
	// workers; this would both lead to lots of 403 rate limit errors...)
	nWorkers int
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", progName)
	fmt.Fprintf(os.Stderr, "  %s MOUNTPOINT\n", progName)
	flag.PrintDefaults()
}

func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

func main() {
	log.SetFlags(0)
	log.SetPrefix(progName + ": ")

	// Initialize skicka
	home := userHomeDir()
	tokenCacheFilename := flag.String("tokencache",
		filepath.Join(home, ".skicka.tokencache.json"),
		"OAuth2 token cache file")
	configFilename := flag.String("config",
		filepath.Join(home, ".skicka.config"),
		"Configuration file")
	metadataCacheFilename := flag.String("metadata-cache-file",
		filepath.Join(home, "/.skicka.metadata.cache"),
		"Filename for local cache of Google Drive file metadata")
	nw := flag.Int("num-threads", 4, "Number of threads to use for uploads/downloads")
	vb := flag.Bool("verbose", false, "Enable verbose output")
	dbg := flag.Bool("debug", false, "Enable debugging output")
	qt := flag.Bool("quiet", false, "Suppress non-error messages")
	dumpHTTP := flag.Bool("dump-http", false, "Dump http traffic")
	flakyHTTP := flag.Bool("flaky-http", false, "Add flakiness to http traffic")
	noBrowserAuth := flag.Bool("no-browser-auth", false,
		"Don't try launching browser for authorization")
	flag.Usage = usage
	flag.Parse()

	var mountpoint string
	if flag.NArg() == 1 {
		mountpoint = flag.Arg(0)
	} else {
		mountpoint = *flag.String("mountpoint", "", "Mountpoint")
	}
	if mountpoint == "" {
		usage()
		os.Exit(2)
	}

	nWorkers = *nw

	debug = debugging(*dbg)
	verbose = debugging(*vb || bool(debug))
	quiet = *qt

	skicka.ReadConfigFile(*configFilename)

	// Set up the basic http.Transport.
	transport := http.DefaultTransport
	if tr, ok := transport.(*http.Transport); ok {
		// Increase the default number of open connections per destination host
		// to be enough for the number of goroutines we run concurrently for
		// uploads/downloads; this gives some benefit especially for uploading
		// small files.
		tr.MaxIdleConnsPerHost = 4
	} else {
		skicka.PrintErrorAndExit(fmt.Errorf("DefaultTransport not an *http.Transport?"))
	}
	if *flakyHTTP {
		transport = skicka.NewFlakyTransport(transport)
	}
	if *dumpHTTP {
		//transport = skicka.LoggingTransport{transport: transport}
		// TODO
	}

	// And now upgrade to the OAuth Transport *http.Client.
	client, err := skicka.GetOAuthClient(*tokenCacheFilename, !*noBrowserAuth,
		transport)
	if err != nil {
		skicka.PrintErrorAndExit(fmt.Errorf("error with OAuth2 Authorization: %v ", err))
	}

	// Choose the appropriate callback function for the GDrive object to
	// use for debugging output.
	var dpf func(s string, args ...interface{})
	if debug {
		dpf = skicka.DebugPrint
	} else {
		dpf = skicka.DebugNoPrint
	}

	gd, err := gdrive.New(config.Upload.Bytes_per_second_limit,
		config.Download.Bytes_per_second_limit, dpf, client,
		*metadataCacheFilename, quiet)
	if err != nil {
		log.Fatal(fmt.Errorf("error creating Google Drive "+
			"client: %v", err))
		os.Exit(3)
	}

	c, err := fuse.Mount(mountpoint)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	filesys := &FS{
		gd: gd,
	}
	if err := fs.Serve(c, filesys); err != nil {
		log.Fatal(err)
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		log.Fatal(err)
	}
}

type FS struct {
	gd *gdrive.GDrive
}

var _ fs.FS = (*FS)(nil)

func (f *FS) Root() (fs.Node, error) {
	gd_file, err := f.gd.GetFile("/")
	return Dir{
		sk_file: gd_file,
		fs:      f,
	}, err
}

var _ fs.Node = (*Dir)(nil)

type Dir struct {
	sk_file *gdrive.File
	fs      *FS
}

func (n Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
	var sum uint64
	for _, c := range n.sk_file.Id {
		sum += uint64(c)
	}
	attr.Inode = sum
	attr.Size = 4096 // XXX
	attr.Blocks = 0
	attr.Atime = n.sk_file.ModTime
	attr.Mtime = n.sk_file.ModTime
	attr.Ctime = n.sk_file.ModTime
	attr.Crtime = n.sk_file.ModTime
	attr.Mode = os.ModeDir | 0755
	attr.Nlink = 0

	return nil
}

var _ fs.Node = (*File)(nil)

type File struct {
	sk_file *gdrive.File
	fs      *FS
}

func (n File) Attr(ctx context.Context, attr *fuse.Attr) error {
	var sum uint64
	for _, c := range n.sk_file.Id {
		sum += uint64(c)
	}
	attr.Inode = sum
	attr.Size = uint64(n.sk_file.FileSize)
	attr.Blocks = uint64(n.sk_file.FileSize / 1024) // XXX: block size 1024 bytes
	attr.Atime = n.sk_file.ModTime
	attr.Mtime = n.sk_file.ModTime
	attr.Ctime = n.sk_file.ModTime
	attr.Crtime = n.sk_file.ModTime
	attr.Mode = os.ModeDir | 0755
	attr.Nlink = 0

	return nil
}

var _ fs.NodeRequestLookuper = (*Dir)(nil)

func (n Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) {
	path := req.Name
	gd_file, err := n.fs.gd.GetFile(path)
	if err != nil {
		log.Fatal(err)
		return nil, fuse.ENOENT
	}
	if gd_file.IsFolder() {
		return Dir{
			sk_file: gd_file,
			fs:      n.fs,
		}, nil
	} else {
		return File{
			sk_file: gd_file,
			fs:      n.fs,
		}, nil
	}
}

var _ fs.NodeOpener = (*File)(nil)

func (n File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
	resp.Flags |= fuse.OpenNonSeekable
	return &FileHandle{
		sk_file: n.sk_file,
	}, nil
}

var _ fs.Handle = (*FileHandle)(nil)

type FileHandle struct {
	sk_file *gdrive.File
}

var _ fs.HandleReleaser = (*FileHandle)(nil)

func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
	return nil
}

var _ fs.HandleReader = (*FileHandle)(nil)

func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
	buf := make([]byte, req.Size)
	reader, err := gd.GetFileContents(fh.sk_file)
	n, err := reader.Read(buf)
	resp.Data = buf[:n]
	return err
}
<commit_msg>Update# implement interface HandleReadDirAller<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"golang.org/x/net/context"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"

	skicka "github.com/pellaeon/skicka"
	"github.com/pellaeon/skicka/gdrive"
)

///////////////////////////////////////////////////////////////////////////
// Global Variables

var progName = filepath.Base(os.Args[0])

type debugging bool

var (
	gd *gdrive.GDrive

	// The key is only set if encryption is needed (i.e. if -encrypt is
	// provided for an upload, or if an encrypted file is encountered
	// during 'download' or 'cat').
	key []byte

	debug   debugging
	verbose debugging
	quiet   bool

	// Configuration read in from the skicka config file.
	config struct {
		Google struct {
			ClientId     string
			ClientSecret string
			// If set, is appended to all http requests via ?key=XXX.
			ApiKey string
		}
		Encryption struct {
			Salt             string
			Passphrase_hash  string
			Encrypted_key    string
			Encrypted_key_iv string
		}
		Upload struct {
			Ignored_Regexp         []string
			Bytes_per_second_limit int
		}
		Download struct {
			Bytes_per_second_limit int
		}
	}

	// Various statistics gathered along the way. These all should be
	// updated using atomic operations since we often have multiple threads
	// working concurrently for uploads and downloads.
	stats struct {
		DiskReadBytes     int64
		DiskWriteBytes    int64
		UploadBytes       int64
		DownloadBytes     int64
		LocalFilesUpdated int64
		DriveFilesUpdated int64
	}

	// Smaller files will be handled with multiple threads going at once;
	// doing so improves bandwidth utilization since round-trips to the
	// Drive APIs take a while.  (However, we don't want too have too many
	// workers; this would both lead to lots of 403 rate limit errors...)
	nWorkers int
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", progName)
	fmt.Fprintf(os.Stderr, "  %s MOUNTPOINT\n", progName)
	flag.PrintDefaults()
}

func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

func main() {
	log.SetFlags(0)
	log.SetPrefix(progName + ": ")

	// Initialize skicka
	home := userHomeDir()
	tokenCacheFilename := flag.String("tokencache",
		filepath.Join(home, ".skicka.tokencache.json"),
		"OAuth2 token cache file")
	configFilename := flag.String("config",
		filepath.Join(home, ".skicka.config"),
		"Configuration file")
	metadataCacheFilename := flag.String("metadata-cache-file",
		filepath.Join(home, "/.skicka.metadata.cache"),
		"Filename for local cache of Google Drive file metadata")
	nw := flag.Int("num-threads", 4, "Number of threads to use for uploads/downloads")
	vb := flag.Bool("verbose", false, "Enable verbose output")
	dbg := flag.Bool("debug", false, "Enable debugging output")
	qt := flag.Bool("quiet", false, "Suppress non-error messages")
	dumpHTTP := flag.Bool("dump-http", false, "Dump http traffic")
	flakyHTTP := flag.Bool("flaky-http", false, "Add flakiness to http traffic")
	noBrowserAuth := flag.Bool("no-browser-auth", false,
		"Don't try launching browser for authorization")
	flag.Usage = usage
	flag.Parse()

	var mountpoint string
	if flag.NArg() == 1 {
		mountpoint = flag.Arg(0)
	} else {
		mountpoint = *flag.String("mountpoint", "", "Mountpoint")
	}
	if mountpoint == "" {
		usage()
		os.Exit(2)
	}

	nWorkers = *nw

	debug = debugging(*dbg)
	verbose = debugging(*vb || bool(debug))
	quiet = *qt

	skicka.ReadConfigFile(*configFilename)

	// Set up the basic http.Transport.
	transport := http.DefaultTransport
	if tr, ok := transport.(*http.Transport); ok {
		// Increase the default number of open connections per destination host
		// to be enough for the number of goroutines we run concurrently for
		// uploads/downloads; this gives some benefit especially for uploading
		// small files.
		tr.MaxIdleConnsPerHost = 4
	} else {
		skicka.PrintErrorAndExit(fmt.Errorf("DefaultTransport not an *http.Transport?"))
	}
	if *flakyHTTP {
		transport = skicka.NewFlakyTransport(transport)
	}
	if *dumpHTTP {
		//transport = skicka.LoggingTransport{transport: transport}
		// TODO
	}

	// And now upgrade to the OAuth Transport *http.Client.
	client, err := skicka.GetOAuthClient(*tokenCacheFilename, !*noBrowserAuth,
		transport)
	if err != nil {
		skicka.PrintErrorAndExit(fmt.Errorf("error with OAuth2 Authorization: %v ", err))
	}

	// Choose the appropriate callback function for the GDrive object to
	// use for debugging output.
	var dpf func(s string, args ...interface{})
	if debug {
		dpf = skicka.DebugPrint
	} else {
		dpf = skicka.DebugNoPrint
	}

	gd, err := gdrive.New(config.Upload.Bytes_per_second_limit,
		config.Download.Bytes_per_second_limit, dpf, client,
		*metadataCacheFilename, quiet)
	if err != nil {
		log.Fatal(fmt.Errorf("error creating Google Drive "+
			"client: %v", err))
		os.Exit(3)
	}

	c, err := fuse.Mount(mountpoint)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	filesys := &FS{
		gd: gd,
	}
	if err := fs.Serve(c, filesys); err != nil {
		log.Fatal(err)
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		log.Fatal(err)
	}
}

type FS struct {
	gd *gdrive.GDrive
}

var _ fs.FS = (*FS)(nil)

func (f *FS) Root() (fs.Node, error) {
	gd_file, err := f.gd.GetFile("/")
	return Dir{
		sk_file: gd_file,
		fs:      f,
	}, err
}

var _ fs.Node = (*Dir)(nil)

type Dir struct {
	sk_file *gdrive.File
	fs      *FS
}

func (n Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
	var sum uint64
	for _, c := range n.sk_file.Id {
		sum += uint64(c)
	}
	attr.Inode = sum
	attr.Size = 4096 // XXX
	attr.Blocks = 0
	attr.Atime = n.sk_file.ModTime
	attr.Mtime = n.sk_file.ModTime
	attr.Ctime = n.sk_file.ModTime
	attr.Crtime = n.sk_file.ModTime
	attr.Mode = os.ModeDir | 0755
	attr.Nlink = 0

	return nil
}

var _ fs.HandleReadDirAller = (*Dir)(nil)

func (d *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	files_in_folder, err := d.fs.gd.GetFilesInFolder(d.sk_file.Path)
	var res []fuse.Dirent
	for _, file := range files_in_folder {
		var de fuse.Dirent
		de.Name = file.Path[strings.LastIndex(file.Path, "/")+1:]
		if file.IsFolder() {
			de.Type = fuse.DT_Dir
		} else {
			de.Type = fuse.DT_File
		}
		sum := uint64(0)
		for _, c := range file.Id {
			sum += uint64(c)
		}
		de.Inode = sum
		res = append(res, de)
	}
	return res, err
}

var _ fs.Node = (*File)(nil)

type File struct {
	sk_file *gdrive.File
	fs      *FS
}

func (n File) Attr(ctx context.Context, attr *fuse.Attr) error {
	var sum uint64
	for _, c := range n.sk_file.Id {
		sum += uint64(c)
	}
	attr.Inode = sum
	attr.Size = uint64(n.sk_file.FileSize)
	attr.Blocks = uint64(n.sk_file.FileSize / 1024) // XXX: block size 1024 bytes
	attr.Atime = n.sk_file.ModTime
	attr.Mtime = n.sk_file.ModTime
	attr.Ctime = n.sk_file.ModTime
	attr.Crtime = n.sk_file.ModTime
	attr.Mode = os.ModeDir | 0755
	attr.Nlink = 0

	return nil
}

var _ fs.NodeRequestLookuper = (*Dir)(nil)

func (n Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) {
	path := req.Name
	gd_file, err := n.fs.gd.GetFile(path)
	if err != nil {
		log.Fatal(err)
		return nil, fuse.ENOENT
	}
	if gd_file.IsFolder() {
		return Dir{
			sk_file: gd_file,
			fs:      n.fs,
		}, nil
	} else {
		return File{
			sk_file: gd_file,
			fs:      n.fs,
		}, nil
	}
}

var _ fs.NodeOpener = (*File)(nil)

func (n File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
	resp.Flags |= fuse.OpenNonSeekable
	return &FileHandle{
		sk_file: n.sk_file,
	}, nil
}

var _ fs.Handle = (*FileHandle)(nil)

type FileHandle struct {
	sk_file *gdrive.File
}

var _ fs.HandleReleaser = (*FileHandle)(nil)

func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
	return nil
}

var _ fs.HandleReader = (*FileHandle)(nil)

func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
	buf := make([]byte, req.Size)
	reader, err := gd.GetFileContents(fh.sk_file)
	n, err := reader.Read(buf)
	resp.Data = buf[:n]
	return err
}
<|endoftext|>