<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"sync/atomic"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/httpstream"
	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/proxy"
	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	genericfeatures "k8s.io/apiserver/pkg/features"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/transport"
	apiregistrationapi "k8s.io/kube-aggregator/pkg/apis/apiregistration"
)

// proxyHandler provides a http.Handler which will proxy traffic to locations
// specified by items implementing Redirector.
type proxyHandler struct {
	contextMapper genericapirequest.RequestContextMapper

	// localDelegate is used to satisfy local APIServices
	localDelegate http.Handler

	// proxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
	// this to confirm the proxy's identity
	proxyClientCert []byte
	proxyClientKey  []byte
	proxyTransport  *http.Transport

	// Endpoints based routing to map from cluster IP to routable IP
	serviceResolver ServiceResolver

	handlingInfo atomic.Value
}

type proxyHandlingInfo struct {
	// local indicates that this APIService is locally satisfied
	local bool

	// restConfig holds the information for building a roundtripper
	restConfig *restclient.Config
	// transportBuildingError is an error produced while building the transport.  If this
	// is non-nil, it will be reported to clients.
	transportBuildingError error
	// proxyRoundTripper is the re-useable portion of the transport.  It does not vary with any request.
	proxyRoundTripper http.RoundTripper
	// serviceName is the name of the service this handler proxies to
	serviceName string
	// namespace is the namespace the service lives in
	serviceNamespace string
}

func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	value := r.handlingInfo.Load()
	if value == nil {
		r.localDelegate.ServeHTTP(w, req)
		return
	}
	handlingInfo := value.(proxyHandlingInfo)
	if handlingInfo.local {
		if r.localDelegate == nil {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		r.localDelegate.ServeHTTP(w, req)
		return
	}

	if handlingInfo.transportBuildingError != nil {
		http.Error(w, handlingInfo.transportBuildingError.Error(), http.StatusInternalServerError)
		return
	}

	ctx, ok := r.contextMapper.Get(req)
	if !ok {
		http.Error(w, "missing context", http.StatusInternalServerError)
		return
	}
	user, ok := genericapirequest.UserFrom(ctx)
	if !ok {
		http.Error(w, "missing user", http.StatusInternalServerError)
		return
	}

	// write a new location based on the existing request pointed at the target service
	location := &url.URL{}
	location.Scheme = "https"
	rloc, err := r.serviceResolver.ResolveEndpoint(handlingInfo.serviceNamespace, handlingInfo.serviceName)
	if err != nil {
		http.Error(w, fmt.Sprintf("missing route (%s)", err.Error()), http.StatusInternalServerError)
		return
	}
	location.Host = rloc.Host
	location.Path = req.URL.Path
	location.RawQuery = req.URL.Query().Encode()

	// WithContext creates a shallow clone of the request with the new context.
	newReq := req.WithContext(context.Background())
	newReq.Header = utilnet.CloneHeader(req.Header)
	newReq.URL = location

	if handlingInfo.proxyRoundTripper == nil {
		http.Error(w, "", http.StatusNotFound)
		return
	}

	// we need to wrap the roundtripper in another roundtripper which will apply the front proxy headers
	proxyRoundTripper, upgrade, err := maybeWrapForConnectionUpgrades(handlingInfo.restConfig, handlingInfo.proxyRoundTripper, req)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)

	// if we are upgrading, then the upgrade path tries to use this request with the TLS config we provide, but it does
	// NOT use the roundtripper.  Its a direct call that bypasses the round tripper.  This means that we have to
	// attach the "correct" user headers to the request ahead of time.  After the initial upgrade, we'll be back
	// at the roundtripper flow, so we only have to muck with this request, but we do have to do it.
	if upgrade {
		transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetGroups(), user.GetExtra())
	}

	handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{w: w})
	handler.ServeHTTP(w, newReq)
}

// maybeWrapForConnectionUpgrades wraps the roundtripper for upgrades.  The bool indicates if it was wrapped
func maybeWrapForConnectionUpgrades(restConfig *restclient.Config, rt http.RoundTripper, req *http.Request) (http.RoundTripper, bool, error) {
	if !httpstream.IsUpgradeRequest(req) {
		return rt, false, nil
	}

	tlsConfig, err := restclient.TLSConfigFor(restConfig)
	if err != nil {
		return nil, true, err
	}
	followRedirects := utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StreamingProxyRedirects)
	upgradeRoundTripper := spdy.NewRoundTripper(tlsConfig, followRedirects)
	wrappedRT, err := restclient.HTTPWrappersForConfig(restConfig, upgradeRoundTripper)
	if err != nil {
		return nil, true, err
	}

	return wrappedRT, true, nil
}

// responder implements rest.Responder for assisting a connector in writing objects or errors.
type responder struct {
	w http.ResponseWriter
}

// TODO this should properly handle content type negotiation
// if the caller asked for protobuf and you write JSON bad things happen.
func (r *responder) Object(statusCode int, obj runtime.Object) {
	responsewriters.WriteRawJSON(statusCode, obj, r.w)
}

func (r *responder) Error(_ http.ResponseWriter, _ *http.Request, err error) {
	http.Error(r.w, err.Error(), http.StatusInternalServerError)
}

// these methods provide locked access to fields

func (r *proxyHandler) updateAPIService(apiService *apiregistrationapi.APIService) {
	if apiService.Spec.Service == nil {
		r.handlingInfo.Store(proxyHandlingInfo{local: true})
		return
	}

	newInfo := proxyHandlingInfo{
		restConfig: &restclient.Config{
			TLSClientConfig: restclient.TLSClientConfig{
				Insecure:   apiService.Spec.InsecureSkipTLSVerify,
				ServerName: apiService.Spec.Service.Name + "." + apiService.Spec.Service.Namespace + ".svc",
				CertData:   r.proxyClientCert,
				KeyData:    r.proxyClientKey,
				CAData:     apiService.Spec.CABundle,
			},
		},
		serviceName:      apiService.Spec.Service.Name,
		serviceNamespace: apiService.Spec.Service.Namespace,
	}
	newInfo.proxyRoundTripper, newInfo.transportBuildingError = restclient.TransportFor(newInfo.restConfig)
	if newInfo.transportBuildingError == nil && r.proxyTransport.Dial != nil {
		switch transport := newInfo.proxyRoundTripper.(type) {
		case *http.Transport:
			transport.Dial = r.proxyTransport.Dial
		default:
			newInfo.transportBuildingError = fmt.Errorf("unable to set dialer for %s/%s as rest transport is of type %T", apiService.Spec.Service.Namespace, apiService.Spec.Service.Name, newInfo.proxyRoundTripper)
			glog.Warning(newInfo.transportBuildingError.Error())
		}
	}
	r.handlingInfo.Store(newInfo)
}
<commit_msg>fix possible panic<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"sync/atomic"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/httpstream"
	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/proxy"
	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	genericfeatures "k8s.io/apiserver/pkg/features"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/transport"
	apiregistrationapi "k8s.io/kube-aggregator/pkg/apis/apiregistration"
)

// proxyHandler provides a http.Handler which will proxy traffic to locations
// specified by items implementing Redirector.
type proxyHandler struct {
	contextMapper genericapirequest.RequestContextMapper

	// localDelegate is used to satisfy local APIServices
	localDelegate http.Handler

	// proxyClientCert/Key are the client cert used to identify this proxy. Backing APIServices use
	// this to confirm the proxy's identity
	proxyClientCert []byte
	proxyClientKey  []byte
	proxyTransport  *http.Transport

	// Endpoints based routing to map from cluster IP to routable IP
	serviceResolver ServiceResolver

	handlingInfo atomic.Value
}

type proxyHandlingInfo struct {
	// local indicates that this APIService is locally satisfied
	local bool

	// restConfig holds the information for building a roundtripper
	restConfig *restclient.Config
	// transportBuildingError is an error produced while building the transport.  If this
	// is non-nil, it will be reported to clients.
	transportBuildingError error
	// proxyRoundTripper is the re-useable portion of the transport.  It does not vary with any request.
	proxyRoundTripper http.RoundTripper
	// serviceName is the name of the service this handler proxies to
	serviceName string
	// namespace is the namespace the service lives in
	serviceNamespace string
}

func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	value := r.handlingInfo.Load()
	if value == nil {
		r.localDelegate.ServeHTTP(w, req)
		return
	}
	handlingInfo := value.(proxyHandlingInfo)
	if handlingInfo.local {
		if r.localDelegate == nil {
			http.Error(w, "", http.StatusNotFound)
			return
		}
		r.localDelegate.ServeHTTP(w, req)
		return
	}

	if handlingInfo.transportBuildingError != nil {
		http.Error(w, handlingInfo.transportBuildingError.Error(), http.StatusInternalServerError)
		return
	}

	ctx, ok := r.contextMapper.Get(req)
	if !ok {
		http.Error(w, "missing context", http.StatusInternalServerError)
		return
	}
	user, ok := genericapirequest.UserFrom(ctx)
	if !ok {
		http.Error(w, "missing user", http.StatusInternalServerError)
		return
	}

	// write a new location based on the existing request pointed at the target service
	location := &url.URL{}
	location.Scheme = "https"
	rloc, err := r.serviceResolver.ResolveEndpoint(handlingInfo.serviceNamespace, handlingInfo.serviceName)
	if err != nil {
		http.Error(w, fmt.Sprintf("missing route (%s)", err.Error()), http.StatusInternalServerError)
		return
	}
	location.Host = rloc.Host
	location.Path = req.URL.Path
	location.RawQuery = req.URL.Query().Encode()

	// WithContext creates a shallow clone of the request with the new context.
	newReq := req.WithContext(context.Background())
	newReq.Header = utilnet.CloneHeader(req.Header)
	newReq.URL = location

	if handlingInfo.proxyRoundTripper == nil {
		http.Error(w, "", http.StatusNotFound)
		return
	}

	// we need to wrap the roundtripper in another roundtripper which will apply the front proxy headers
	proxyRoundTripper, upgrade, err := maybeWrapForConnectionUpgrades(handlingInfo.restConfig, handlingInfo.proxyRoundTripper, req)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)

	// if we are upgrading, then the upgrade path tries to use this request with the TLS config we provide, but it does
	// NOT use the roundtripper.  Its a direct call that bypasses the round tripper.  This means that we have to
	// attach the "correct" user headers to the request ahead of time.  After the initial upgrade, we'll be back
	// at the roundtripper flow, so we only have to muck with this request, but we do have to do it.
	if upgrade {
		transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetGroups(), user.GetExtra())
	}

	handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{w: w})
	handler.ServeHTTP(w, newReq)
}

// maybeWrapForConnectionUpgrades wraps the roundtripper for upgrades.  The bool indicates if it was wrapped
func maybeWrapForConnectionUpgrades(restConfig *restclient.Config, rt http.RoundTripper, req *http.Request) (http.RoundTripper, bool, error) {
	if !httpstream.IsUpgradeRequest(req) {
		return rt, false, nil
	}

	tlsConfig, err := restclient.TLSConfigFor(restConfig)
	if err != nil {
		return nil, true, err
	}
	followRedirects := utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StreamingProxyRedirects)
	upgradeRoundTripper := spdy.NewRoundTripper(tlsConfig, followRedirects)
	wrappedRT, err := restclient.HTTPWrappersForConfig(restConfig, upgradeRoundTripper)
	if err != nil {
		return nil, true, err
	}

	return wrappedRT, true, nil
}

// responder implements rest.Responder for assisting a connector in writing objects or errors.
type responder struct {
	w http.ResponseWriter
}

// TODO this should properly handle content type negotiation
// if the caller asked for protobuf and you write JSON bad things happen.
func (r *responder) Object(statusCode int, obj runtime.Object) {
	responsewriters.WriteRawJSON(statusCode, obj, r.w)
}

func (r *responder) Error(_ http.ResponseWriter, _ *http.Request, err error) {
	http.Error(r.w, err.Error(), http.StatusInternalServerError)
}

// these methods provide locked access to fields

func (r *proxyHandler) updateAPIService(apiService *apiregistrationapi.APIService) {
	if apiService.Spec.Service == nil {
		r.handlingInfo.Store(proxyHandlingInfo{local: true})
		return
	}

	newInfo := proxyHandlingInfo{
		restConfig: &restclient.Config{
			TLSClientConfig: restclient.TLSClientConfig{
				Insecure:   apiService.Spec.InsecureSkipTLSVerify,
				ServerName: apiService.Spec.Service.Name + "." + apiService.Spec.Service.Namespace + ".svc",
				CertData:   r.proxyClientCert,
				KeyData:    r.proxyClientKey,
				CAData:     apiService.Spec.CABundle,
			},
		},
		serviceName:      apiService.Spec.Service.Name,
		serviceNamespace: apiService.Spec.Service.Namespace,
	}
	newInfo.proxyRoundTripper, newInfo.transportBuildingError = restclient.TransportFor(newInfo.restConfig)
	if newInfo.transportBuildingError == nil && r.proxyTransport != nil && r.proxyTransport.Dial != nil {
		switch transport := newInfo.proxyRoundTripper.(type) {
		case *http.Transport:
			transport.Dial = r.proxyTransport.Dial
		default:
			newInfo.transportBuildingError = fmt.Errorf("unable to set dialer for %s/%s as rest transport is of type %T", apiService.Spec.Service.Namespace, apiService.Spec.Service.Name, newInfo.proxyRoundTripper)
			glog.Warning(newInfo.transportBuildingError.Error())
		}
	}
	r.handlingInfo.Store(newInfo)
}
<|endoftext|>


<commit_before>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 16777216
const maxMemory = 805306368
const httpPrefix = `http://`
const portSeparator = `:`
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const traefikHeatlhCheckLabel = `traefik.backend.healthcheck.path`
const traefikPortLabel = `traefik.port`
const linkSeparator = `:`
const waitTime = 10 * time.Second
const maxHealthCheckTry = 8

var httpClient = http.Client{Timeout: 5 * time.Second}

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices *map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (*deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Print(user.Username + ` starts pulling for ` + image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`Error while pulling image: %v`, err)
	}

	readBody(pull)
	log.Print(user.Username + ` ends pulling for ` + image)
	return nil
}

func healthCheckContainers(containers []*types.ContainerJSON) bool {
	healthCheckSuccess := make(map[string]bool)

	sleepDuration := waitTime

	for i := 0; i < maxHealthCheckTry; i++ {
		if i != 0 {
			time.Sleep(waitTime)
			sleepDuration = sleepDuration * 2
		}

		for _, container := range containers {
			if !healthCheckSuccess[container.ID] && healthCheckContainer(container) {
				healthCheckSuccess[container.ID] = true
			}
		}

		if len(healthCheckSuccess) == len(containers) {
			return true
		}
	}

	return false
}

func healthCheckContainer(container *types.ContainerJSON) bool {
	if container.Config.Labels[traefikHeatlhCheckLabel] != `` {
		log.Printf(`Checking health of container %s`, container.Name)

		request, err := http.NewRequest(`GET`, httpPrefix+container.NetworkSettings.Networks[networkMode].IPAddress+portSeparator+container.Config.Labels[traefikPortLabel]+container.Config.Labels[traefikHeatlhCheckLabel], nil)
		if err != nil {
			log.Printf(`Unable to prepare health check request for container %s : %v`, container.Name, err)
			return true
		}

		response, err := httpClient.Do(request)
		if err != nil {
			log.Printf(`Unable to health check for container %s : %v`, container.Name, err)
			return false
		}

		defer response.Body.Close()
		if response.StatusCode != http.StatusOK {
			log.Printf(`Health check failed for container %s : HTTP/%d`, container.Name, response.StatusCode)
			return false
		}
	}

	return true
}

func cleanContainers(containers *[]types.Container, user *auth.User) {
	for _, container := range *containers {
		log.Print(user.Username + ` stops ` + strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Print(user.Username + ` rm ` + strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]deployedService) error {
	for _, service := range *containers {
		if err := docker.ContainerRename(context.Background(), service.ID, getFinalName(service.Name)); err != nil {
			return fmt.Errorf(`Error while renaming container %s: %v`, service.Name, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]deployedService) {
	log.Printf(`Deleting container %s`, appName)
	for service, container := range services {
		if err := rmContainer(container.ID); err != nil {
			log.Printf(`Error while deleting container %s : %v`, service, err)
		}
	}
}

func startServices(appName []byte, services map[string]deployedService) {
	log.Printf(`Starting containers for app %s`, appName)
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			log.Printf(`Error while starting container %s : %v`, service, err)
		}
	}
}

func inspectServices(services map[string]deployedService) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`Error while inspecting container %s : %v`, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func createAppHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		http.Error(w, `An application name and a compose file are required`, http.StatusBadRequest)
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`Error while unmarshalling compose file: %v`, err))
		return
	}

	appNameStr := string(appName)
	log.Print(user.Username + ` deploys ` + appNameStr)

	ownerContainers, err := listContainers(user, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	deployedServices := make(map[string]deployedService)

	var creationError = false
	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, user); err != nil {
			errorHandler(w, err)
			return
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Printf(`%s starts %s`, user.Username, serviceFullName)

		createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(&service, user, appNameStr), getHostConfig(&service), getNetworkConfig(&service, &deployedServices), serviceFullName)
		if err != nil {
			errorHandler(w, fmt.Errorf(`Error while creating container: %v`, err))
			creationError = true
			break
		}

		deployedServices[serviceName] = deployedService{ID: createdContainer.ID, Name: serviceFullName}
	}

	if creationError {
		deleteServices(appName, deployedServices)
		return
	}

	startServices(appName, deployedServices)

	go func() {
		log.Printf(`Waiting for new containers to start...`)

		if healthCheckContainers(inspectServices(deployedServices)) {
			log.Printf(`Health check succeed for started app %s`, appName)
			cleanContainers(&ownerContainers, user)

			if err := renameDeployedContainers(&deployedServices); err != nil {
				log.Print(err)
			}
		} else {
			log.Printf(`Health check failed for started app %s`, appName)
			deleteServices(appName, deployedServices)
		}
	}()

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<commit_msg>Update compose.go<commit_after>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 16777216
const maxMemory = 805306368
const httpPrefix = `http://`
const portSeparator = `:`
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const traefikHeatlhCheckLabel = `traefik.backend.healthcheck.path`
const traefikPortLabel = `traefik.port`
const linkSeparator = `:`
const waitTime = 10 * time.Second
const maxHealthCheckTry = 8

var httpClient = http.Client{Timeout: 5 * time.Second}

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices *map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (*deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Print(user.Username + ` starts pulling for ` + image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`Error while pulling image: %v`, err)
	}

	readBody(pull)
	log.Print(user.Username + ` ends pulling for ` + image)
	return nil
}

func healthCheckContainers(containers []*types.ContainerJSON) bool {
	healthCheckSuccess := make(map[string]bool)

	sleepDuration := waitTime

	for i := 0; i < maxHealthCheckTry; i++ {
		if i != 0 {
			time.Sleep(sleepDuration)
			sleepDuration = sleepDuration * 2
		}

		for _, container := range containers {
			if !healthCheckSuccess[container.ID] && healthCheckContainer(container) {
				healthCheckSuccess[container.ID] = true
			}
		}

		if len(healthCheckSuccess) == len(containers) {
			return true
		}
	}

	return false
}

func healthCheckContainer(container *types.ContainerJSON) bool {
	if container.Config.Labels[traefikHeatlhCheckLabel] != `` {
		log.Printf(`Checking health of container %s`, container.Name)

		request, err := http.NewRequest(`GET`, httpPrefix+container.NetworkSettings.Networks[networkMode].IPAddress+portSeparator+container.Config.Labels[traefikPortLabel]+container.Config.Labels[traefikHeatlhCheckLabel], nil)
		if err != nil {
			log.Printf(`Unable to prepare health check request for container %s : %v`, container.Name, err)
			return true
		}

		response, err := httpClient.Do(request)
		if err != nil {
			log.Printf(`Unable to health check for container %s : %v`, container.Name, err)
			return false
		}

		defer response.Body.Close()
		if response.StatusCode != http.StatusOK {
			log.Printf(`Health check failed for container %s : HTTP/%d`, container.Name, response.StatusCode)
			return false
		}
	}

	return true
}

func cleanContainers(containers *[]types.Container, user *auth.User) {
	for _, container := range *containers {
		log.Print(user.Username + ` stops ` + strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Print(user.Username + ` rm ` + strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]deployedService) error {
	for _, service := range *containers {
		if err := docker.ContainerRename(context.Background(), service.ID, getFinalName(service.Name)); err != nil {
			return fmt.Errorf(`Error while renaming container %s: %v`, service.Name, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]deployedService) {
	log.Printf(`Deleting container %s`, appName)
	for service, container := range services {
		if err := rmContainer(container.ID); err != nil {
			log.Printf(`Error while deleting container %s : %v`, service, err)
		}
	}
}

func startServices(appName []byte, services map[string]deployedService) {
	log.Printf(`Starting containers for app %s`, appName)
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			log.Printf(`Error while starting container %s : %v`, service, err)
		}
	}
}

func inspectServices(services map[string]deployedService) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`Error while inspecting container %s : %v`, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func createAppHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		http.Error(w, `An application name and a compose file are required`, http.StatusBadRequest)
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`Error while unmarshalling compose file: %v`, err))
		return
	}

	appNameStr := string(appName)
	log.Print(user.Username + ` deploys ` + appNameStr)

	ownerContainers, err := listContainers(user, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	deployedServices := make(map[string]deployedService)

	var creationError = false
	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, user); err != nil {
			errorHandler(w, err)
			return
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Printf(`%s create %s container for app %s`, user.Username, serviceFullName, appName)

		createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(&service, user, appNameStr), getHostConfig(&service), getNetworkConfig(&service, &deployedServices), serviceFullName)
		if err != nil {
			errorHandler(w, fmt.Errorf(`Error while creating container: %v`, err))
			creationError = true
			break
		}

		deployedServices[serviceName] = deployedService{ID: createdContainer.ID, Name: serviceFullName}
	}

	if creationError {
		deleteServices(appName, deployedServices)
		return
	}

	startServices(appName, deployedServices)

	go func() {
		log.Printf(`Waiting for containers of app %s to start...`, appName)

		if healthCheckContainers(inspectServices(deployedServices)) {
			log.Printf(`Health check succeeded for app %s`, appName)
			cleanContainers(&ownerContainers, user)

			if err := renameDeployedContainers(&deployedServices); err != nil {
				log.Print(err)
			}
		} else {
			log.Printf(`Health check failed for app %s`, appName)
			deleteServices(appName, deployedServices)
		}
	}()

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package networker

import (
	"fmt"
	"io/ioutil"
	"net"
	"strings"

	"github.com/juju/loggo"
	"github.com/juju/utils/exec"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/network"
	apinetworker "github.com/juju/juju/state/api/networker"
	//"github.com/juju/juju/state/api/params"
	"github.com/juju/juju/state/api/watcher"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.networker")

// Patches for testing.
var (
	NetworkDir          = "/etc/network"
	InterfaceIsUp       = interfaceIsUp
	InterfaceHasAddress = interfaceHasAddress
	ExecuteCommands     = executeCommands
)

// Networker configures network interfaces on the machine, as needed.
type Networker struct {
	st  *apinetworker.State
	tag string
}

// Internal struct to store information required to setup networks.
type setupInfo struct {
	// The current contents of '/etc/network/interfaces' file.
	ifxData []byte

	// The result of MachineNetworkInfo API call.
	networks []network.Info

	// Generated list of commands to execute.
	commands []string
}

// NewNetworker returns a Worker that handles machine networking configuration.
func NewNetworker(st *apinetworker.State, agentConfig agent.Config) worker.Worker {
	nw := &Networker{
		st:  st,
		tag: agentConfig.Tag(),
	}
	return worker.NewNotifyWorker(nw)
}

func (nw *Networker) SetUp() (watcher.NotifyWatcher, error) {
	return nw.st.WatchInterfaces(nw.tag)
}

func (s *setupInfo) ensureVLANModule() {
	commands := []string{
		`dpkg-query -s vlan || apt-get --option Dpkg::Options::=--force-confold --assume-yes install vlan`,
		`lsmod | grep -q 8021q || modprobe 8021q`,
		`grep -q 8021q /etc/modules || echo 8021q >> /etc/modules`,
		`vconfig set_name_type DEV_PLUS_VID_NO_PAD`,
	}
	s.commands = append(s.commands, commands...)
}

func (s *setupInfo) bringUpInterfaces() {
	for _, network := range s.networks {
		name := network.ActualInterfaceName()
		if name != "eth0" && !InterfaceIsUp(name) {
			command := fmt.Sprintf(`ifup %s`, name)
			s.commands = append(s.commands, command)
		}
	}
}

func interfacePattern(interfaceName string) string {
	if interfaceName == "eth0" {
		return fmt.Sprintf("\nauto %s\nsource %s/%s.config\n", interfaceName, NetworkDir, interfaceName)
	}
	return fmt.Sprintf("\nauto %s\niface %s inet dhcp\n", interfaceName, interfaceName)
}

func (s *setupInfo) interfaceIsConfigured(interfaceName string) bool {
	substr := interfacePattern(interfaceName)
	return strings.Contains(string(s.ifxData), substr)
}

func (s *setupInfo) configureInterfaces() {
	for _, network := range s.networks {
		name := network.ActualInterfaceName()
		if name != "eth0" {
			s.configureInterface(name)
		}
	}
}

func (s *setupInfo) configureInterface(interfaceName string) {
	substr := interfacePattern(interfaceName)
	if !strings.Contains(string(s.ifxData), substr) {
		s.ifxData = append(s.ifxData, []byte(substr)...)
	}
}

// Execute slice of commands one by one.
func executeCommands(commands []string) error {
	for _, command := range commands {
		result, err := exec.RunCommands(exec.RunParams{
			Commands:   command,
			WorkingDir: "/",
		})
		if err != nil {
			err := fmt.Errorf("failed to execute %q: %v", command, err)
			logger.Errorf("%s", err.Error())
			return err
		}
		if result.Code != 0 {
			err := fmt.Errorf("command %q failed (code: %d, stdout: %s, stderr: %s)",
				command, result.Code, result.Stdout, result.Stderr)
			logger.Errorf("%s", err.Error())
			return err
		}
	}
	return nil
}

// Verify that the interface is up.
func interfaceIsUp(interfaceName string) bool {
	iface, err := net.InterfaceByName(interfaceName)
	if err != nil {
		logger.Errorf("cannot find network interface %q: %v", interfaceName, err)
		return false
	}
	return (iface.Flags & net.FlagUp) != 0
}

// Verify that the interface has assigned address.
func interfaceHasAddress(interfaceName string) bool {
	iface, err := net.InterfaceByName(interfaceName)
	if err != nil {
		logger.Errorf("cannot find network interface %q: %v", interfaceName, err)
		return false
	}
	addrs, err := iface.Addrs()
	if err != nil {
		logger.Errorf("cannot get addresses for network interface %q: %v", interfaceName, err)
		return false
	}
	return len(addrs) != 0
}

func (nw *Networker) Handle() error {
	s := &setupInfo{}
	var err error
	interfacesFile := NetworkDir + "/interfaces"
	s.ifxData, err = ioutil.ReadFile(interfacesFile)
	if err != nil {
		logger.Errorf("failed to read %s: %v", interfacesFile, err)
		return err
	}
	s.networks, err = nw.st.MachineNetworkInfo(nw.tag)
	logger.Infof("s.networks=%#v", s.networks)
	if err != nil {
		logger.Errorf("failed to process network info: %v", err)
		return err
	}

	// Verify that eth0 interfaces is properly configured and brought up.
	if !s.interfaceIsConfigured("eth0") || !InterfaceHasAddress("eth0") {
		if !s.interfaceIsConfigured("eth0") {
			logger.Errorf("interface eth0 has to be configured")
		}
		if !InterfaceHasAddress("eth0") {
			logger.Errorf("interface eth0 has to be bring up")
		}
		logger.Errorf("interface eth0 has to be configured and bring up")
		return fmt.Errorf("interface eth0 has to be configured and bring up")
	}

	// Update interface file
	s.configureInterfaces()
	err = ioutil.WriteFile(interfacesFile, s.ifxData, 0644)
	if err != nil {
		logger.Errorf("failed to write %s: %v", interfacesFile, err)
		return err
	}

	// Generate a list of commands to configure networks.
	s.ensureVLANModule()
	s.bringUpInterfaces()

	err = ExecuteCommands(s.commands)
	return err
}

func (nw *Networker) TearDown() error {
	// Nothing to do here.
	return nil
}
<commit_msg>read/write multiple interfaces files<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package networker

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"syscall"

	"github.com/juju/loggo"
	"github.com/juju/utils/exec"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/network"
	apinetworker "github.com/juju/juju/state/api/networker"
	//"github.com/juju/juju/state/api/params"
	"github.com/juju/juju/state/api/watcher"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.networker")

// Patches for testing.
var (
	NetworkDir          = "/etc/network"
	InterfaceIsUp       = interfaceIsUp
	InterfaceHasAddress = interfaceHasAddress
	ExecuteCommands     = executeCommands
)

// Networker configures network interfaces on the machine, as needed.
type Networker struct {
	st  *apinetworker.State
	tag string
}

// Internal struct to store information required to setup networks.
type setupInfo struct {
	// The name and contents of '/etc/network/interfaces' file.
	ifaceFileName string
	ifaceFileData []byte

	// The name and contents of files from '/etc/network/interfaces.d/' directory.
	ifaceDirName string
	ifaceDirData map[string][]byte

	// The result of MachineNetworkInfo API call.
	networks []network.Info

	// Generated list of commands to execute.
	commands []string
}

// NewNetworker returns a Worker that handles machine networking configuration.
func NewNetworker(st *apinetworker.State, agentConfig agent.Config) worker.Worker {
	nw := &Networker{
		st:  st,
		tag: agentConfig.Tag(),
	}
	return worker.NewNotifyWorker(nw)
}

func (nw *Networker) SetUp() (watcher.NotifyWatcher, error) {
	return nw.st.WatchInterfaces(nw.tag)
}

func (s *setupInfo) ensureVLANModule() {
	commands := []string{
		`dpkg-query -s vlan || apt-get --option Dpkg::Options::=--force-confold --assume-yes install vlan`,
		`lsmod | grep -q 8021q || modprobe 8021q`,
		`grep -q 8021q /etc/modules || echo 8021q >> /etc/modules`,
		`vconfig set_name_type DEV_PLUS_VID_NO_PAD`,
	}
	s.commands = append(s.commands, commands...)
}

func (s *setupInfo) bringUpInterfaces() {
	for _, network := range s.networks {
		name := network.ActualInterfaceName()
		if name != "eth0" && !InterfaceIsUp(name) {
			command := fmt.Sprintf(`ifup %s`, name)
			s.commands = append(s.commands, command)
		}
	}
}

func interfacePattern(interfaceName string) string {
	if interfaceName == "eth0" {
		return fmt.Sprintf("\nauto %s\nsource %s/%s.config\n", interfaceName, NetworkDir, interfaceName)
	}
	return fmt.Sprintf("\nauto %s\niface %s inet dhcp\n", interfaceName, interfaceName)
}

func (s *setupInfo) interfaceIsConfigured(interfaceName string) bool {
	substr := interfacePattern(interfaceName)
	return strings.Contains(string(s.ifaceFileData), substr)
}

func (s *setupInfo) configureInterfaces() {
	for _, network := range s.networks {
		name := network.ActualInterfaceName()
		if name != "eth0" {
			s.configureInterface(name)
		}
	}
}

func (s *setupInfo) configureInterface(interfaceName string) {
	substr := interfacePattern(interfaceName)
	if !strings.Contains(string(s.ifaceFileData), substr) {
		s.ifaceFileData = append(s.ifaceFileData, []byte(substr)...)
	}
}

// Execute slice of commands one by one.
func executeCommands(commands []string) error {
	for _, command := range commands {
		result, err := exec.RunCommands(exec.RunParams{
			Commands:   command,
			WorkingDir: "/",
		})
		if err != nil {
			err := fmt.Errorf("failed to execute %q: %v", command, err)
			logger.Errorf("%s", err.Error())
			return err
		}
		if result.Code != 0 {
			err := fmt.Errorf("command %q failed (code: %d, stdout: %s, stderr: %s)",
				command, result.Code, result.Stdout, result.Stderr)
			logger.Errorf("%s", err.Error())
			return err
		}
	}
	return nil
}

// Verify that the interface is up.
func interfaceIsUp(interfaceName string) bool {
	iface, err := net.InterfaceByName(interfaceName)
	if err != nil {
		logger.Errorf("cannot find network interface %q: %v", interfaceName, err)
		return false
	}
	return (iface.Flags & net.FlagUp) != 0
}

// Verify that the interface has assigned address.
func interfaceHasAddress(interfaceName string) bool {
	iface, err := net.InterfaceByName(interfaceName)
	if err != nil {
		logger.Errorf("cannot find network interface %q: %v", interfaceName, err)
		return false
	}
	addrs, err := iface.Addrs()
	if err != nil {
		logger.Errorf("cannot get addresses for network interface %q: %v", interfaceName, err)
		return false
	}
	return len(addrs) != 0
}

func (s *setupInfo) ReadInterfacesFiles() error {
	var err error
	s.ifaceFileData, err = ioutil.ReadFile(s.ifaceFileName)
	if err != nil {
		logger.Errorf("failed to read file %q: %v", s.ifaceFileName, err)
		return err
	}
	files, err := ioutil.ReadDir(s.ifaceDirName)
	if err != nil {
		if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOENT {
                        // ignore error when directory is absent
                } else {
                        logger.Errorf("failed to read directory %q: %#v\n", s.ifaceDirName, err)
                        return err
                }
	}
	for _, file := range files {
		if file.Mode().IsRegular() {
			fileName := s.ifaceDirName + "/" + file.Name()
			data, err := ioutil.ReadFile(fileName)
			if err != nil {
				// just report and ignore the error
				logger.Errorf("failed to read file %q: %v", fileName, err)
			}
			s.ifaceDirData[file.Name()] = data
		}
	}
	return nil
}

func (s *setupInfo) WriteInterfacesFiles() error {
	err := ioutil.WriteFile(s.ifaceFileName, s.ifaceFileData, 0644)
	if err != nil {
		logger.Errorf("failed to white file %q: %v", s.ifaceFileName, err)
		return err
	}
	err = os.Mkdir(s.ifaceDirName, 0755)
	if err != nil {
		if e, ok := err.(*os.PathError); ok && e.Err == syscall.EEXIST {
                        // ignore error when directory already exists
                } else {
                        logger.Errorf("failed to create directory %q: %#v\n", s.ifaceDirName, err)
                        return err
                }
	}
	for name, data := range s.ifaceDirData {
		fileName := s.ifaceDirName + "/" + name
		err = ioutil.WriteFile(fileName, data, 0644)
		if err != nil {
			logger.Errorf("failed to write file %q: %v", fileName, err)
			return err
		}
	}
	return nil
}

func (nw *Networker) Handle() error {
	s := &setupInfo{
		ifaceFileName: NetworkDir + "/interfaces",
		ifaceDirName: NetworkDir + "/interfaces.d",
		ifaceDirData: make(map[string][]byte),
	}
	err := s.ReadInterfacesFiles()
	if err != nil {
		return err
	}
	s.networks, err = nw.st.MachineNetworkInfo(nw.tag)
	logger.Infof("s.networks=%#v", s.networks)
	if err != nil {
		logger.Errorf("failed to process network info: %v", err)
		return err
	}

	// Verify that eth0 interfaces is properly configured and brought up.
	if !s.interfaceIsConfigured("eth0") || !InterfaceHasAddress("eth0") {
		if !s.interfaceIsConfigured("eth0") {
			logger.Errorf("interface eth0 has to be configured")
		}
		if !InterfaceHasAddress("eth0") {
			logger.Errorf("interface eth0 has to be bring up")
		}
		logger.Errorf("interface eth0 has to be configured and bring up")
		return fmt.Errorf("interface eth0 has to be configured and bring up")
	}

	// Update interface file
	s.configureInterfaces()
	err = s.WriteInterfacesFiles()
	if err != nil {
		return err
	}

	// Generate a list of commands to configure networks.
	s.ensureVLANModule()
	s.bringUpInterfaces()

	err = ExecuteCommands(s.commands)
	return err
}

func (nw *Networker) TearDown() error {
	// Nothing to do here.
	return nil
}
<|endoftext|>


<commit_before>package dataProcess

import (
	"../autils"
	"bytes"
	"database/sql"
	"github.com/gin-gonic/gin"
	"net/http"
	"strconv"
)

type tStruct struct {
	Name      string `json:"name"`
	Id        string `json:"id"`
	TextAlign string `json:"textAlign"`
}

type tRowsInfo struct {
	All             string `json:"all"`
	Core            string `json:"core"`
	Official        string `json:"official"`
	Plat            string `json:"plat"`
	Useless         string `json:"useless"`
	All_ishtml      bool   `json:"all_ishtml"`
	Core_ishtml     bool   `json:"core_ishtml"`
	Official_ishtml bool   `json:"official_ishtml"`
	Plat_ishtml     bool   `json:"plat_ishtml"`
	Useless_ishtml  bool   `json:"useless_ishtml"`
}

type tData struct {
	Columns []tStruct   `json:"columns"`
	Rows    []tRowsInfo `json:"rows"`
}

type tTypeStruct struct {
	Name    string `json:"name"`
	TagType int    `json:"tagType"`
}

// 组件概况数据处理
func TotalData(c *gin.Context, db *sql.DB) {

	types := []string{"core", "official", "plat", "useless", "all"}
	center := "center"

	td := tData{}

	tagCh := make(chan []int)
	useTagCh := make(chan []string)
	fullTagCh := make(chan []tTypeStruct)

	go getTagCount(db, tagCh)
	go getUseTag(db, useTagCh)
	go getFullTag(db, fullTagCh)

	counts := <-tagCh
	useTag := <-useTagCh
	fullTag := <-fullTagCh
	row := tRowsInfo{}

	row.Core = getHrefStr(c, types[0], counts[0])
	row.Official = getHrefStr(c, types[1], counts[1])
	row.Plat = getHrefStr(c, types[2], counts[2])
	row.All = getHrefStr(c, types[4], counts[0]+counts[1]+counts[2])
	row.Core_ishtml = true
	row.All_ishtml = true
	row.Plat_ishtml = true
	row.Official_ishtml = true
	row.Useless_ishtml = true

	var uselessTags []string
	for _, v := range fullTag {
		use := false
		for _, val := range useTag {
			if v.Name == val {
				use = true
			}
		}
		if !use {
			uselessTags = append(uselessTags, v.Name)
		}
	}
	row.Useless = getHrefStr(c, types[3], len(uselessTags))

	td.Rows = append(td.Rows, row)

	td.Columns = []tStruct{{
		"组件总量",
		types[4],
		center,
	}, {
		"核心组件数",
		types[0],
		center,
	}, {
		"扩展组件数",
		types[1],
		center,
	}, {
		"站长组件数",
		types[2],
		center,
	}, {
		"未使用组件数",
		types[3],
		center,
	}}

	c.JSON(http.StatusOK, gin.H{
		"status": 0,
		"msg":    "ok",
		"data":   td,
	})
}

func getUseTag(db *sql.DB, ch chan []string) {
	tagCtt := []string{}

	sqlStr := "select distinct tag_name from tags where date_sub(curdate(), INTERVAL ? DAY) <= date(`ana_date`)"

	rows, err := db.Query(sqlStr, 30)
	autils.ErrHadle(err)

	var name string
	for rows.Next() {
		err := rows.Scan(&name)
		autils.ErrHadle(err)
		tagCtt = append(tagCtt, name)
	}
	err = rows.Err()
	autils.ErrHadle(err)

	defer rows.Close()

	ch <- tagCtt
}

func getTagCount(db *sql.DB, ch chan []int) {
	counts := []int{}

	var buf bytes.Buffer
	for i := 1; i < 4; i++ {
		if i != 1 {
			buf.WriteString(" union all ")
		}
		buf.WriteString(" select count(*) from taglist where type =  " + strconv.Itoa(i))
	}
	rows, err := db.Query(buf.String())
	autils.ErrHadle(err)

	var count int
	for rows.Next() {
		err := rows.Scan(&count)
		autils.ErrHadle(err)
		counts = append(counts, count)
	}

	err = rows.Err()
	autils.ErrHadle(err)

	defer rows.Close()

	ch <- counts
}

func getFullTag(db *sql.DB, ch chan []tTypeStruct) {
	tags := []tTypeStruct{}

	rows, err := db.Query("select name, type from taglist")
	autils.ErrHadle(err)

	var name string
	var tagType int
	for rows.Next() {
		err := rows.Scan(&name, &tagType)
		autils.ErrHadle(err)
		t := tTypeStruct{}
		t.Name = name
		t.TagType = tagType
		tags = append(tags, t)
	}

	err = rows.Err()
	autils.ErrHadle(err)

	defer rows.Close()

	ch <- tags
}

func getHrefStr(c *gin.Context, t string, num int) string {

	return "" + strconv.Itoa(num) + ""
}
<commit_msg>update flow part db.<commit_after>package dataProcess

import (
	"../autils"
	"bytes"
	"database/sql"
	"github.com/gin-gonic/gin"
	"net/http"
	"strconv"
	"time"
)

type tStruct struct {
	Name      string `json:"name"`
	Id        string `json:"id"`
	TextAlign string `json:"textAlign"`
}

type tRowsInfo struct {
	All             string `json:"all"`
	Core            string `json:"core"`
	Official        string `json:"official"`
	Plat            string `json:"plat"`
	Useless         string `json:"useless"`
	All_ishtml      bool   `json:"all_ishtml"`
	Core_ishtml     bool   `json:"core_ishtml"`
	Official_ishtml bool   `json:"official_ishtml"`
	Plat_ishtml     bool   `json:"plat_ishtml"`
	Useless_ishtml  bool   `json:"useless_ishtml"`
}

type tData struct {
	Columns []tStruct   `json:"columns"`
	Rows    []tRowsInfo `json:"rows"`
}

type tTypeStruct struct {
	Name    string `json:"name"`
	TagType int    `json:"tagType"`
}

// 组件概况数据处理
func TotalData(c *gin.Context, db *sql.DB) {

	types := []string{"core", "official", "plat", "useless", "all"}
	center := "center"

	td := tData{}

	tagCh := make(chan []int)
	useTagCh := make(chan []string)
	fullTagCh := make(chan []tTypeStruct)

	go getTagCount(db, tagCh)
	go getUseTag(db, useTagCh)
	go getFullTag(db, fullTagCh)

	counts := <-tagCh
	useTag := <-useTagCh
	fullTag := <-fullTagCh
	row := tRowsInfo{}

	row.Core = getHrefStr(c, types[0], counts[0])
	row.Official = getHrefStr(c, types[1], counts[1])
	row.Plat = getHrefStr(c, types[2], counts[2])
	row.All = getHrefStr(c, types[4], counts[0]+counts[1]+counts[2])
	row.Core_ishtml = true
	row.All_ishtml = true
	row.Plat_ishtml = true
	row.Official_ishtml = true
	row.Useless_ishtml = true

	var uselessTags []string
	for _, v := range fullTag {
		use := false
		for _, val := range useTag {
			if v.Name == val {
				use = true
			}
		}
		if !use {
			uselessTags = append(uselessTags, v.Name)
		}
	}
	row.Useless = getHrefStr(c, types[3], len(uselessTags))

	td.Rows = append(td.Rows, row)

	td.Columns = []tStruct{{
		"组件总量",
		types[4],
		center,
	}, {
		"核心组件数",
		types[0],
		center,
	}, {
		"扩展组件数",
		types[1],
		center,
	}, {
		"站长组件数",
		types[2],
		center,
	}, {
		"未使用组件数",
		types[3],
		center,
	}}

	c.JSON(http.StatusOK, gin.H{
		"status": 0,
		"msg":    "ok",
		"data":   td,
	})
}

func getUseTag(db *sql.DB, ch chan []string) {
	tagCtt := []string{}
	now := time.Now()
	farAway := autils.GetCurrentData(now.AddDate(0, -1, 0))
	day := autils.GetCurrentData(now)

	rows, err := db.Query("select distinct tag_name from tags where ana_date >= '"+farAway+"' and ana_date < '"+day+"'")
	autils.ErrHadle(err)

	var name string
	for rows.Next() {
		err := rows.Scan(&name)
		autils.ErrHadle(err)
		tagCtt = append(tagCtt, name)
	}
	err = rows.Err()
	autils.ErrHadle(err)

	defer rows.Close()

	ch <- tagCtt
}

func getTagCount(db *sql.DB, ch chan []int) {
	counts := []int{}

	var buf bytes.Buffer
	for i := 1; i < 4; i++ {
		if i != 1 {
			buf.WriteString(" union all ")
		}
		buf.WriteString(" select count(*) from taglist where type =  " + strconv.Itoa(i))
	}
	rows, err := db.Query(buf.String())
	autils.ErrHadle(err)

	var count int
	for rows.Next() {
		err := rows.Scan(&count)
		autils.ErrHadle(err)
		counts = append(counts, count)
	}

	err = rows.Err()
	autils.ErrHadle(err)

	defer rows.Close()

	ch <- counts
}

func getFullTag(db *sql.DB, ch chan []tTypeStruct) {
	tags := []tTypeStruct{}

	rows, err := db.Query("select name, type from taglist")
	autils.ErrHadle(err)

	var name string
	var tagType int
	for rows.Next() {
		err := rows.Scan(&name, &tagType)
		autils.ErrHadle(err)
		t := tTypeStruct{}
		t.Name = name
		t.TagType = tagType
		tags = append(tags, t)
	}

	err = rows.Err()
	autils.ErrHadle(err)

	defer rows.Close()

	ch <- tags
}

func getHrefStr(c *gin.Context, t string, num int) string {

	return "" + strconv.Itoa(num) + ""
}
<|endoftext|>


<commit_before>/*
 * A Go package to copy a Dataset
 *
 * Copyright (C) 2017 Lawrence Woodman <lwoodman@vlifesystems.com>
 *
 * Licensed under an MIT licence.  Please see LICENCE.md for details.
 */

// Package dcopy copies a Dataset so that you can work consistently on
// the same Dataset.  This is important where a database is likely to be
// updated while you are working on it.  The copy of the database is stored
// in an sqlite3 database located in a temporary directory.
package dcopy

import (
	"encoding/csv"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/lawrencewoodman/ddataset"
	"github.com/lawrencewoodman/ddataset/dcsv"
)

// DCopy represents a copy of a Dataset
type DCopy struct {
	dataset    ddataset.Dataset
	tmpDir     string
	isReleased bool
}

// DCopyConn represents a connection to a DCopy Dataset
type DCopyConn struct {
	conn ddataset.Conn
	err  error
}

// New creates a new DCopy Dataset which will be a copy of the Dataset
// supplied at the time it is run. Please note that this creates a file
// on the disk containing a copy of the supplied Dataset.
func New(dataset ddataset.Dataset) (ddataset.Dataset, error) {
	tmpDir, err := ioutil.TempDir("", "dcopy")
	if err != nil {
		return nil, err
	}
	copyFilename := filepath.Join(tmpDir, "copy.csv")
	f, err := os.Create(copyFilename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	w := csv.NewWriter(f)

	conn, err := dataset.Open()
	if err != nil {
		os.RemoveAll(tmpDir)
		return nil, err
	}
	defer conn.Close()

	strRecord := make([]string, len(dataset.Fields()))
	for conn.Next() {
		record := conn.Read()
		for i, f := range dataset.Fields() {
			strRecord[i] = record[f].String()
		}
		if err := w.Write(strRecord); err != nil {
			os.RemoveAll(tmpDir)
			return nil, fmt.Errorf("error writing record to csv copy: %s", err)
		}
	}

	if err := conn.Err(); err != nil {
		os.RemoveAll(tmpDir)
		return nil, err
	}

	w.Flush()
	if err := w.Error(); err != nil {
		os.RemoveAll(tmpDir)
		return nil, err
	}

	return &DCopy{
		dataset:    dcsv.New(copyFilename, false, ',', dataset.Fields()),
		tmpDir:     tmpDir,
		isReleased: false,
	}, nil
}

// Open creates a connection to the Dataset
func (d *DCopy) Open() (ddataset.Conn, error) {
	if d.isReleased {
		return nil, ddataset.ErrReleased
	}
	conn, err := d.dataset.Open()
	if err != nil {
		return nil, err
	}
	return &DCopyConn{
		conn: conn,
		err:  nil,
	}, nil
}

// Fields returns the field names used by the Dataset
func (d *DCopy) Fields() []string {
	if d.isReleased {
		return []string{}
	}
	return d.dataset.Fields()
}

// Release releases any resources associated with the Dataset d,
// rendering it unusable in the future.  In this case it deletes
// the temporary copy of the Dataset.
func (d *DCopy) Release() error {
	if !d.isReleased {
		if err := d.dataset.Release(); err != nil {
			fmt.Printf("can't release underlying Dataset: %s", err)
		}
		d.isReleased = true
		return os.RemoveAll(d.tmpDir)
	}
	return ddataset.ErrReleased
}

// Next returns whether there is a Record to be Read
func (c *DCopyConn) Next() bool {
	return c.conn.Next()
}

// Err returns any errors from the connection
func (c *DCopyConn) Err() error {
	return c.conn.Err()
}

// Read returns the current Record
func (c *DCopyConn) Read() ddataset.Record {
	return c.conn.Read()
}

// Close closes the connection and deletes the copy
func (c *DCopyConn) Close() error {
	return c.conn.Close()
}

func getRecords(conn ddataset.Conn, num int) ([]ddataset.Record, error) {
	n := 0
	records := []ddataset.Record{}
	for n < num && conn.Next() {
		record := conn.Read().Clone()
		n++
		records = append(records, record)
	}
	return records, conn.Err()
}
<commit_msg>Remove release of sub-dataset in DCopy.Release<commit_after>/*
 * A Go package to copy a Dataset
 *
 * Copyright (C) 2017 Lawrence Woodman <lwoodman@vlifesystems.com>
 *
 * Licensed under an MIT licence.  Please see LICENCE.md for details.
 */

// Package dcopy copies a Dataset so that you can work consistently on
// the same Dataset.  This is important where a database is likely to be
// updated while you are working on it.  The copy of the database is stored
// in an sqlite3 database located in a temporary directory.
package dcopy

import (
	"encoding/csv"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/lawrencewoodman/ddataset"
	"github.com/lawrencewoodman/ddataset/dcsv"
)

// DCopy represents a copy of a Dataset
type DCopy struct {
	dataset    ddataset.Dataset
	tmpDir     string
	isReleased bool
}

// DCopyConn represents a connection to a DCopy Dataset
type DCopyConn struct {
	conn ddataset.Conn
	err  error
}

// New creates a new DCopy Dataset which will be a copy of the Dataset
// supplied at the time it is run. Please note that this creates a file
// on the disk containing a copy of the supplied Dataset.
func New(dataset ddataset.Dataset) (ddataset.Dataset, error) {
	tmpDir, err := ioutil.TempDir("", "dcopy")
	if err != nil {
		return nil, err
	}
	copyFilename := filepath.Join(tmpDir, "copy.csv")
	f, err := os.Create(copyFilename)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	w := csv.NewWriter(f)

	conn, err := dataset.Open()
	if err != nil {
		os.RemoveAll(tmpDir)
		return nil, err
	}
	defer conn.Close()

	strRecord := make([]string, len(dataset.Fields()))
	for conn.Next() {
		record := conn.Read()
		for i, f := range dataset.Fields() {
			strRecord[i] = record[f].String()
		}
		if err := w.Write(strRecord); err != nil {
			os.RemoveAll(tmpDir)
			return nil, fmt.Errorf("error writing record to csv copy: %s", err)
		}
	}

	if err := conn.Err(); err != nil {
		os.RemoveAll(tmpDir)
		return nil, err
	}

	w.Flush()
	if err := w.Error(); err != nil {
		os.RemoveAll(tmpDir)
		return nil, err
	}

	return &DCopy{
		dataset:    dcsv.New(copyFilename, false, ',', dataset.Fields()),
		tmpDir:     tmpDir,
		isReleased: false,
	}, nil
}

// Open creates a connection to the Dataset
func (d *DCopy) Open() (ddataset.Conn, error) {
	if d.isReleased {
		return nil, ddataset.ErrReleased
	}
	conn, err := d.dataset.Open()
	if err != nil {
		return nil, err
	}
	return &DCopyConn{
		conn: conn,
		err:  nil,
	}, nil
}

// Fields returns the field names used by the Dataset
func (d *DCopy) Fields() []string {
	if d.isReleased {
		return []string{}
	}
	return d.dataset.Fields()
}

// Release releases any resources associated with the Dataset d,
// rendering it unusable in the future.  In this case it deletes
// the temporary copy of the Dataset.
func (d *DCopy) Release() error {
	if !d.isReleased {
		d.isReleased = true
		return os.RemoveAll(d.tmpDir)
	}
	return ddataset.ErrReleased
}

// Next returns whether there is a Record to be Read
func (c *DCopyConn) Next() bool {
	return c.conn.Next()
}

// Err returns any errors from the connection
func (c *DCopyConn) Err() error {
	return c.conn.Err()
}

// Read returns the current Record
func (c *DCopyConn) Read() ddataset.Record {
	return c.conn.Read()
}

// Close closes the connection and deletes the copy
func (c *DCopyConn) Close() error {
	return c.conn.Close()
}

func getRecords(conn ddataset.Conn, num int) ([]ddataset.Record, error) {
	n := 0
	records := []ddataset.Record{}
	for n < num && conn.Next() {
		record := conn.Read().Clone()
		n++
		records = append(records, record)
	}
	return records, conn.Err()
}
<|endoftext|>


<commit_before>package b2

import (
	"bytes"
	"fmt"
	"net/http"
	"strconv"
	"strings"
)

const (
	DELETE_FILE_VERSION_URL string = "/b2api/v1/b2_delete_file_version"
	DOWNLOAD_FILE_BY_ID_URL string = "/b2api/v1/b2_download_file_by_id"
	GET_FILE_INFO_URL       string = "/b2api/v1/b2_get_file_info"
)

type FileVersion struct {
	FileId   string `json:"fileId"`
	FileName string `json:"fileName"`
}

type FileInfo struct {
	ContentLength int               `json:"contentLength"`
	ContentType   string            `json:"contentType"`
	FileId        string            `json:"fileId"`
	FileName      string            `json:"fileName"`
	ContentSha1   string            `json:"contentSha1"`
	BucketId      string            `json:"bucketId"`
	AccountId     string            `json:"accountId"`
	Info          map[string]string `json:"fileInfo"`
}

func (c *Client) DeleteFileVersion(fileName string, fileId string) (*FileVersion, error) {
	reqBody := bytes.NewBufferString(fmt.Sprintf(`{"fileName": "%s", "fileId": "%s"}`, fileName, fileId))
	if req, err := http.NewRequest("POST", c.buildRequestUrl(DELETE_FILE_VERSION_URL), reqBody); err != nil {
		return nil, err
	} else {
		c.setHeaders(req)

		var result FileVersion
		err = c.requestJson(req, &result)

		if err != nil {
			return nil, err
		}

		return &result, nil
	}
}

func (c *Client) DownloadFileById(fileId string) ([]byte, *FileInfo, error) {
	reqBody := bytes.NewBufferString(fmt.Sprintf(`{"fileId": "%s"}`, fileId))
	if req, err := http.NewRequest("POST", c.buildFileRequestUrl(DOWNLOAD_FILE_BY_ID_URL), reqBody); err != nil {
		return nil, nil, err
	} else {
		c.setHeaders(req)

		data, header, err := c.requestBytes(req)

		if err != nil {
			return nil, nil, err
		}

		contentLen, err := strconv.Atoi(header.Get("Content-Length"))

		info := &FileInfo{
			ContentLength: contentLen,
			ContentType:   header.Get("Content-Type"),
			FileId:        header.Get("X-Bz-File-Id"),
			FileName:      header.Get("X-Bz-File-Name"),
			ContentSha1:   header.Get("X-Bz-Content-Sha1"),
			Info:          make(map[string]string),
		}

		for key, _ := range header {
			// make the key lowercase as some of the headers B2 returns are lowercase, some uppercase...
			if strings.HasPrefix(strings.ToLower(key), "x-bz-info-") {
				info.Info[key] = header.Get(key)
			}
		}

		return data, info, nil
	}
}

func (c *Client) DownloadFileByName(bucketName string, fileName string) ([]byte, *FileInfo, error) {
	requestPath := fmt.Sprintf("/file/%s/%s", bucketName, fileName)
	if req, err := http.NewRequest("GET", c.buildFileRequestUrl(requestPath), nil); err != nil {
		return nil, nil, err
	} else {
		req.Header.Set("Authorization", c.AuthToken)

		data, header, err := c.requestBytes(req)

		if err != nil {
			return nil, nil, err
		}

		contentLen, err := strconv.Atoi(header.Get("Content-Length"))

		info := &FileInfo{
			ContentLength: contentLen,
			ContentType:   header.Get("Content-Type"),
			FileId:        header.Get("X-Bz-File-Id"),
			FileName:      header.Get("X-Bz-File-Name"),
			ContentSha1:   header.Get("X-Bz-Content-Sha1"),
			Info:          make(map[string]string),
		}

		for key, _ := range header {
			// make the key lowercase as some of the headers B2 returns are lowercase, some uppercase...
			if strings.HasPrefix(strings.ToLower(key), "x-bz-info-") {
				info.Info[key] = header.Get(key)
			}
		}

		return data, info, nil
	}
}

func (c *Client) GetFileInfo(fileId string) (*FileInfo, error) {
	reqBody := bytes.NewBufferString(fmt.Sprintf(`{"fileId": "%s"}`, fileId))
	if req, err := http.NewRequest("POST", c.buildRequestUrl(GET_FILE_INFO_URL), reqBody); err != nil {
		return nil, err
	} else {
		c.setHeaders(req)

		var result FileInfo
		err = c.requestJson(req, &result)

		if err != nil {
			return nil, err
		}

		return &result, nil
	}
}
<commit_msg>Added GetUploadUrl() and HideFile()<commit_after>package b2

import (
	"bytes"
	"fmt"
	"net/http"
	"strconv"
	"strings"
)

type HideFileAction string

const (
	HIDE_FILE_ACTION_HIDE   HideFileAction = "hide"
	HIDE_FILE_ACTION_UPLOAD                = "upload"
)

const (
	DELETE_FILE_VERSION_URL string = "/b2api/v1/b2_delete_file_version"
	DOWNLOAD_FILE_BY_ID_URL string = "/b2api/v1/b2_download_file_by_id"
	GET_FILE_INFO_URL       string = "/b2api/v1/b2_get_file_info"
	GET_UPLOAD_URL_URL      string = "/b2api/v1/b2_get_upload_url"
	HIDE_FILE_URL           string = "/b2api/v1/b2_hide_file"
)

type FileVersion struct {
	FileId   string `json:"fileId"`
	FileName string `json:"fileName"`
}

type FileInfo struct {
	ContentLength int               `json:"contentLength"`
	ContentType   string            `json:"contentType"`
	FileId        string            `json:"fileId"`
	FileName      string            `json:"fileName"`
	ContentSha1   string            `json:"contentSha1"`
	BucketId      string            `json:"bucketId"`
	AccountId     string            `json:"accountId"`
	Info          map[string]string `json:"fileInfo"`
}

type UploadUrlDetails struct {
	BucketId  string `json:"bucketId"`
	Url       string `json:"uploadUrl"`
	AuthToken string `json:"authorizationToken"`
}

type HiddenFile struct {
	FileId          string         `json:"fileId"`
	FileName        string         `json:"fileName"`
	UploadTimestamp int            `json:"uploadTimestamp"`
	Action          HideFileAction `json:"action"`
}

func (c *Client) DeleteFileVersion(fileName string, fileId string) (*FileVersion, error) {
	reqBody := bytes.NewBufferString(fmt.Sprintf(`{"fileName": "%s", "fileId": "%s"}`, fileName, fileId))
	if req, err := http.NewRequest("POST", c.buildRequestUrl(DELETE_FILE_VERSION_URL), reqBody); err != nil {
		return nil, err
	} else {
		c.setHeaders(req)

		var result FileVersion
		err = c.requestJson(req, &result)

		if err != nil {
			return nil, err
		}

		return &result, nil
	}
}

func (c *Client) DownloadFileById(fileId string) ([]byte, *FileInfo, error) {
	reqBody := bytes.NewBufferString(fmt.Sprintf(`{"fileId": "%s"}`, fileId))
	if req, err := http.NewRequest("POST", c.buildFileRequestUrl(DOWNLOAD_FILE_BY_ID_URL), reqBody); err != nil {
		return nil, nil, err
	} else {
		c.setHeaders(req)

		data, header, err := c.requestBytes(req)

		if err != nil {
			return nil, nil, err
		}

		contentLen, err := strconv.Atoi(header.Get("Content-Length"))

		info := &FileInfo{
			ContentLength: contentLen,
			ContentType:   header.Get("Content-Type"),
			FileId:        header.Get("X-Bz-File-Id"),
			FileName:      header.Get("X-Bz-File-Name"),
			ContentSha1:   header.Get("X-Bz-Content-Sha1"),
			Info:          make(map[string]string),
		}

		for key, _ := range header {
			// make the key lowercase as some of the headers B2 returns are lowercase, some uppercase...
			if strings.HasPrefix(strings.ToLower(key), "x-bz-info-") {
				info.Info[key] = header.Get(key)
			}
		}

		return data, info, nil
	}
}

func (c *Client) DownloadFileByName(bucketName string, fileName string) ([]byte, *FileInfo, error) {
	requestPath := fmt.Sprintf("/file/%s/%s", bucketName, fileName)
	if req, err := http.NewRequest("GET", c.buildFileRequestUrl(requestPath), nil); err != nil {
		return nil, nil, err
	} else {
		req.Header.Set("Authorization", c.AuthToken)

		data, header, err := c.requestBytes(req)

		if err != nil {
			return nil, nil, err
		}

		contentLen, err := strconv.Atoi(header.Get("Content-Length"))

		info := &FileInfo{
			ContentLength: contentLen,
			ContentType:   header.Get("Content-Type"),
			FileId:        header.Get("X-Bz-File-Id"),
			FileName:      header.Get("X-Bz-File-Name"),
			ContentSha1:   header.Get("X-Bz-Content-Sha1"),
			Info:          make(map[string]string),
		}

		for key, _ := range header {
			// make the key lowercase as some of the headers B2 returns are lowercase, some uppercase...
			if strings.HasPrefix(strings.ToLower(key), "x-bz-info-") {
				info.Info[key] = header.Get(key)
			}
		}

		return data, info, nil
	}
}

func (c *Client) GetFileInfo(fileId string) (*FileInfo, error) {
	reqBody := bytes.NewBufferString(fmt.Sprintf(`{"fileId": "%s"}`, fileId))
	if req, err := http.NewRequest("POST", c.buildRequestUrl(GET_FILE_INFO_URL), reqBody); err != nil {
		return nil, err
	} else {
		c.setHeaders(req)

		var result FileInfo
		err = c.requestJson(req, &result)

		if err != nil {
			return nil, err
		}

		return &result, nil
	}
}

func (c *Client) GetUploadUrl(bucketId string) (*UploadUrlDetails, error) {
	reqBody := bytes.NewBufferString(fmt.Sprintf(`{"bucketId": "%s"}`, bucketId))
	if req, err := http.NewRequest("POST", c.buildRequestUrl(GET_UPLOAD_URL_URL), reqBody); err != nil {
		return nil, err
	} else {
		c.setHeaders(req)

		var result UploadUrlDetails
		err = c.requestJson(req, &result)

		if err != nil {
			return nil, err
		}

		return &result, nil
	}
}

func (c *Client) HideFile(bucketId, fileName string) (*HiddenFile, error) {
	reqBody := bytes.NewBufferString(fmt.Sprintf(`{"bucketId": "%s", "fileName": "%s"}`, bucketId, fileName))
	if req, err := http.NewRequest("POST", c.buildRequestUrl(HIDE_FILE_URL), reqBody); err != nil {
		return nil, err
	} else {
		c.setHeaders(req)

		var result HiddenFile
		err = c.requestJson(req, &result)

		if err != nil {
			return nil, err
		}

		return &result, nil
	}
}
<|endoftext|>


<commit_before>package controllers

import (
	"github.com/robfig/revel"
	"go-url-shortener/app/shortener"
	"net/http"
)

type URL struct {
	*revel.Controller
}

func (c URL) Create(url string) revel.Result {
	slug, err := shortener.Put(url)
	if err != nil {
		return c.RenderError(err)
	}

	c.Response.Status = http.StatusCreated
	return c.RenderText(slug)
}

func (c URL) Retrieve(slug string) revel.Result {
	url, err := shortener.Get(slug)
	if err != nil {
		if _, ok := err.(*shortener.CannotFindShortUrlError); ok {
			return c.NotFound("Short url not found for %s", slug)
		}

		return c.RenderError(err)
	}

	return c.RenderText(url)
}
<commit_msg>return json instead of plain text<commit_after>package controllers

import (
	"github.com/robfig/revel"
	"go-url-shortener/app/shortener"
	"net/http"
)

type URL struct {
	*revel.Controller
}

type CreateResponse struct {
	Slug string
}

func (c URL) Create(url string) revel.Result {
	slug, err := shortener.Put(url)
	if err != nil {
		return c.RenderError(err)
	}

	c.Response.Status = http.StatusCreated
	return c.RenderJson(CreateResponse{slug})
}

type RetrieveResponse struct {
	URL string
}

func (c URL) Retrieve(slug string) revel.Result {
	url, err := shortener.Get(slug)
	if err != nil {
		if _, ok := err.(*shortener.CannotFindShortUrlError); ok {
			return c.NotFound("Short url not found for %s", slug)
		}

		return c.RenderError(err)
	}

	return c.RenderJson(RetrieveResponse{url})
}
<|endoftext|>


<commit_before>/*
Package bloom provides data structures and methods for creating Bloom filters.

A Bloom filter is a representation of a set of _n_ items, where the main
requirement is to make membership queries; _i.e._, whether an item is a
member of a set.

A Bloom filter has two parameters: _m_, a maximum size (typically a reasonably large
multiple of the cardinality of the set to represent) and _k_, the number of hashing
functions on elements of the set. (The actual hashing functions are important, too,
but this is not a parameter for this implementation). A Bloom filter is backed by
a BitSet; a key is represented in the filter by setting the bits at each value of the
hashing functions (modulo _m_). Set membership is done by _testing_ whether the
bits at each value of the hashing functions (again, modulo _m_) are set. If so,
the item is in the set. If the item is actually in the set, a Bloom filter will
never fail (the true positive rate is 1.0); but it is susceptible to false
positives. The art is to choose _k_ and _m_ correctly.

In this implementation, the hashing functions used is murmurhash,
a non-cryptographic hashing function.

This implementation accepts keys for setting as testing as []byte. Thus, to
add a string item, "Love":

    uint n = 1000
    filter := bloom.New(20*n, 5) // load of 20, 5 keys
    filter.Add([]byte("Love"))

Similarly, to test if "Love" is in bloom:

    if filter.Test([]byte("Love"))

For numeric data, I recommend that you look into the binary/encoding library. But,
for example, to add a uint32 to the filter:

    i := uint32(100)
    n1 := make([]byte,4)
    binary.BigEndian.PutUint32(n1,i)
    f.Add(n1)

Finally, there is a method to estimate the false positive rate of a particular
Bloom filter for a set of size _n_:

    if filter.EstimateFalsePositiveRate(1000) > 0.001

Given the particular hashing scheme, it's best to be empirical about this. Note
that estimating the FP rate will clear the Bloom filter.
*/
package bloom

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"math"

	"github.com/bits-and-blooms/bitset"
)

// A BloomFilter is a representation of a set of _n_ items, where the main
// requirement is to make membership queries; _i.e._, whether an item is a
// member of a set.
type BloomFilter struct {
	m uint
	k uint
	b *bitset.BitSet
}

func max(x, y uint) uint {
	if x > y {
		return x
	}
	return y
}

// New creates a new Bloom filter with _m_ bits and _k_ hashing functions
// We force _m_ and _k_ to be at least one to avoid panics.
func New(m uint, k uint) *BloomFilter {
	return &BloomFilter{max(1, m), max(1, k), bitset.New(m)}
}

// From creates a new Bloom filter with len(_data_) * 64 bits and _k_ hashing
// functions. The data slice is not going to be reset.
func From(data []uint64, k uint) *BloomFilter {
	m := uint(len(data) * 64)
	return &BloomFilter{m, k, bitset.From(data)}
}

// baseHashes returns the four hash values of data that are used to create k
// hashes
func baseHashes(data []byte) [4]uint64 {
	var d digest128 // murmur hashing
	hash1, hash2, hash3, hash4 := d.sum256(data)
	return [4]uint64{
		hash1, hash2, hash3, hash4,
	}
}

// location returns the ith hashed location using the four base hash values
func location(h [4]uint64, i uint) uint64 {
	ii := uint64(i)
	return h[ii%2] + ii*h[2+(((ii+(ii%2))%4)/2)]
}

// location returns the ith hashed location using the four base hash values
func (f *BloomFilter) location(h [4]uint64, i uint) uint {
	return uint(location(h, i) % uint64(f.m))
}

// EstimateParameters estimates requirements for m and k.
// Based on https://bitbucket.org/ww/bloom/src/829aa19d01d9/bloom.go
// used with permission.
func EstimateParameters(n uint, p float64) (m uint, k uint) {
	m = uint(math.Ceil(-1 * float64(n) * math.Log(p) / math.Pow(math.Log(2), 2)))
	k = uint(math.Ceil(math.Log(2) * float64(m) / float64(n)))
	return
}

// NewWithEstimates creates a new Bloom filter for about n items with fp
// false positive rate
func NewWithEstimates(n uint, fp float64) *BloomFilter {
	m, k := EstimateParameters(n, fp)
	return New(m, k)
}

// Cap returns the capacity, _m_, of a Bloom filter
func (f *BloomFilter) Cap() uint {
	return f.m
}

// K returns the number of hash functions used in the BloomFilter
func (f *BloomFilter) K() uint {
	return f.k
}

// Add data to the Bloom Filter. Returns the filter (allows chaining)
func (f *BloomFilter) Add(data []byte) *BloomFilter {
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		f.b.Set(f.location(h, i))
	}
	return f
}

// Merge the data from two Bloom Filters.
func (f *BloomFilter) Merge(g *BloomFilter) error {
	// Make sure the m's and k's are the same, otherwise merging has no real use.
	if f.m != g.m {
		return fmt.Errorf("m's don't match: %d != %d", f.m, g.m)
	}

	if f.k != g.k {
		return fmt.Errorf("k's don't match: %d != %d", f.m, g.m)
	}

	f.b.InPlaceUnion(g.b)
	return nil
}

// Copy creates a copy of a Bloom filter.
func (f *BloomFilter) Copy() *BloomFilter {
	fc := New(f.m, f.k)
	fc.Merge(f) // #nosec
	return fc
}

// AddString to the Bloom Filter. Returns the filter (allows chaining)
func (f *BloomFilter) AddString(data string) *BloomFilter {
	return f.Add([]byte(data))
}

// Test returns true if the data is in the BloomFilter, false otherwise.
// If true, the result might be a false positive. If false, the data
// is definitely not in the set.
func (f *BloomFilter) Test(data []byte) bool {
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		if !f.b.Test(f.location(h, i)) {
			return false
		}
	}
	return true
}

// TestString returns true if the string is in the BloomFilter, false otherwise.
// If true, the result might be a false positive. If false, the data
// is definitely not in the set.
func (f *BloomFilter) TestString(data string) bool {
	return f.Test([]byte(data))
}

// TestLocations returns true if all locations are set in the BloomFilter, false
// otherwise.
func (f *BloomFilter) TestLocations(locs []uint64) bool {
	for i := 0; i < len(locs); i++ {
		if !f.b.Test(uint(locs[i] % uint64(f.m))) {
			return false
		}
	}
	return true
}

// TestAndAdd is the equivalent to calling Test(data) then Add(data).
// Returns the result of Test.
func (f *BloomFilter) TestAndAdd(data []byte) bool {
	present := true
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		l := f.location(h, i)
		if !f.b.Test(l) {
			present = false
		}
		f.b.Set(l)
	}
	return present
}

// TestAndAddString is the equivalent to calling Test(string) then Add(string).
// Returns the result of Test.
func (f *BloomFilter) TestAndAddString(data string) bool {
	return f.TestAndAdd([]byte(data))
}

// ClearAll clears all the data in a Bloom filter, removing all keys
func (f *BloomFilter) ClearAll() *BloomFilter {
	f.b.ClearAll()
	return f
}

// EstimateFalsePositiveRate returns, for a BloomFilter with a estimate of m bits
// and k hash functions, what the false positive rate will be
// while storing n entries; runs 100,000 tests. This is an empirical
// test using integers as keys. As a side-effect, it clears the BloomFilter.
func (f *BloomFilter) EstimateFalsePositiveRate(n uint) (fpRate float64) {
	rounds := uint32(100000)
	f.ClearAll()
	n1 := make([]byte, 4)
	for i := uint32(0); i < uint32(n); i++ {
		binary.BigEndian.PutUint32(n1, i)
		f.Add(n1)
	}
	fp := 0
	// test for number of rounds
	for i := uint32(0); i < rounds; i++ {
		binary.BigEndian.PutUint32(n1, i+uint32(n)+1)
		if f.Test(n1) {
			//fmt.Printf("%v failed.\n", i+uint32(n)+1)
			fp++
		}
	}
	fpRate = float64(fp) / (float64(rounds))
	f.ClearAll()
	return
}

// bloomFilterJSON is an unexported type for marshaling/unmarshaling BloomFilter struct.
type bloomFilterJSON struct {
	M uint           `json:"m"`
	K uint           `json:"k"`
	B *bitset.BitSet `json:"b"`
}

// MarshalJSON implements json.Marshaler interface.
func (f *BloomFilter) MarshalJSON() ([]byte, error) {
	return json.Marshal(bloomFilterJSON{f.m, f.k, f.b})
}

// UnmarshalJSON implements json.Unmarshaler interface.
func (f *BloomFilter) UnmarshalJSON(data []byte) error {
	var j bloomFilterJSON
	err := json.Unmarshal(data, &j)
	if err != nil {
		return err
	}
	f.m = j.M
	f.k = j.K
	f.b = j.B
	return nil
}

// WriteTo writes a binary representation of the BloomFilter to an i/o stream.
// It returns the number of bytes written.
func (f *BloomFilter) WriteTo(stream io.Writer) (int64, error) {
	err := binary.Write(stream, binary.BigEndian, uint64(f.m))
	if err != nil {
		return 0, err
	}
	err = binary.Write(stream, binary.BigEndian, uint64(f.k))
	if err != nil {
		return 0, err
	}
	numBytes, err := f.b.WriteTo(stream)
	return numBytes + int64(2*binary.Size(uint64(0))), err
}

// ReadFrom reads a binary representation of the BloomFilter (such as might
// have been written by WriteTo()) from an i/o stream. It returns the number
// of bytes read.
func (f *BloomFilter) ReadFrom(stream io.Reader) (int64, error) {
	var m, k uint64
	err := binary.Read(stream, binary.BigEndian, &m)
	if err != nil {
		return 0, err
	}
	err = binary.Read(stream, binary.BigEndian, &k)
	if err != nil {
		return 0, err
	}
	b := &bitset.BitSet{}
	numBytes, err := b.ReadFrom(stream)
	if err != nil {
		return 0, err
	}
	f.m = uint(m)
	f.k = uint(k)
	f.b = b
	return numBytes + int64(2*binary.Size(uint64(0))), nil
}

// GobEncode implements gob.GobEncoder interface.
func (f *BloomFilter) GobEncode() ([]byte, error) {
	var buf bytes.Buffer
	_, err := f.WriteTo(&buf)
	if err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

// GobDecode implements gob.GobDecoder interface.
func (f *BloomFilter) GobDecode(data []byte) error {
	buf := bytes.NewBuffer(data)
	_, err := f.ReadFrom(buf)

	return err
}

// Equal tests for the equality of two Bloom filters
func (f *BloomFilter) Equal(g *BloomFilter) bool {
	return f.m == g.m && f.k == g.k && f.b.Equal(g.b)
}

// Locations returns a list of hash locations representing a data item.
func Locations(data []byte, k uint) []uint64 {
	locs := make([]uint64, k)

	// calculate locations
	h := baseHashes(data)
	for i := uint(0); i < k; i++ {
		locs[i] = location(h, i)
	}

	return locs
}
<commit_msg>bloom: expose bitset with a func<commit_after>/*
Package bloom provides data structures and methods for creating Bloom filters.

A Bloom filter is a representation of a set of _n_ items, where the main
requirement is to make membership queries; _i.e._, whether an item is a
member of a set.

A Bloom filter has two parameters: _m_, a maximum size (typically a reasonably large
multiple of the cardinality of the set to represent) and _k_, the number of hashing
functions on elements of the set. (The actual hashing functions are important, too,
but this is not a parameter for this implementation). A Bloom filter is backed by
a BitSet; a key is represented in the filter by setting the bits at each value of the
hashing functions (modulo _m_). Set membership is done by _testing_ whether the
bits at each value of the hashing functions (again, modulo _m_) are set. If so,
the item is in the set. If the item is actually in the set, a Bloom filter will
never fail (the true positive rate is 1.0); but it is susceptible to false
positives. The art is to choose _k_ and _m_ correctly.

In this implementation, the hashing functions used is murmurhash,
a non-cryptographic hashing function.

This implementation accepts keys for setting as testing as []byte. Thus, to
add a string item, "Love":

    uint n = 1000
    filter := bloom.New(20*n, 5) // load of 20, 5 keys
    filter.Add([]byte("Love"))

Similarly, to test if "Love" is in bloom:

    if filter.Test([]byte("Love"))

For numeric data, I recommend that you look into the binary/encoding library. But,
for example, to add a uint32 to the filter:

    i := uint32(100)
    n1 := make([]byte,4)
    binary.BigEndian.PutUint32(n1,i)
    f.Add(n1)

Finally, there is a method to estimate the false positive rate of a particular
Bloom filter for a set of size _n_:

    if filter.EstimateFalsePositiveRate(1000) > 0.001

Given the particular hashing scheme, it's best to be empirical about this. Note
that estimating the FP rate will clear the Bloom filter.
*/
package bloom

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"math"

	"github.com/bits-and-blooms/bitset"
)

// A BloomFilter is a representation of a set of _n_ items, where the main
// requirement is to make membership queries; _i.e._, whether an item is a
// member of a set.
type BloomFilter struct {
	m uint
	k uint
	b *bitset.BitSet
}

func max(x, y uint) uint {
	if x > y {
		return x
	}
	return y
}

// New creates a new Bloom filter with _m_ bits and _k_ hashing functions
// We force _m_ and _k_ to be at least one to avoid panics.
func New(m uint, k uint) *BloomFilter {
	return &BloomFilter{max(1, m), max(1, k), bitset.New(m)}
}

// From creates a new Bloom filter with len(_data_) * 64 bits and _k_ hashing
// functions. The data slice is not going to be reset.
func From(data []uint64, k uint) *BloomFilter {
	m := uint(len(data) * 64)
	return &BloomFilter{m, k, bitset.From(data)}
}

// baseHashes returns the four hash values of data that are used to create k
// hashes
func baseHashes(data []byte) [4]uint64 {
	var d digest128 // murmur hashing
	hash1, hash2, hash3, hash4 := d.sum256(data)
	return [4]uint64{
		hash1, hash2, hash3, hash4,
	}
}

// location returns the ith hashed location using the four base hash values
func location(h [4]uint64, i uint) uint64 {
	ii := uint64(i)
	return h[ii%2] + ii*h[2+(((ii+(ii%2))%4)/2)]
}

// location returns the ith hashed location using the four base hash values
func (f *BloomFilter) location(h [4]uint64, i uint) uint {
	return uint(location(h, i) % uint64(f.m))
}

// EstimateParameters estimates requirements for m and k.
// Based on https://bitbucket.org/ww/bloom/src/829aa19d01d9/bloom.go
// used with permission.
func EstimateParameters(n uint, p float64) (m uint, k uint) {
	m = uint(math.Ceil(-1 * float64(n) * math.Log(p) / math.Pow(math.Log(2), 2)))
	k = uint(math.Ceil(math.Log(2) * float64(m) / float64(n)))
	return
}

// NewWithEstimates creates a new Bloom filter for about n items with fp
// false positive rate
func NewWithEstimates(n uint, fp float64) *BloomFilter {
	m, k := EstimateParameters(n, fp)
	return New(m, k)
}

// Cap returns the capacity, _m_, of a Bloom filter
func (f *BloomFilter) Cap() uint {
	return f.m
}

// K returns the number of hash functions used in the BloomFilter
func (f *BloomFilter) K() uint {
	return f.k
}

// BitSet returns the underlying bit set for this filter.
func (f *BloomFilter) BitSet() *bitset.BitSet {
	return f.b
}

// Add data to the Bloom Filter. Returns the filter (allows chaining)
func (f *BloomFilter) Add(data []byte) *BloomFilter {
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		f.b.Set(f.location(h, i))
	}
	return f
}

// Merge the data from two Bloom Filters.
func (f *BloomFilter) Merge(g *BloomFilter) error {
	// Make sure the m's and k's are the same, otherwise merging has no real use.
	if f.m != g.m {
		return fmt.Errorf("m's don't match: %d != %d", f.m, g.m)
	}

	if f.k != g.k {
		return fmt.Errorf("k's don't match: %d != %d", f.m, g.m)
	}

	f.b.InPlaceUnion(g.b)
	return nil
}

// Copy creates a copy of a Bloom filter.
func (f *BloomFilter) Copy() *BloomFilter {
	fc := New(f.m, f.k)
	fc.Merge(f) // #nosec
	return fc
}

// AddString to the Bloom Filter. Returns the filter (allows chaining)
func (f *BloomFilter) AddString(data string) *BloomFilter {
	return f.Add([]byte(data))
}

// Test returns true if the data is in the BloomFilter, false otherwise.
// If true, the result might be a false positive. If false, the data
// is definitely not in the set.
func (f *BloomFilter) Test(data []byte) bool {
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		if !f.b.Test(f.location(h, i)) {
			return false
		}
	}
	return true
}

// TestString returns true if the string is in the BloomFilter, false otherwise.
// If true, the result might be a false positive. If false, the data
// is definitely not in the set.
func (f *BloomFilter) TestString(data string) bool {
	return f.Test([]byte(data))
}

// TestLocations returns true if all locations are set in the BloomFilter, false
// otherwise.
func (f *BloomFilter) TestLocations(locs []uint64) bool {
	for i := 0; i < len(locs); i++ {
		if !f.b.Test(uint(locs[i] % uint64(f.m))) {
			return false
		}
	}
	return true
}

// TestAndAdd is the equivalent to calling Test(data) then Add(data).
// Returns the result of Test.
func (f *BloomFilter) TestAndAdd(data []byte) bool {
	present := true
	h := baseHashes(data)
	for i := uint(0); i < f.k; i++ {
		l := f.location(h, i)
		if !f.b.Test(l) {
			present = false
		}
		f.b.Set(l)
	}
	return present
}

// TestAndAddString is the equivalent to calling Test(string) then Add(string).
// Returns the result of Test.
func (f *BloomFilter) TestAndAddString(data string) bool {
	return f.TestAndAdd([]byte(data))
}

// ClearAll clears all the data in a Bloom filter, removing all keys
func (f *BloomFilter) ClearAll() *BloomFilter {
	f.b.ClearAll()
	return f
}

// EstimateFalsePositiveRate returns, for a BloomFilter with a estimate of m bits
// and k hash functions, what the false positive rate will be
// while storing n entries; runs 100,000 tests. This is an empirical
// test using integers as keys. As a side-effect, it clears the BloomFilter.
func (f *BloomFilter) EstimateFalsePositiveRate(n uint) (fpRate float64) {
	rounds := uint32(100000)
	f.ClearAll()
	n1 := make([]byte, 4)
	for i := uint32(0); i < uint32(n); i++ {
		binary.BigEndian.PutUint32(n1, i)
		f.Add(n1)
	}
	fp := 0
	// test for number of rounds
	for i := uint32(0); i < rounds; i++ {
		binary.BigEndian.PutUint32(n1, i+uint32(n)+1)
		if f.Test(n1) {
			//fmt.Printf("%v failed.\n", i+uint32(n)+1)
			fp++
		}
	}
	fpRate = float64(fp) / (float64(rounds))
	f.ClearAll()
	return
}

// bloomFilterJSON is an unexported type for marshaling/unmarshaling BloomFilter struct.
type bloomFilterJSON struct {
	M uint           `json:"m"`
	K uint           `json:"k"`
	B *bitset.BitSet `json:"b"`
}

// MarshalJSON implements json.Marshaler interface.
func (f *BloomFilter) MarshalJSON() ([]byte, error) {
	return json.Marshal(bloomFilterJSON{f.m, f.k, f.b})
}

// UnmarshalJSON implements json.Unmarshaler interface.
func (f *BloomFilter) UnmarshalJSON(data []byte) error {
	var j bloomFilterJSON
	err := json.Unmarshal(data, &j)
	if err != nil {
		return err
	}
	f.m = j.M
	f.k = j.K
	f.b = j.B
	return nil
}

// WriteTo writes a binary representation of the BloomFilter to an i/o stream.
// It returns the number of bytes written.
func (f *BloomFilter) WriteTo(stream io.Writer) (int64, error) {
	err := binary.Write(stream, binary.BigEndian, uint64(f.m))
	if err != nil {
		return 0, err
	}
	err = binary.Write(stream, binary.BigEndian, uint64(f.k))
	if err != nil {
		return 0, err
	}
	numBytes, err := f.b.WriteTo(stream)
	return numBytes + int64(2*binary.Size(uint64(0))), err
}

// ReadFrom reads a binary representation of the BloomFilter (such as might
// have been written by WriteTo()) from an i/o stream. It returns the number
// of bytes read.
func (f *BloomFilter) ReadFrom(stream io.Reader) (int64, error) {
	var m, k uint64
	err := binary.Read(stream, binary.BigEndian, &m)
	if err != nil {
		return 0, err
	}
	err = binary.Read(stream, binary.BigEndian, &k)
	if err != nil {
		return 0, err
	}
	b := &bitset.BitSet{}
	numBytes, err := b.ReadFrom(stream)
	if err != nil {
		return 0, err
	}
	f.m = uint(m)
	f.k = uint(k)
	f.b = b
	return numBytes + int64(2*binary.Size(uint64(0))), nil
}

// GobEncode implements gob.GobEncoder interface.
func (f *BloomFilter) GobEncode() ([]byte, error) {
	var buf bytes.Buffer
	_, err := f.WriteTo(&buf)
	if err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

// GobDecode implements gob.GobDecoder interface.
func (f *BloomFilter) GobDecode(data []byte) error {
	buf := bytes.NewBuffer(data)
	_, err := f.ReadFrom(buf)

	return err
}

// Equal tests for the equality of two Bloom filters
func (f *BloomFilter) Equal(g *BloomFilter) bool {
	return f.m == g.m && f.k == g.k && f.b.Equal(g.b)
}

// Locations returns a list of hash locations representing a data item.
func Locations(data []byte, k uint) []uint64 {
	locs := make([]uint64, k)

	// calculate locations
	h := baseHashes(data)
	for i := uint(0); i < k; i++ {
		locs[i] = location(h, i)
	}

	return locs
}
<|endoftext|>


<commit_before>// +build pkcs11,darwin

package yubikey

var possiblePkcs11Libs = []string{
	"/usr/local/lib/libykcs11.dylib",
	"/usr/local/docker/lib/libykcs11.dylib",
}
<commit_msg>Adding another path to search for ykcs libs<commit_after>// +build pkcs11,darwin

package yubikey

var possiblePkcs11Libs = []string{
	"/usr/local/lib/libykcs11.dylib",
	"/usr/local/docker/lib/libykcs11.dylib",
	"/usr/local/docker-experimental/lib/libykcs11.dylib",
}
<|endoftext|>