<commit_before>// Package camoproxy provides an HTTP proxy server with content type
// restrictions as well as regex host allow and deny list support.
package camoproxy

import (
	"code.google.com/p/gorilla/mux"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/cactus/gologit"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

// Logger for handling logging.
var Logger = gologit.New(false)

var ServerName = "go-camo"

// Headers that are acceptible to pass from the client to the remote
// server. Only those present and true, are forwarded.
var validReqHeaders = map[string]bool{
	"Accept":            true,
	"Accept-Charset":    true,
	"Accept-Encoding":   true,
	"Accept-Language":   true,
	"Cache-Control":     true,
	"If-None-Match":     true,
	"If-Modified-Since": true,
}

// Headers that are acceptible to pass from the remote server to the
// client. Only those present and true, are forwarded. Empty implies
// no filtering.
var validRespHeaders = map[string]bool{}

// ProxyConfig holds configuration data used when creating a
// ProxyHandler with New.
type ProxyConfig struct {
	// HmacKey is a string to be used as the hmac key
	HmacKey         string
	// AllowList is a list of string represenstations of regex (not compiled
	// regex) that are used as a whitelist filter. If an AllowList is present,
	// then anything not matching is dropped. If no AllowList is present,
	// no Allow filtering is done.
	AllowList       []string
	// DenyList is a list of string represenstations of regex (not compiled
	// regex). The deny filter check occurs after the allow filter check
	// (if any).
	DenyList        []string
	// MaxSize is the maximum valid image size response (in bytes).
	MaxSize         int64
	// FollowRedirects is a boolean that specifies whether upstream redirects
	// are followed (10 depth) or not.
	FollowRedirects bool
	// Request timeout is a timeout for fetching upstream data.
	RequestTimeout  uint
}

// A ProxyHandler is a Camo like HTTP proxy, that provides content type
// restrictions as well as regex host allow and deny list support
type ProxyHandler struct {
	client    *http.Client
	hmacKey   []byte
	allowList []*regexp.Regexp
	denyList  []*regexp.Regexp
	maxSize   int64
	stats     *proxyStats
}

// StatsHandler returns an http.Handler that returns running totals and stats
// about the server.
func (p *ProxyHandler) StatsHandler() http.Handler {
	p.stats.Enable = true
	return http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.WriteHeader(200)
			c, b := p.stats.GetStats()
			fmt.Fprintf(w, "ClientsServed, BytesServed\n%d, %d\n", c, b)
		})
}

// ServerHTTP handles the client request, validates the request is validly
// HMAC signed, filters based on the Allow/Deny list, and then proxies
// valid requests to the desired endpoint. Responses are filtered for
// proper image content types.
func (p *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	Logger.Debugln("Request:", req.URL)
	if p.stats.Enable {
		p.stats.AddServed()
	}

	if ServerName != "" {
		h := w.Header()
		h.Set("Server", ServerName)
	}

	vars := mux.Vars(req)
	surl, ok := DecodeUrl(&p.hmacKey, vars["sigHash"], vars["encodedUrl"])
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	Logger.Debugln("URL:", surl)

	u, err := url.Parse(surl)
	if err != nil {
		Logger.Debugln(err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	if u.Host == "" {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if allowList is set, require match
	matchFound := true
	if len(p.allowList) > 0 {
		matchFound = false
		for _, rgx := range p.allowList {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out denyList urls based on regexes. Do this second
	// as denyList takes precedence
	for _, rgx := range p.denyList {
		if rgx.MatchString(u.Host) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest("GET", surl, nil)
	if err != nil {
		Logger.Debugln("Could not create NewRequest", err)
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	}

	// filter headers
	p.copyHeader(&nreq.Header, &req.Header, &validReqHeaders)
	nreq.Header.Add("connection", "close")
	nreq.Header.Add("user-agent", "pew pew pew")

	resp, err := p.client.Do(nreq)
	if err != nil {
		Logger.Debugln("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()

	// check for too large a response
	if resp.ContentLength > p.maxSize {
		Logger.Debugln("Content length exceeded", surl)
		http.Error(w, "Content length exceeded", http.StatusNotFound)
		return
	}

	switch resp.StatusCode {
	case 200:
		// check content type
		ct, ok := resp.Header[http.CanonicalHeaderKey("content-type")]
		if !ok || ct[0][:6] != "image/" {
			Logger.Debugln("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	case 300:
		Logger.Debugln("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303:
		// if we get a redirect here, we either disabled following,
		// or followed until max depth and still got one (redirect loop)
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 304:
		h := w.Header()
		h.Set("X-Content-Type-Options", "nosniff")
		w.WriteHeader(304)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		// upstream errors should probably just 502. client can try later.
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	default:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	h := w.Header()
	p.copyHeader(&h, &resp.Header, &validRespHeaders)
	h.Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(resp.StatusCode)
	bW, err := io.Copy(w, resp.Body)
	if err != nil {
		Logger.Println("Error writing response:", err)
		return
	}
	if p.stats.Enable {
		p.stats.AddBytes(bW)
	}
	Logger.Debugln(req, resp.StatusCode)
}

// copy headers from src into dst
// empty filter map will result in no filtering being done
func (p *ProxyHandler) copyHeader(dst, src *http.Header, filter *map[string]bool) {
	f := *filter
	filtering := false
	if len(f) > 0 {
		filtering = true
	}

	for k, vv := range *src {
		if x, ok := f[k]; filtering && (!ok || !x) {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

// Returns a new ProxyHandler. An error is returned if there was a failure
// to parse the regex from the passed ProxyConfig.
func New(pc ProxyConfig) (*ProxyHandler, error) {
	tr := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			// 2 second timeout on requests
			timeout := time.Second * time.Duration(pc.RequestTimeout)
			c, err := net.DialTimeout(netw, addr, timeout)
			if err != nil {
				return nil, err
			}
			// also set time limit on reading
			c.SetDeadline(time.Now().Add(timeout))
			return c, nil
		}}

	// spawn an idle conn trimmer
	go func() {
		time.Sleep(5 * time.Minute)
		tr.CloseIdleConnections()
	}()

	// build/compile regex
	client := &http.Client{Transport: tr}
	if !pc.FollowRedirects {
		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return errors.New("Not following redirect")
		}
	}

	allow := make([]*regexp.Regexp, 0)
	deny := make([]*regexp.Regexp, 0)

	var c *regexp.Regexp
	var err error
	for _, v := range pc.DenyList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		deny = append(deny, c)
	}
	for _, v := range pc.AllowList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		allow = append(allow, c)
	}

	return &ProxyHandler{
		client:    client,
		hmacKey:   []byte(pc.HmacKey),
		allowList: allow,
		denyList:  deny,
		maxSize:   pc.MaxSize,
		stats:     &proxyStats{}}, nil
}

// DecodeUrl ensures the url is properly verified via HMAC, and then
// unencodes the url, returning the url (if valid) and whether the
// HMAC was verified.
func DecodeUrl(hmackey *[]byte, hexdig string, hexurl string) (surl string, valid bool) {
	urlBytes, err := hex.DecodeString(hexurl)
	if err != nil {
		Logger.Debugln("Bad Hex Decode", hexurl)
		return
	}
	surl = string(urlBytes)
	mac := hmac.New(sha1.New, *hmackey)
	mac.Write([]byte(surl))
	macSum := hex.EncodeToString(mac.Sum([]byte{}))
	if macSum != hexdig {
		Logger.Debugf("Bad signature: %s != %s\n", macSum, hexdig)
		return
	}
	valid = true
	return
}

func EncodeUrl(hmacKey *[]byte, oUrl string) string {
	mac := hmac.New(sha1.New, *hmacKey)
	mac.Write([]byte(oUrl))
	macSum := hex.EncodeToString(mac.Sum([]byte{}))
	encodedUrl := hex.EncodeToString([]byte(oUrl))
	hexurl := "/" + macSum + "/" + encodedUrl
	return hexurl
}

<commit_msg>make header filters public (and modifiable) and filter response headers<commit_after>// Package camoproxy provides an HTTP proxy server with content type
// restrictions as well as regex host allow and deny list support.
package camoproxy

import (
	"code.google.com/p/gorilla/mux"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/cactus/gologit"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

// Logger for handling logging.
var Logger = gologit.New(false)

var ServerName = "go-camo"

// Headers that are acceptible to pass from the client to the remote
// server. Only those present and true, are forwarded. Empty implies
// no filtering.
var ValidReqHeaders = map[string]bool{
	"Accept":            true,
	"Accept-Charset":    true,
	"Accept-Encoding":   true,
	"Accept-Language":   true,
	"Cache-Control":     true,
	"If-None-Match":     true,
	"If-Modified-Since": true,
}

// Headers that are acceptible to pass from the remote server to the
// client. Only those present and true, are forwarded. Empty implies
// no filtering.
var ValidRespHeaders = map[string]bool{
	// Do not offer to accept range requests
	"Accept-Ranges":     false,
	"Cache-Control":     true,
	"Content-Encoding":  true,
	"Content-Type":      true,
	"Transfer-Encoding": true,
	"Expires":           true,
	"Last-Modified":     true,
	// override with either nothing, or ServerName
	"Server":            false,
	}

// ProxyConfig holds configuration data used when creating a
// ProxyHandler with New.
type ProxyConfig struct {
	// HmacKey is a string to be used as the hmac key
	HmacKey         string
	// AllowList is a list of string represenstations of regex (not compiled
	// regex) that are used as a whitelist filter. If an AllowList is present,
	// then anything not matching is dropped. If no AllowList is present,
	// no Allow filtering is done.
	AllowList       []string
	// DenyList is a list of string represenstations of regex (not compiled
	// regex). The deny filter check occurs after the allow filter check
	// (if any).
	DenyList        []string
	// MaxSize is the maximum valid image size response (in bytes).
	MaxSize         int64
	// FollowRedirects is a boolean that specifies whether upstream redirects
	// are followed (10 depth) or not.
	FollowRedirects bool
	// Request timeout is a timeout for fetching upstream data.
	RequestTimeout  uint
}

// A ProxyHandler is a Camo like HTTP proxy, that provides content type
// restrictions as well as regex host allow and deny list support
type ProxyHandler struct {
	client    *http.Client
	hmacKey   []byte
	allowList []*regexp.Regexp
	denyList  []*regexp.Regexp
	maxSize   int64
	stats     *proxyStats
}

// StatsHandler returns an http.Handler that returns running totals and stats
// about the server.
func (p *ProxyHandler) StatsHandler() http.Handler {
	p.stats.Enable = true
	return http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.WriteHeader(200)
			c, b := p.stats.GetStats()
			fmt.Fprintf(w, "ClientsServed, BytesServed\n%d, %d\n", c, b)
		})
}

// ServerHTTP handles the client request, validates the request is validly
// HMAC signed, filters based on the Allow/Deny list, and then proxies
// valid requests to the desired endpoint. Responses are filtered for
// proper image content types.
func (p *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	Logger.Debugln("Request:", req.URL)
	if p.stats.Enable {
		p.stats.AddServed()
	}

	if ServerName != "" {
		h := w.Header()
		h.Set("Server", ServerName)
	}

	vars := mux.Vars(req)
	surl, ok := DecodeUrl(&p.hmacKey, vars["sigHash"], vars["encodedUrl"])
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	Logger.Debugln("URL:", surl)

	u, err := url.Parse(surl)
	if err != nil {
		Logger.Debugln(err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	if u.Host == "" {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if allowList is set, require match
	matchFound := true
	if len(p.allowList) > 0 {
		matchFound = false
		for _, rgx := range p.allowList {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out denyList urls based on regexes. Do this second
	// as denyList takes precedence
	for _, rgx := range p.denyList {
		if rgx.MatchString(u.Host) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest("GET", surl, nil)
	if err != nil {
		Logger.Debugln("Could not create NewRequest", err)
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	}

	// filter headers
	p.copyHeader(&nreq.Header, &req.Header, &ValidReqHeaders)
	nreq.Header.Add("connection", "close")
	nreq.Header.Add("user-agent", "pew pew pew")

	resp, err := p.client.Do(nreq)
	if err != nil {
		Logger.Debugln("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()

	// check for too large a response
	if resp.ContentLength > p.maxSize {
		Logger.Debugln("Content length exceeded", surl)
		http.Error(w, "Content length exceeded", http.StatusNotFound)
		return
	}

	switch resp.StatusCode {
	case 200:
		// check content type
		ct, ok := resp.Header[http.CanonicalHeaderKey("content-type")]
		if !ok || ct[0][:6] != "image/" {
			Logger.Debugln("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	case 300:
		Logger.Debugln("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303:
		// if we get a redirect here, we either disabled following,
		// or followed until max depth and still got one (redirect loop)
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 304:
		h := w.Header()
		p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
		h.Set("X-Content-Type-Options", "nosniff")
		w.WriteHeader(304)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		// upstream errors should probably just 502. client can try later.
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	default:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	h := w.Header()
	p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
	h.Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(resp.StatusCode)
	bW, err := io.Copy(w, resp.Body)
	if err != nil {
		Logger.Println("Error writing response:", err)
		return
	}
	if p.stats.Enable {
		p.stats.AddBytes(bW)
	}
	Logger.Debugln(req, resp.StatusCode)
}

// copy headers from src into dst
// empty filter map will result in no filtering being done
func (p *ProxyHandler) copyHeader(dst, src *http.Header, filter *map[string]bool) {
	f := *filter
	filtering := false
	if len(f) > 0 {
		filtering = true
	}

	for k, vv := range *src {
		if x, ok := f[k]; filtering && (!ok || !x) {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

// Returns a new ProxyHandler. An error is returned if there was a failure
// to parse the regex from the passed ProxyConfig.
func New(pc ProxyConfig) (*ProxyHandler, error) {
	tr := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			// 2 second timeout on requests
			timeout := time.Second * time.Duration(pc.RequestTimeout)
			c, err := net.DialTimeout(netw, addr, timeout)
			if err != nil {
				return nil, err
			}
			// also set time limit on reading
			c.SetDeadline(time.Now().Add(timeout))
			return c, nil
		}}

	// spawn an idle conn trimmer
	go func() {
		time.Sleep(5 * time.Minute)
		tr.CloseIdleConnections()
	}()

	// build/compile regex
	client := &http.Client{Transport: tr}
	if !pc.FollowRedirects {
		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return errors.New("Not following redirect")
		}
	}

	allow := make([]*regexp.Regexp, 0)
	deny := make([]*regexp.Regexp, 0)

	var c *regexp.Regexp
	var err error
	for _, v := range pc.DenyList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		deny = append(deny, c)
	}
	for _, v := range pc.AllowList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		allow = append(allow, c)
	}

	return &ProxyHandler{
		client:    client,
		hmacKey:   []byte(pc.HmacKey),
		allowList: allow,
		denyList:  deny,
		maxSize:   pc.MaxSize,
		stats:     &proxyStats{}}, nil
}

// DecodeUrl ensures the url is properly verified via HMAC, and then
// unencodes the url, returning the url (if valid) and whether the
// HMAC was verified.
func DecodeUrl(hmackey *[]byte, hexdig string, hexurl string) (surl string, valid bool) {
	urlBytes, err := hex.DecodeString(hexurl)
	if err != nil {
		Logger.Debugln("Bad Hex Decode", hexurl)
		return
	}
	surl = string(urlBytes)
	mac := hmac.New(sha1.New, *hmackey)
	mac.Write([]byte(surl))
	macSum := hex.EncodeToString(mac.Sum([]byte{}))
	if macSum != hexdig {
		Logger.Debugf("Bad signature: %s != %s\n", macSum, hexdig)
		return
	}
	valid = true
	return
}

func EncodeUrl(hmacKey *[]byte, oUrl string) string {
	mac := hmac.New(sha1.New, *hmacKey)
	mac.Write([]byte(oUrl))
	macSum := hex.EncodeToString(mac.Sum([]byte{}))
	encodedUrl := hex.EncodeToString([]byte(oUrl))
	hexurl := "/" + macSum + "/" + encodedUrl
	return hexurl
}

<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ha

import (
	"log"
	"sort"
	"testing"

	"knative.dev/pkg/system"
	"knative.dev/pkg/test/logstream"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"knative.dev/pkg/ptr"
	"knative.dev/serving/pkg/apis/autoscaling"
	revisionresourcenames "knative.dev/serving/pkg/reconciler/revision/resources/names"
	rtesting "knative.dev/serving/pkg/testing/v1"
	"knative.dev/serving/test"
	"knative.dev/serving/test/e2e"
)

const (
	activatorDeploymentName = "activator"
	activatorLabel          = "app=activator"
	minProbes               = 400  // We want to send at least 400 requests.
	SLO                     = 0.99 // We permit 0.01 of requests to fail due to killing the Activator.
)

// The Activator does not have leader election enabled.
// The test ensures that stopping one of the activator pods doesn't affect user applications.
// One service is probed during activator restarts and another service is used for testing
// that we can scale from zero after activator restart.
func TestActivatorHA(t *testing.T) {
	clients := e2e.Setup(t)
	cancel := logstream.Start(t)
	defer cancel()

	if err := waitForDeploymentScale(clients, activatorDeploymentName, haReplicas); err != nil {
		t.Fatalf("Deployment %s not scaled to %d: %v", activatorDeploymentName, haReplicas, err)
	}

	// Create first service that we will continually probe during activator restart.
	names, resources := createPizzaPlanetService(t,
		rtesting.WithConfigAnnotations(map[string]string{
			autoscaling.MinScaleAnnotationKey:  "1",  // Make sure we don't scale to zero during the test.
			autoscaling.TargetBurstCapacityKey: "-1", // Make sure all requests go through the activator.
		}),
	)
	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	// Create second service that will be scaled to zero and after stopping the activator we'll
	// ensure it can be scaled back from zero.
	namesScaleToZero, resourcesScaleToZero := createPizzaPlanetService(t,
		rtesting.WithConfigAnnotations(map[string]string{
			autoscaling.WindowAnnotationKey:    autoscaling.WindowMin.String(), // Make sure we scale to zero quickly.
			autoscaling.TargetBurstCapacityKey: "-1",                           // Make sure all requests go through the activator.
		}),
	)
	test.CleanupOnInterrupt(func() { test.TearDown(clients, namesScaleToZero) })
	defer test.TearDown(clients, namesScaleToZero)

	if err := e2e.WaitForScaleToZero(t, revisionresourcenames.Deployment(resourcesScaleToZero.Revision), clients); err != nil {
		t.Fatal("Failed to scale to zero:", err)
	}

	prober := test.NewProberManager(log.Printf, clients, minProbes)
	prober.Spawn(resources.Service.Status.URL.URL())
	defer assertSLO(t, prober)

	pods, err := clients.KubeClient.Kube.CoreV1().Pods(system.Namespace()).List(metav1.ListOptions{
		LabelSelector: activatorLabel,
	})
	if err != nil {
		t.Fatal("Failed to get activator pods:", err)
	}
	activatorPod := pods.Items[0].Name

	origEndpoints, err := getPublicEndpoints(t, clients, resourcesScaleToZero.Revision.Name)
	if err != nil {
		t.Fatalf("Unable to get public endpoints for revision %s: %v", resourcesScaleToZero.Revision.Name, err)
	}

	clients.KubeClient.Kube.CoreV1().Pods(system.Namespace()).Delete(activatorPod, &metav1.DeleteOptions{
		GracePeriodSeconds: ptr.Int64(0),
	})

	// Wait for the killed activator to disappear from the knative service's endpoints.
	if err := waitForChangedPublicEndpoints(t, clients, resourcesScaleToZero.Revision.Name, origEndpoints); err != nil {
		t.Fatal("Failed to wait for the service to update its endpoints:", err)
	}

	assertServiceEventuallyWorks(t, clients, namesScaleToZero, resourcesScaleToZero.Service.Status.URL.URL(), test.PizzaPlanetText1)

	if err := waitForPodDeleted(t, clients, activatorPod); err != nil {
		t.Fatalf("Did not observe %s to actually be deleted: %v", activatorPod, err)
	}
	if err := waitForDeploymentScale(clients, activatorDeploymentName, haReplicas); err != nil {
		t.Fatalf("Deployment %s failed to scale up: %v", activatorDeploymentName, err)
	}

	pods, err = clients.KubeClient.Kube.CoreV1().Pods(system.Namespace()).List(metav1.ListOptions{
		LabelSelector: activatorLabel,
	})
	if err != nil {
		t.Fatal("Failed to get activator pods:", err)
	}

	// Sort the pods according to creation timestamp so that we can kill the oldest one. We want to
	// gradually kill both activator pods that were started at the beginning.
	sort.Slice(pods.Items, func(i, j int) bool { return pods.Items[i].CreationTimestamp.Before(&pods.Items[j].CreationTimestamp) })

	activatorPod = pods.Items[0].Name // Stop the oldest activator pod remaining.

	origEndpoints, err = getPublicEndpoints(t, clients, resourcesScaleToZero.Revision.Name)
	if err != nil {
		t.Fatalf("Unable to get public endpoints for revision %s: %v", resourcesScaleToZero.Revision.Name, err)
	}

	clients.KubeClient.Kube.CoreV1().Pods(system.Namespace()).Delete(activatorPod, &metav1.DeleteOptions{
		GracePeriodSeconds: ptr.Int64(0),
	})

	// Wait for the killed activator to disappear from the knative service's endpoints.
	if err := waitForChangedPublicEndpoints(t, clients, resourcesScaleToZero.Revision.Name, origEndpoints); err != nil {
		t.Fatal("Failed to wait for the service to update its endpoints:", err)
	}

	assertServiceEventuallyWorks(t, clients, namesScaleToZero, resourcesScaleToZero.Service.Status.URL.URL(), test.PizzaPlanetText1)
}

func assertSLO(t *testing.T, p test.Prober) {
	t.Helper()
	if err := p.Stop(); err != nil {
		t.Error("Failed to stop prober:", err)
	}
	if err := test.CheckSLO(SLO, t.Name(), p); err != nil {
		t.Error("CheckSLO failed:", err)
	}
}
<commit_msg>Use graceful shutdowns in the activator HA test. (#7963)<commit_after>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ha

import (
	"log"
	"sort"
	"testing"

	"knative.dev/pkg/ptr"
	"knative.dev/pkg/system"
	"knative.dev/pkg/test/logstream"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"knative.dev/serving/pkg/apis/autoscaling"
	revisionresourcenames "knative.dev/serving/pkg/reconciler/revision/resources/names"
	rtesting "knative.dev/serving/pkg/testing/v1"
	"knative.dev/serving/test"
	"knative.dev/serving/test/e2e"
)

const (
	activatorDeploymentName = "activator"
	activatorLabel          = "app=activator"
	minProbes               = 400 // We want to send at least 400 requests.
)

func TestActivatorHAGraceful(t *testing.T) {
	testActivatorHA(t, nil, 1)
}

func TestActivatorHANonGraceful(t *testing.T) {
	// For non-graceful tests, we want the pod to receive a SIGKILL straight away.
	testActivatorHA(t, ptr.Int64(0), 0.95)
}

// The Activator does not have leader election enabled.
// The test ensures that stopping one of the activator pods doesn't affect user applications.
// One service is probed during activator restarts and another service is used for testing
// that we can scale from zero after activator restart.
func testActivatorHA(t *testing.T, gracePeriod *int64, slo float64) {
	clients := e2e.Setup(t)
	cancel := logstream.Start(t)
	defer cancel()

	podDeleteOptions := &metav1.DeleteOptions{GracePeriodSeconds: gracePeriod}

	if err := waitForDeploymentScale(clients, activatorDeploymentName, haReplicas); err != nil {
		t.Fatalf("Deployment %s not scaled to %d: %v", activatorDeploymentName, haReplicas, err)
	}

	// Create first service that we will continually probe during activator restart.
	names, resources := createPizzaPlanetService(t,
		rtesting.WithConfigAnnotations(map[string]string{
			autoscaling.MinScaleAnnotationKey:  "1",  // Make sure we don't scale to zero during the test.
			autoscaling.TargetBurstCapacityKey: "-1", // Make sure all requests go through the activator.
		}),
	)
	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	// Create second service that will be scaled to zero and after stopping the activator we'll
	// ensure it can be scaled back from zero.
	namesScaleToZero, resourcesScaleToZero := createPizzaPlanetService(t,
		rtesting.WithConfigAnnotations(map[string]string{
			autoscaling.WindowAnnotationKey:    autoscaling.WindowMin.String(), // Make sure we scale to zero quickly.
			autoscaling.TargetBurstCapacityKey: "-1",                           // Make sure all requests go through the activator.
		}),
	)
	test.CleanupOnInterrupt(func() { test.TearDown(clients, namesScaleToZero) })
	defer test.TearDown(clients, namesScaleToZero)

	if err := e2e.WaitForScaleToZero(t, revisionresourcenames.Deployment(resourcesScaleToZero.Revision), clients); err != nil {
		t.Fatal("Failed to scale to zero:", err)
	}

	prober := test.NewProberManager(log.Printf, clients, minProbes)
	prober.Spawn(resources.Service.Status.URL.URL())
	defer assertSLO(t, prober, slo)

	pods, err := clients.KubeClient.Kube.CoreV1().Pods(system.Namespace()).List(metav1.ListOptions{
		LabelSelector: activatorLabel,
	})
	if err != nil {
		t.Fatal("Failed to get activator pods:", err)
	}
	activatorPod := pods.Items[0].Name

	origEndpoints, err := getPublicEndpoints(t, clients, resourcesScaleToZero.Revision.Name)
	if err != nil {
		t.Fatalf("Unable to get public endpoints for revision %s: %v", resourcesScaleToZero.Revision.Name, err)
	}

	clients.KubeClient.Kube.CoreV1().Pods(system.Namespace()).Delete(activatorPod, podDeleteOptions)

	// Wait for the killed activator to disappear from the knative service's endpoints.
	if err := waitForChangedPublicEndpoints(t, clients, resourcesScaleToZero.Revision.Name, origEndpoints); err != nil {
		t.Fatal("Failed to wait for the service to update its endpoints:", err)
	}

	assertServiceEventuallyWorks(t, clients, namesScaleToZero, resourcesScaleToZero.Service.Status.URL.URL(), test.PizzaPlanetText1)

	if err := waitForPodDeleted(t, clients, activatorPod); err != nil {
		t.Fatalf("Did not observe %s to actually be deleted: %v", activatorPod, err)
	}
	if err := waitForDeploymentScale(clients, activatorDeploymentName, haReplicas); err != nil {
		t.Fatalf("Deployment %s failed to scale up: %v", activatorDeploymentName, err)
	}

	pods, err = clients.KubeClient.Kube.CoreV1().Pods(system.Namespace()).List(metav1.ListOptions{
		LabelSelector: activatorLabel,
	})
	if err != nil {
		t.Fatal("Failed to get activator pods:", err)
	}

	// Sort the pods according to creation timestamp so that we can kill the oldest one. We want to
	// gradually kill both activator pods that were started at the beginning.
	sort.Slice(pods.Items, func(i, j int) bool { return pods.Items[i].CreationTimestamp.Before(&pods.Items[j].CreationTimestamp) })

	activatorPod = pods.Items[0].Name // Stop the oldest activator pod remaining.

	origEndpoints, err = getPublicEndpoints(t, clients, resourcesScaleToZero.Revision.Name)
	if err != nil {
		t.Fatalf("Unable to get public endpoints for revision %s: %v", resourcesScaleToZero.Revision.Name, err)
	}

	clients.KubeClient.Kube.CoreV1().Pods(system.Namespace()).Delete(activatorPod, podDeleteOptions)

	// Wait for the killed activator to disappear from the knative service's endpoints.
	if err := waitForChangedPublicEndpoints(t, clients, resourcesScaleToZero.Revision.Name, origEndpoints); err != nil {
		t.Fatal("Failed to wait for the service to update its endpoints:", err)
	}

	assertServiceEventuallyWorks(t, clients, namesScaleToZero, resourcesScaleToZero.Service.Status.URL.URL(), test.PizzaPlanetText1)
}

func assertSLO(t *testing.T, p test.Prober, slo float64) {
	t.Helper()
	if err := p.Stop(); err != nil {
		t.Error("Failed to stop prober:", err)
	}
	if err := test.CheckSLO(slo, t.Name(), p); err != nil {
		t.Error("CheckSLO failed:", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author: polaris	polaris@studygolang.com

package controller

import (
	"errors"
	"http/middleware"
	"logic"
	"model"
	"net/http"
	"strconv"

	. "http"

	"github.com/labstack/echo"
	"github.com/polaris1119/echoutils"
	"github.com/polaris1119/goutils"
	"github.com/polaris1119/slices"
)

type CommentController struct{}

func (self CommentController) RegisterRoute(g *echo.Group) {
	g.Get("/at/users", self.AtUsers)
	g.Post("/comment/:objid", self.Create, middleware.NeedLogin(), middleware.Sensivite(), middleware.BalanceCheck(), middleware.PublishNotice())
	g.Get("/object/comments", self.CommentList)
	g.Post("/object/comments/:cid", self.Modify, middleware.NeedLogin(), middleware.Sensivite(), middleware.PublishNotice())

	g.Get("/topics/:objid/comment/:cid", self.TopicDetail)
	g.Get("/articles/:objid/comment/:cid", self.ArticleDetail)
}

// AtUsers 评论或回复 @ 某人 suggest
func (CommentController) AtUsers(ctx echo.Context) error {
	term := ctx.QueryParam("term")
	isHttps := CheckIsHttps(ctx)
	users := logic.DefaultUser.GetUserMentions(term, 10, isHttps)
	return ctx.JSON(http.StatusOK, users)
}

// Create 评论（或回复）
func (CommentController) Create(ctx echo.Context) error {
	user := ctx.Get("user").(*model.Me)

	// 入库
	objid := goutils.MustInt(ctx.Param("objid"))
	if objid == 0 {
		return fail(ctx, 1, "参数有误，请刷新后重试！")
	}
	comment, err := logic.DefaultComment.Publish(ctx, user.Uid, objid, ctx.FormParams())
	if err != nil {
		return fail(ctx, 2, "服务器内部错误")
	}

	return success(ctx, comment)
}

// 修改评论
func (CommentController) Modify(ctx echo.Context) error {
	cid := goutils.MustInt(ctx.Param("cid"))
	content := ctx.FormValue("content")
	comment, err := logic.DefaultComment.FindById(cid)

	if err != nil {
		return fail(ctx, 2, "评论不存在")
	}

	if content == "" {
		return fail(ctx, 1, "内容不能为空")
	}

	me := ctx.Get("user").(*model.Me)
	// CanEdit 已包含修改时间限制
	if !logic.CanEdit(me, comment) {
		return fail(ctx, 3, "没有修改权限")
	}

	errMsg, err := logic.DefaultComment.Modify(echoutils.WrapEchoContext(ctx), cid, content)
	if err != nil {
		return fail(ctx, 4, errMsg)
	}

	return success(ctx, map[string]interface{}{"cid": cid})
}

// CommentList 获取某对象的评论信息
func (CommentController) CommentList(ctx echo.Context) error {
	objid := goutils.MustInt(ctx.QueryParam("objid"))
	objtype := goutils.MustInt(ctx.QueryParam("objtype"))

	commentList, err := logic.DefaultComment.FindObjectComments(ctx, objid, objtype)
	if err != nil {
		return fail(ctx, 1, "服务器内部错误")
	}

	uids := slices.StructsIntSlice(commentList, "Uid")
	users := logic.DefaultUser.FindUserInfos(ctx, uids)

	result := map[string]interface{}{
		"comments": commentList,
	}

	// json encode 不支持 map[int]...
	for uid, user := range users {
		result[strconv.Itoa(uid)] = user
	}

	return success(ctx, result)
}

func (self CommentController) TopicDetail(ctx echo.Context) error {
	objid := goutils.MustInt(ctx.Param("objid"))
	cid := goutils.MustInt(ctx.Param("cid"))

	topicMaps := logic.DefaultTopic.FindFullinfoByTids([]int{objid})
	if len(topicMaps) < 1 {
		return ctx.Redirect(http.StatusSeeOther, "/topics")
	}

	topic := topicMaps[0]
	topic["node"] = logic.GetNode(topic["nid"].(int))

	data := map[string]interface{}{
		"topic": topic,
	}
	data["appends"] = logic.DefaultTopic.FindAppend(ctx, objid)

	err := self.fillCommentAndUser(ctx, data, cid, objid, model.TypeTopic)

	if err != nil {
		return ctx.Redirect(http.StatusSeeOther, "/topics/"+strconv.Itoa(objid))
	}

	return render(ctx, "topics/comment.html", data)
}

func (self CommentController) ArticleDetail(ctx echo.Context) error {
	objid := goutils.MustInt(ctx.Param("objid"))
	cid := goutils.MustInt(ctx.Param("cid"))

	article, err := logic.DefaultArticle.FindById(ctx, objid)
	if err != nil {
		return ctx.Redirect(http.StatusSeeOther, "/articles")
	}
	articleGCTT := logic.DefaultArticle.FindArticleGCTT(ctx, article)

	data := map[string]interface{}{
		"article":      article,
		"article_gctt": articleGCTT,
	}

	err = self.fillCommentAndUser(ctx, data, cid, objid, model.TypeArticle)

	if err != nil {
		return ctx.Redirect(http.StatusSeeOther, "/articles/"+strconv.Itoa(objid))
	}

	return render(ctx, "articles/comment.html", data)
}

func (CommentController) fillCommentAndUser(ctx echo.Context, data map[string]interface{}, cid, objid, objtype int) error {
	comment, comments := logic.DefaultComment.FindComment(ctx, cid, objid, objtype)

	if comment.Cid == 0 {
		return errors.New("comment not exists!")
	}

	uids := make([]int, 1+len(comments))
	uids[0] = comment.Uid
	for i, comment := range comments {
		uids[i+1] = comment.Uid
	}
	users := logic.DefaultUser.FindUserInfos(ctx, uids)

	data["comment"] = comment
	data["comments"] = comments
	data["users"] = users

	return nil
}
<commit_msg>修改评论不通知<commit_after>// Copyright 2016 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author: polaris	polaris@studygolang.com

package controller

import (
	"errors"
	"http/middleware"
	"logic"
	"model"
	"net/http"
	"strconv"

	. "http"

	"github.com/labstack/echo"
	"github.com/polaris1119/echoutils"
	"github.com/polaris1119/goutils"
	"github.com/polaris1119/slices"
)

type CommentController struct{}

func (self CommentController) RegisterRoute(g *echo.Group) {
	g.Get("/at/users", self.AtUsers)
	g.Post("/comment/:objid", self.Create, middleware.NeedLogin(), middleware.Sensivite(), middleware.BalanceCheck(), middleware.PublishNotice())
	g.Get("/object/comments", self.CommentList)
	g.Post("/object/comments/:cid", self.Modify, middleware.NeedLogin(), middleware.Sensivite())

	g.Get("/topics/:objid/comment/:cid", self.TopicDetail)
	g.Get("/articles/:objid/comment/:cid", self.ArticleDetail)
}

// AtUsers 评论或回复 @ 某人 suggest
func (CommentController) AtUsers(ctx echo.Context) error {
	term := ctx.QueryParam("term")
	isHttps := CheckIsHttps(ctx)
	users := logic.DefaultUser.GetUserMentions(term, 10, isHttps)
	return ctx.JSON(http.StatusOK, users)
}

// Create 评论（或回复）
func (CommentController) Create(ctx echo.Context) error {
	user := ctx.Get("user").(*model.Me)

	// 入库
	objid := goutils.MustInt(ctx.Param("objid"))
	if objid == 0 {
		return fail(ctx, 1, "参数有误，请刷新后重试！")
	}
	comment, err := logic.DefaultComment.Publish(ctx, user.Uid, objid, ctx.FormParams())
	if err != nil {
		return fail(ctx, 2, "服务器内部错误")
	}

	return success(ctx, comment)
}

// 修改评论
func (CommentController) Modify(ctx echo.Context) error {
	cid := goutils.MustInt(ctx.Param("cid"))
	content := ctx.FormValue("content")
	comment, err := logic.DefaultComment.FindById(cid)

	if err != nil {
		return fail(ctx, 2, "评论不存在")
	}

	if content == "" {
		return fail(ctx, 1, "内容不能为空")
	}

	me := ctx.Get("user").(*model.Me)
	// CanEdit 已包含修改时间限制
	if !logic.CanEdit(me, comment) {
		return fail(ctx, 3, "没有修改权限")
	}

	errMsg, err := logic.DefaultComment.Modify(echoutils.WrapEchoContext(ctx), cid, content)
	if err != nil {
		return fail(ctx, 4, errMsg)
	}

	return success(ctx, map[string]interface{}{"cid": cid})
}

// CommentList 获取某对象的评论信息
func (CommentController) CommentList(ctx echo.Context) error {
	objid := goutils.MustInt(ctx.QueryParam("objid"))
	objtype := goutils.MustInt(ctx.QueryParam("objtype"))

	commentList, err := logic.DefaultComment.FindObjectComments(ctx, objid, objtype)
	if err != nil {
		return fail(ctx, 1, "服务器内部错误")
	}

	uids := slices.StructsIntSlice(commentList, "Uid")
	users := logic.DefaultUser.FindUserInfos(ctx, uids)

	result := map[string]interface{}{
		"comments": commentList,
	}

	// json encode 不支持 map[int]...
	for uid, user := range users {
		result[strconv.Itoa(uid)] = user
	}

	return success(ctx, result)
}

func (self CommentController) TopicDetail(ctx echo.Context) error {
	objid := goutils.MustInt(ctx.Param("objid"))
	cid := goutils.MustInt(ctx.Param("cid"))

	topicMaps := logic.DefaultTopic.FindFullinfoByTids([]int{objid})
	if len(topicMaps) < 1 {
		return ctx.Redirect(http.StatusSeeOther, "/topics")
	}

	topic := topicMaps[0]
	topic["node"] = logic.GetNode(topic["nid"].(int))

	data := map[string]interface{}{
		"topic": topic,
	}
	data["appends"] = logic.DefaultTopic.FindAppend(ctx, objid)

	err := self.fillCommentAndUser(ctx, data, cid, objid, model.TypeTopic)

	if err != nil {
		return ctx.Redirect(http.StatusSeeOther, "/topics/"+strconv.Itoa(objid))
	}

	return render(ctx, "topics/comment.html", data)
}

func (self CommentController) ArticleDetail(ctx echo.Context) error {
	objid := goutils.MustInt(ctx.Param("objid"))
	cid := goutils.MustInt(ctx.Param("cid"))

	article, err := logic.DefaultArticle.FindById(ctx, objid)
	if err != nil {
		return ctx.Redirect(http.StatusSeeOther, "/articles")
	}
	articleGCTT := logic.DefaultArticle.FindArticleGCTT(ctx, article)

	data := map[string]interface{}{
		"article":      article,
		"article_gctt": articleGCTT,
	}

	err = self.fillCommentAndUser(ctx, data, cid, objid, model.TypeArticle)

	if err != nil {
		return ctx.Redirect(http.StatusSeeOther, "/articles/"+strconv.Itoa(objid))
	}

	return render(ctx, "articles/comment.html", data)
}

func (CommentController) fillCommentAndUser(ctx echo.Context, data map[string]interface{}, cid, objid, objtype int) error {
	comment, comments := logic.DefaultComment.FindComment(ctx, cid, objid, objtype)

	if comment.Cid == 0 {
		return errors.New("comment not exists!")
	}

	uids := make([]int, 1+len(comments))
	uids[0] = comment.Uid
	for i, comment := range comments {
		uids[i+1] = comment.Uid
	}
	users := logic.DefaultUser.FindUserInfos(ctx, uids)

	data["comment"] = comment
	data["comments"] = comments
	data["users"] = users

	return nil
}
<|endoftext|>


<commit_before>package qshell

import (
	"bufio"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/qiniu/api.v6/auth/digest"
	"github.com/qiniu/log"
	"github.com/qiniu/rpc"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

/*
{
	"dest_dir"		:	"/Users/jemy/Backup",
	"bucket"		:	"test-bucket",
	"domain"		:	"<Your bucket domain>",
	"access_key"	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"is_private"	:	false,
	"prefix"		:	"demo/"
}
*/

const (
	MIN_DOWNLOAD_THREAD_COUNT = 1
	MAX_DOWNLOAD_THREAD_COUNT = 100
)

type DownloadConfig struct {
	DestDir   string `json:"dest_dir"`
	Bucket    string `json:"bucket"`
	Domain    string `json:"domain"`
	AccessKey string `json:"access_key"`
	SecretKey string `json:"secret_key"`
	IsPrivate bool   `json:"is_private"`
	Prefix    string `json:"prefix,omitempty"`
}

func QiniuDownload(threadCount int, downloadConfigFile string) {
	cnfFp, err := os.Open(downloadConfigFile)
	if err != nil {
		log.Error("Open download config file", downloadConfigFile, "failed,", err)
		return
	}
	defer cnfFp.Close()
	cnfData, err := ioutil.ReadAll(cnfFp)
	if err != nil {
		log.Error("Read download config file error", err)
		return
	}
	downConfig := DownloadConfig{}
	cnfErr := json.Unmarshal(cnfData, &downConfig)
	if cnfErr != nil {
		log.Error("Parse download config error", err)
		return
	}
	cnfJson, _ := json.Marshal(&downConfig)
	jobId := fmt.Sprintf("%x", md5.Sum(cnfJson))
	jobListName := fmt.Sprintf("%s.list.txt", jobId)
	acct := Account{
		AccessKey: downConfig.AccessKey,
		SecretKey: downConfig.SecretKey,
	}
	bLister := ListBucket{
		Account: acct,
	}
	log.Debug("List bucket...")
	listErr := bLister.List(downConfig.Bucket, downConfig.Prefix, jobListName)
	if listErr != nil {
		log.Error("List bucket error", listErr)
		return
	}
	listFp, openErr := os.Open(jobListName)
	if openErr != nil {
		log.Error("Open list file error", openErr)
		return
	}
	defer listFp.Close()
	listScanner := bufio.NewScanner(listFp)
	listScanner.Split(bufio.ScanLines)
	downWorkGroup := sync.WaitGroup{}
	downCounter := 0

	threadThresold := threadCount + 1
	for listScanner.Scan() {
		downCounter += 1
		if downCounter%threadThresold == 0 {
			downWorkGroup.Wait()
		}
		line := strings.TrimSpace(listScanner.Text())
		items := strings.Split(line, "\t")
		if len(items) > 2 {
			fileKey := items[0]
			fileSize, _ := strconv.ParseInt(items[1], 10, 64)
			//not backup yet
			if !checkLocalDuplicate(downConfig.DestDir, fileKey, fileSize) {
				downWorkGroup.Add(1)
				go func() {
					defer downWorkGroup.Done()
					downloadFile(downConfig, fileKey)
				}()
			}
		}
	}
	downWorkGroup.Wait()
	fmt.Println("All downloaded!")
}

func checkLocalDuplicate(destDir string, fileKey string, fileSize int64) bool {
	dup := false
	filePath := filepath.Join(destDir, fileKey)
	fStat, statErr := os.Stat(filePath)
	if statErr == nil {
		//exist, check file size
		localFileSize := fStat.Size()
		if localFileSize == fileSize {
			dup = true
		}
	}
	return dup
}

func downloadFile(downConfig DownloadConfig, fileKey string) {
	localFilePath := filepath.Join(downConfig.DestDir, fileKey)
	ldx := strings.LastIndex(localFilePath, string(os.PathSeparator))
	if ldx != -1 {
		localFileDir := localFilePath[:ldx]
		err := os.MkdirAll(localFileDir, 0775)
		if err != nil {
			log.Error("MkdirAll failed for", localFileDir)
			return
		}
	}
	fmt.Println("Downloading", fileKey, "=>", localFilePath, "...")
	downUrl := strings.Join([]string{downConfig.Domain, fileKey}, "/")
	if downConfig.IsPrivate {
		now := time.Now().Add(time.Second * 3600 * 24)
		downUrl = fmt.Sprintf("%s?e=%d", downUrl, now.Unix())
		mac := digest.Mac{downConfig.AccessKey, []byte(downConfig.SecretKey)}
		token := digest.Sign(&mac, []byte(downUrl))
		downUrl = fmt.Sprintf("%s&token=%s", downUrl, token)
	}
	resp, respErr := rpc.DefaultClient.Get(nil, downUrl)
	if respErr != nil {
		log.Error("Download", fileKey, "failed by url", downUrl)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode == 200 {
		localFp, openErr := os.OpenFile(localFilePath, os.O_CREATE|os.O_WRONLY, 0666)
		if openErr != nil {
			log.Error("Open local file", localFilePath, "failed")
			return
		}
		defer localFp.Close()
		_, err := io.Copy(localFp, resp.Body)
		if err != nil {
			log.Error("Download", fileKey, "failed", err)
		}
	} else {
		log.Error("Download", fileKey, "failed by url", downUrl)
	}
}
<commit_msg>Add suffix for qdownload to filter the unwanted files.<commit_after>package qshell

import (
	"bufio"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/qiniu/api.v6/auth/digest"
	"github.com/qiniu/log"
	"github.com/qiniu/rpc"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

/*
{
	"dest_dir"		:	"/Users/jemy/Backup",
	"bucket"		:	"test-bucket",
	"domain"		:	"<Your bucket domain>",
	"access_key"	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"is_private"	:	false,
	"prefix"		:	"demo/",
	"suffix"		: ".mp4"
}
*/

const (
	MIN_DOWNLOAD_THREAD_COUNT = 1
	MAX_DOWNLOAD_THREAD_COUNT = 100
)

type DownloadConfig struct {
	DestDir   string `json:"dest_dir"`
	Bucket    string `json:"bucket"`
	Domain    string `json:"domain"`
	AccessKey string `json:"access_key"`
	SecretKey string `json:"secret_key"`
	IsPrivate bool   `json:"is_private"`
	Prefix    string `json:"prefix,omitempty"`
	Suffix    string `json:"suffix,omitempty"`
}

func QiniuDownload(threadCount int, downloadConfigFile string) {
	cnfFp, err := os.Open(downloadConfigFile)
	if err != nil {
		log.Error("Open download config file", downloadConfigFile, "failed,", err)
		return
	}
	defer cnfFp.Close()
	cnfData, err := ioutil.ReadAll(cnfFp)
	if err != nil {
		log.Error("Read download config file error", err)
		return
	}
	downConfig := DownloadConfig{}
	cnfErr := json.Unmarshal(cnfData, &downConfig)
	if cnfErr != nil {
		log.Error("Parse download config error", err)
		return
	}
	cnfJson, _ := json.Marshal(&downConfig)
	jobId := fmt.Sprintf("%x", md5.Sum(cnfJson))
	jobListName := fmt.Sprintf("%s.list.txt", jobId)
	acct := Account{
		AccessKey: downConfig.AccessKey,
		SecretKey: downConfig.SecretKey,
	}
	bLister := ListBucket{
		Account: acct,
	}
	log.Debug("List bucket...")
	listErr := bLister.List(downConfig.Bucket, downConfig.Prefix, jobListName)
	if listErr != nil {
		log.Error("List bucket error", listErr)
		return
	}
	listFp, openErr := os.Open(jobListName)
	if openErr != nil {
		log.Error("Open list file error", openErr)
		return
	}
	defer listFp.Close()
	listScanner := bufio.NewScanner(listFp)
	listScanner.Split(bufio.ScanLines)
	downWorkGroup := sync.WaitGroup{}
	downCounter := 0

	threadThresold := threadCount + 1
	for listScanner.Scan() {
		downCounter += 1
		if downCounter%threadThresold == 0 {
			downWorkGroup.Wait()
		}
		line := strings.TrimSpace(listScanner.Text())
		items := strings.Split(line, "\t")
		if len(items) > 2 {
			fileKey := items[0]
			//check suffix
			if downConfig.Suffix != "" && !strings.HasSuffix(fileKey, downConfig.Suffix) {
				continue
			}
			fileSize, _ := strconv.ParseInt(items[1], 10, 64)
			//not backup yet
			if !checkLocalDuplicate(downConfig.DestDir, fileKey, fileSize) {
				downWorkGroup.Add(1)
				go func() {
					defer downWorkGroup.Done()
					downloadFile(downConfig, fileKey)
				}()
			}
		}
	}
	downWorkGroup.Wait()
	fmt.Println("All downloaded!")
}

func checkLocalDuplicate(destDir string, fileKey string, fileSize int64) bool {
	dup := false
	filePath := filepath.Join(destDir, fileKey)
	fStat, statErr := os.Stat(filePath)
	if statErr == nil {
		//exist, check file size
		localFileSize := fStat.Size()
		if localFileSize == fileSize {
			dup = true
		}
	}
	return dup
}

func downloadFile(downConfig DownloadConfig, fileKey string) {
	localFilePath := filepath.Join(downConfig.DestDir, fileKey)
	ldx := strings.LastIndex(localFilePath, string(os.PathSeparator))
	if ldx != -1 {
		localFileDir := localFilePath[:ldx]
		err := os.MkdirAll(localFileDir, 0775)
		if err != nil {
			log.Error("MkdirAll failed for", localFileDir)
			return
		}
	}
	fmt.Println("Downloading", fileKey, "=>", localFilePath, "...")
	downUrl := strings.Join([]string{downConfig.Domain, fileKey}, "/")
	if downConfig.IsPrivate {
		now := time.Now().Add(time.Second * 3600 * 24)
		downUrl = fmt.Sprintf("%s?e=%d", downUrl, now.Unix())
		mac := digest.Mac{downConfig.AccessKey, []byte(downConfig.SecretKey)}
		token := digest.Sign(&mac, []byte(downUrl))
		downUrl = fmt.Sprintf("%s&token=%s", downUrl, token)
	}
	resp, respErr := rpc.DefaultClient.Get(nil, downUrl)
	if respErr != nil {
		log.Error("Download", fileKey, "failed by url", downUrl)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode == 200 {
		localFp, openErr := os.OpenFile(localFilePath, os.O_CREATE|os.O_WRONLY, 0666)
		if openErr != nil {
			log.Error("Open local file", localFilePath, "failed")
			return
		}
		defer localFp.Close()
		_, err := io.Copy(localFp, resp.Body)
		if err != nil {
			log.Error("Download", fileKey, "failed", err)
		}
	} else {
		log.Error("Download", fileKey, "failed by url", downUrl)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"compress/gzip"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/appc/spec/aci"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
)

type StringVector []string

func (v *StringVector) String() string {
	return `"` + strings.Join(*v, `" "`) + `"`
}

func (v *StringVector) Set(str string) error {
	*v = append(*v, str)
	return nil
}

type options struct {
	exec      StringVector
	goBinary  string
	goPath    string
	useBinary string
	assets    StringVector
	keepTmp   bool
	project   string
}

func getOptions() (*options, error) {
	opts := &options{}

	// --go-binary
	goDefaultBinaryDesc := "Go binary to use"
	gocmd, err := exec.LookPath("go")
	if err != nil {
		goDefaultBinaryDesc += " (default: none found in $PATH, so it must be provided)"
	} else {
		goDefaultBinaryDesc += " (default: whatever go in $PATH)"
	}
	flag.StringVar(&opts.goBinary, "go-binary", gocmd, goDefaultBinaryDesc)

	// --exec
	flag.Var(&opts.exec, "exec", "Parameters passed to app, can be used multiple times")

	// --go-path
	flag.StringVar(&opts.goPath, "go-path", "", "Custom GOPATH (default: a temporary directory)")

	// --use-binary
	flag.StringVar(&opts.useBinary, "use-binary", "", "Which executable to put in ACI image")

	// --asset
	flag.Var(&opts.assets, "asset", "Additional assets, can be used multiple times; format: <path in ACI>"+ListSeparator()+"<local path>; placeholders like <GOPATH> and <PROJPATH> can be used there as well")

	// --keep-tmp
	flag.BoolVar(&opts.keepTmp, "keep-tmp", false, "Do not delete temporary directory used for creating ACI")
	flag.Parse()

	args := flag.Args()
	if len(args) != 1 {
		return nil, fmt.Errorf("Expected exactly one project to build, got %d", len(args))
	}
	opts.project = args[0]
	if opts.goBinary == "" {
		return nil, fmt.Errorf("Go binary not found")
	}

	return opts, nil
}

type pathsAndNames struct {
	tmpDirPath  string
	goPath      string
	goRootPath  string
	projectPath string
	fakeGoPath  string
	goBinPath   string
	aciDirPath  string
	rootFSPath  string
	goExecPath  string

	imageFileName string
	imageACName   string
}

// getGoPath returns go path and fake go path. The former is either in
// /tmp (which is a default) or some other path as specified by
// --go-path parameter. The latter is always in /tmp.
func getGoPath(opts *options, tmpDir string) (string, string) {
	fakeGoPath := filepath.Join(tmpDir, "gopath")
	if opts.goPath == "" {
		return fakeGoPath, fakeGoPath
	}
	return opts.goPath, fakeGoPath
}

// getNamesFromProject returns project name, image ACName and ACI
// filename. Names depend on whether project has several binaries. For
// project with single binary (github.com/appc/goaci) returned values
// would be: github.com/appc/goaci, github.com/appc/goaci and
// goaci.aci. For project with multiple binaries
// (github.com/appc/spec/...) returned values would be (assuming ace
// as selected binary): github.com/appc/spec, github.com/appc/spec-ace
// and spec-ace.aci.
func getNamesFromProject(opts *options) (string, string, string) {
	imageACName := opts.project
	projectName := imageACName
	base := filepath.Base(imageACName)
	threeDotsBase := base == "..."
	if threeDotsBase {
		imageACName = filepath.Dir(imageACName)
		projectName = imageACName
		base = filepath.Base(imageACName)
		if opts.useBinary != "" {
			suffix := "-" + opts.useBinary
			base += suffix
			imageACName += suffix
		}
	}

	return projectName, imageACName, base + schema.ACIExtension
}

func getPathsAndNames(opts *options) (*pathsAndNames, error) {
	tmpDir, err := ioutil.TempDir("", "goaci")
	if err != nil {
		return nil, fmt.Errorf("error setting up temporary directory: %v", err)
	}

	goPath, fakeGoPath := getGoPath(opts, tmpDir)
	projectName, imageACName, imageFileName := getNamesFromProject(opts)

	if os.Getenv("GOPATH") != "" {
		Warn("GOPATH env var is ignored, use --go-path=\"$GOPATH\" option instead")
	}
	goRoot := os.Getenv("GOROOT")
	if goRoot != "" {
		Warn("Overriding GOROOT env var to ", goRoot)
	}

	aciDir := filepath.Join(tmpDir, "aci")
	// Project name is path-like string with slashes, but slash is
	// not a file separator on every OS.
	projectPath := filepath.Join(goPath, "src", filepath.Join(strings.Split(projectName, "/")...))

	return &pathsAndNames{
		tmpDirPath:  tmpDir, // /tmp/XXX
		goPath:      goPath,
		goRootPath:  goRoot,
		projectPath: projectPath,
		fakeGoPath:  fakeGoPath,                       // /tmp/XXX/gopath
		goBinPath:   filepath.Join(fakeGoPath, "bin"), // /tmp/XXX/gopath/bin
		aciDirPath:  aciDir,                           // /tmp/XXX/aci
		rootFSPath:  filepath.Join(aciDir, "rootfs"),  // /tmp/XXX/aci/rootfs
		goExecPath:  opts.goBinary,

		imageFileName: imageFileName,
		imageACName:   imageACName,
	}, nil
}

func makeDirectories(pathsNames *pathsAndNames) error {
	// /tmp/XXX already exists, not creating it here

	// /tmp/XXX/gopath
	if err := os.Mkdir(pathsNames.fakeGoPath, 0755); err != nil {
		return err
	}

	// /tmp/XXX/gopath/bin
	if err := os.Mkdir(pathsNames.goBinPath, 0755); err != nil {
		return err
	}

	// /tmp/XXX/aci
	if err := os.Mkdir(pathsNames.aciDirPath, 0755); err != nil {
		return err
	}

	// /tmp/XXX/aci/rootfs
	if err := os.Mkdir(pathsNames.rootFSPath, 0755); err != nil {
		return err
	}

	return nil
}

func runGoGet(opts *options, pathsNames *pathsAndNames) error {
	// Construct args for a go get that does a static build
	args := []string{
		pathsNames.goExecPath,
		"get",
		"-a",
		"-tags", "netgo",
		"-ldflags", "'-w'",
		"-installsuffix", "nocgo", // for 1.4
		opts.project,
	}

	env := []string{
		"GOPATH=" + pathsNames.goPath,
		"GOBIN=" + pathsNames.goBinPath,
		"CGO_ENABLED=0",
		"PATH=" + os.Getenv("PATH"),
	}
	if pathsNames.goRootPath != "" {
		env = append(env, "GOROOT="+pathsNames.goRootPath)
	}

	cmd := exec.Cmd{
		Env:    env,
		Path:   pathsNames.goExecPath,
		Args:   args,
		Stderr: os.Stderr,
		Stdout: os.Stdout,
	}
	Debug("env: ", cmd.Env)
	Debug("running command: ", strings.Join(cmd.Args, " "))
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

// getBinaryName get a binary name built by go get and selected by
// --use-binary parameter.
func getBinaryName(opts *options, pathsNames *pathsAndNames) (string, error) {
	fi, err := ioutil.ReadDir(pathsNames.goBinPath)
	if err != nil {
		return "", err
	}

	switch {
	case len(fi) < 1:
		return "", fmt.Errorf("No binaries found in gobin.")
	case len(fi) == 1:
		name := fi[0].Name()
		if opts.useBinary != "" && name != opts.useBinary {
			return "", fmt.Errorf("No such binary found in gobin: %q. There is only %q", opts.useBinary, name)
		}
		Debug("found binary: ", name)
		return name, nil
	case len(fi) > 1:
		names := []string{}
		for _, v := range fi {
			names = append(names, v.Name())
		}
		if opts.useBinary == "" {
			return "", fmt.Errorf("Found multiple binaries in gobin, but --use-binary option is not used. Please specify which binary to put in ACI. Following binaries are available: \"%s\"", strings.Join(names, `", "`))
		}
		for _, v := range names {
			if v == opts.useBinary {
				return v, nil
			}
		}
		return "", fmt.Errorf("No such binary found in gobin: %q. There are following binaries available: \"%s\"", opts.useBinary, strings.Join(names, `", "`))
	}
	return "", fmt.Errorf("Reaching this point shouldn't be possible.")
}

func getApp(opts *options, binary string) *types.App {
	exec := []string{filepath.Join("/", binary)}
	exec = append(exec, opts.exec...)

	return &types.App{
		Exec:  exec,
		User:  "0",
		Group: "0",
	}
}

func getVCSLabel(pathsNames *pathsAndNames) (*types.Label, error) {
	name, value, err := GetVCSInfo(pathsNames.projectPath)
	if err != nil {
		return nil, fmt.Errorf("Failed to get VCS info: %v", err)
	}
	acname, err := types.NewACName(name)
	if err != nil {
		return nil, fmt.Errorf("Invalid VCS label: %v", err)
	}
	return &types.Label{
		Name:  *acname,
		Value: value,
	}, nil
}

func prepareManifest(opts *options, pathsNames *pathsAndNames, binary string) (*schema.ImageManifest, error) {
	name, err := types.NewACName(pathsNames.imageACName)
	// TODO(jonboulle): could this ever actually happen?
	if err != nil {
		return nil, err
	}

	app := getApp(opts, binary)

	vcsLabel, err := getVCSLabel(pathsNames)
	if err != nil {
		return nil, err
	}
	labels := types.Labels{
		*vcsLabel,
	}

	manifest := schema.BlankImageManifest()
	manifest.Name = *name
	manifest.App = app
	manifest.Labels = labels
	return manifest, nil
}

func copyAssets(opts *options, pathsNames *pathsAndNames) error {
	placeholderMapping := map[string]string{
		"<PROJPATH>": pathsNames.projectPath,
		"<GOPATH>":   pathsNames.goPath,
	}
	if err := PrepareAssets(opts.assets, pathsNames.rootFSPath, placeholderMapping); err != nil {
		return err
	}
	return nil
}

func moveBinaryToRootFS(pathsNames *pathsAndNames, binary string) error {
	// Move the binary into the rootfs
	ep := filepath.Join(pathsNames.rootFSPath, binary)
	if err := os.Rename(filepath.Join(pathsNames.goBinPath, binary), ep); err != nil {
		return err
	}
	Debug("moved binary to: ", ep)
	return nil
}

func writeACI(pathsNames *pathsAndNames, manifest *schema.ImageManifest) error {
	mode := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
	of, err := os.OpenFile(pathsNames.imageFileName, mode, 0644)
	if err != nil {
		return fmt.Errorf("Error opening output file: %v", err)
	}
	defer of.Close()

	gw := gzip.NewWriter(of)
	defer gw.Close()

	tr := tar.NewWriter(gw)
	defer tr.Close()

	// FIXME: the files in the tar archive are added with the
	// wrong uid/gid. The uid/gid of the aci builder leaks in the
	// tar archive. See: #16
	iw := aci.NewImageWriter(*manifest, tr)
	if err := filepath.Walk(pathsNames.aciDirPath, aci.BuildWalker(pathsNames.aciDirPath, iw)); err != nil {
		return err
	}
	if err := iw.Close(); err != nil {
		return err
	}
	Info("Wrote ", of.Name())
	return nil
}

func mainWithError() error {
	InitDebug()

	opts, err := getOptions()
	if err != nil {
		return err
	}

	pathsNames, err := getPathsAndNames(opts)
	if err != nil {
		return err
	}

	if opts.keepTmp {
		Info(`Preserving temporary directory "`, pathsNames.tmpDirPath, `"`)
	} else {
		defer os.RemoveAll(pathsNames.tmpDirPath)
	}

	if err := makeDirectories(pathsNames); err != nil {
		return err
	}

	if err := runGoGet(opts, pathsNames); err != nil {
		return err
	}

	binary, err := getBinaryName(opts, pathsNames)
	if err != nil {
		return err
	}

	manifest, err := prepareManifest(opts, pathsNames, binary)
	if err != nil {
		return err
	}

	if err := copyAssets(opts, pathsNames); err != nil {
		return err
	}

	if err := moveBinaryToRootFS(pathsNames, binary); err != nil {
		return err
	}

	if err := writeACI(pathsNames, manifest); err != nil {
		return err
	}

	return nil
}

func main() {
	if err := mainWithError(); err != nil {
		Warn(err)
		os.Exit(1)
	}
}
<commit_msg>Use %q instead of "%s"<commit_after>package main

import (
	"archive/tar"
	"compress/gzip"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/appc/spec/aci"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
)

type StringVector []string

func (v *StringVector) String() string {
	return `"` + strings.Join(*v, `" "`) + `"`
}

func (v *StringVector) Set(str string) error {
	*v = append(*v, str)
	return nil
}

type options struct {
	exec      StringVector
	goBinary  string
	goPath    string
	useBinary string
	assets    StringVector
	keepTmp   bool
	project   string
}

func getOptions() (*options, error) {
	opts := &options{}

	// --go-binary
	goDefaultBinaryDesc := "Go binary to use"
	gocmd, err := exec.LookPath("go")
	if err != nil {
		goDefaultBinaryDesc += " (default: none found in $PATH, so it must be provided)"
	} else {
		goDefaultBinaryDesc += " (default: whatever go in $PATH)"
	}
	flag.StringVar(&opts.goBinary, "go-binary", gocmd, goDefaultBinaryDesc)

	// --exec
	flag.Var(&opts.exec, "exec", "Parameters passed to app, can be used multiple times")

	// --go-path
	flag.StringVar(&opts.goPath, "go-path", "", "Custom GOPATH (default: a temporary directory)")

	// --use-binary
	flag.StringVar(&opts.useBinary, "use-binary", "", "Which executable to put in ACI image")

	// --asset
	flag.Var(&opts.assets, "asset", "Additional assets, can be used multiple times; format: <path in ACI>"+ListSeparator()+"<local path>; placeholders like <GOPATH> and <PROJPATH> can be used there as well")

	// --keep-tmp
	flag.BoolVar(&opts.keepTmp, "keep-tmp", false, "Do not delete temporary directory used for creating ACI")
	flag.Parse()

	args := flag.Args()
	if len(args) != 1 {
		return nil, fmt.Errorf("Expected exactly one project to build, got %d", len(args))
	}
	opts.project = args[0]
	if opts.goBinary == "" {
		return nil, fmt.Errorf("Go binary not found")
	}

	return opts, nil
}

type pathsAndNames struct {
	tmpDirPath  string
	goPath      string
	goRootPath  string
	projectPath string
	fakeGoPath  string
	goBinPath   string
	aciDirPath  string
	rootFSPath  string
	goExecPath  string

	imageFileName string
	imageACName   string
}

// getGoPath returns go path and fake go path. The former is either in
// /tmp (which is a default) or some other path as specified by
// --go-path parameter. The latter is always in /tmp.
func getGoPath(opts *options, tmpDir string) (string, string) {
	fakeGoPath := filepath.Join(tmpDir, "gopath")
	if opts.goPath == "" {
		return fakeGoPath, fakeGoPath
	}
	return opts.goPath, fakeGoPath
}

// getNamesFromProject returns project name, image ACName and ACI
// filename. Names depend on whether project has several binaries. For
// project with single binary (github.com/appc/goaci) returned values
// would be: github.com/appc/goaci, github.com/appc/goaci and
// goaci.aci. For project with multiple binaries
// (github.com/appc/spec/...) returned values would be (assuming ace
// as selected binary): github.com/appc/spec, github.com/appc/spec-ace
// and spec-ace.aci.
func getNamesFromProject(opts *options) (string, string, string) {
	imageACName := opts.project
	projectName := imageACName
	base := filepath.Base(imageACName)
	threeDotsBase := base == "..."
	if threeDotsBase {
		imageACName = filepath.Dir(imageACName)
		projectName = imageACName
		base = filepath.Base(imageACName)
		if opts.useBinary != "" {
			suffix := "-" + opts.useBinary
			base += suffix
			imageACName += suffix
		}
	}

	return projectName, imageACName, base + schema.ACIExtension
}

func getPathsAndNames(opts *options) (*pathsAndNames, error) {
	tmpDir, err := ioutil.TempDir("", "goaci")
	if err != nil {
		return nil, fmt.Errorf("error setting up temporary directory: %v", err)
	}

	goPath, fakeGoPath := getGoPath(opts, tmpDir)
	projectName, imageACName, imageFileName := getNamesFromProject(opts)

	if os.Getenv("GOPATH") != "" {
		Warn("GOPATH env var is ignored, use --go-path=\"$GOPATH\" option instead")
	}
	goRoot := os.Getenv("GOROOT")
	if goRoot != "" {
		Warn("Overriding GOROOT env var to ", goRoot)
	}

	aciDir := filepath.Join(tmpDir, "aci")
	// Project name is path-like string with slashes, but slash is
	// not a file separator on every OS.
	projectPath := filepath.Join(goPath, "src", filepath.Join(strings.Split(projectName, "/")...))

	return &pathsAndNames{
		tmpDirPath:  tmpDir, // /tmp/XXX
		goPath:      goPath,
		goRootPath:  goRoot,
		projectPath: projectPath,
		fakeGoPath:  fakeGoPath,                       // /tmp/XXX/gopath
		goBinPath:   filepath.Join(fakeGoPath, "bin"), // /tmp/XXX/gopath/bin
		aciDirPath:  aciDir,                           // /tmp/XXX/aci
		rootFSPath:  filepath.Join(aciDir, "rootfs"),  // /tmp/XXX/aci/rootfs
		goExecPath:  opts.goBinary,

		imageFileName: imageFileName,
		imageACName:   imageACName,
	}, nil
}

func makeDirectories(pathsNames *pathsAndNames) error {
	// /tmp/XXX already exists, not creating it here

	// /tmp/XXX/gopath
	if err := os.Mkdir(pathsNames.fakeGoPath, 0755); err != nil {
		return err
	}

	// /tmp/XXX/gopath/bin
	if err := os.Mkdir(pathsNames.goBinPath, 0755); err != nil {
		return err
	}

	// /tmp/XXX/aci
	if err := os.Mkdir(pathsNames.aciDirPath, 0755); err != nil {
		return err
	}

	// /tmp/XXX/aci/rootfs
	if err := os.Mkdir(pathsNames.rootFSPath, 0755); err != nil {
		return err
	}

	return nil
}

func runGoGet(opts *options, pathsNames *pathsAndNames) error {
	// Construct args for a go get that does a static build
	args := []string{
		pathsNames.goExecPath,
		"get",
		"-a",
		"-tags", "netgo",
		"-ldflags", "'-w'",
		"-installsuffix", "nocgo", // for 1.4
		opts.project,
	}

	env := []string{
		"GOPATH=" + pathsNames.goPath,
		"GOBIN=" + pathsNames.goBinPath,
		"CGO_ENABLED=0",
		"PATH=" + os.Getenv("PATH"),
	}
	if pathsNames.goRootPath != "" {
		env = append(env, "GOROOT="+pathsNames.goRootPath)
	}

	cmd := exec.Cmd{
		Env:    env,
		Path:   pathsNames.goExecPath,
		Args:   args,
		Stderr: os.Stderr,
		Stdout: os.Stdout,
	}
	Debug("env: ", cmd.Env)
	Debug("running command: ", strings.Join(cmd.Args, " "))
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

// getBinaryName get a binary name built by go get and selected by
// --use-binary parameter.
func getBinaryName(opts *options, pathsNames *pathsAndNames) (string, error) {
	fi, err := ioutil.ReadDir(pathsNames.goBinPath)
	if err != nil {
		return "", err
	}

	switch {
	case len(fi) < 1:
		return "", fmt.Errorf("No binaries found in gobin.")
	case len(fi) == 1:
		name := fi[0].Name()
		if opts.useBinary != "" && name != opts.useBinary {
			return "", fmt.Errorf("No such binary found in gobin: %q. There is only %q", opts.useBinary, name)
		}
		Debug("found binary: ", name)
		return name, nil
	case len(fi) > 1:
		names := []string{}
		for _, v := range fi {
			names = append(names, v.Name())
		}
		if opts.useBinary == "" {
			return "", fmt.Errorf("Found multiple binaries in gobin, but --use-binary option is not used. Please specify which binary to put in ACI. Following binaries are available: %q", strings.Join(names, `", "`))
		}
		for _, v := range names {
			if v == opts.useBinary {
				return v, nil
			}
		}
		return "", fmt.Errorf("No such binary found in gobin: %q. There are following binaries available: %q", opts.useBinary, strings.Join(names, `", "`))
	}
	return "", fmt.Errorf("Reaching this point shouldn't be possible.")
}

func getApp(opts *options, binary string) *types.App {
	exec := []string{filepath.Join("/", binary)}
	exec = append(exec, opts.exec...)

	return &types.App{
		Exec:  exec,
		User:  "0",
		Group: "0",
	}
}

func getVCSLabel(pathsNames *pathsAndNames) (*types.Label, error) {
	name, value, err := GetVCSInfo(pathsNames.projectPath)
	if err != nil {
		return nil, fmt.Errorf("Failed to get VCS info: %v", err)
	}
	acname, err := types.NewACName(name)
	if err != nil {
		return nil, fmt.Errorf("Invalid VCS label: %v", err)
	}
	return &types.Label{
		Name:  *acname,
		Value: value,
	}, nil
}

func prepareManifest(opts *options, pathsNames *pathsAndNames, binary string) (*schema.ImageManifest, error) {
	name, err := types.NewACName(pathsNames.imageACName)
	// TODO(jonboulle): could this ever actually happen?
	if err != nil {
		return nil, err
	}

	app := getApp(opts, binary)

	vcsLabel, err := getVCSLabel(pathsNames)
	if err != nil {
		return nil, err
	}
	labels := types.Labels{
		*vcsLabel,
	}

	manifest := schema.BlankImageManifest()
	manifest.Name = *name
	manifest.App = app
	manifest.Labels = labels
	return manifest, nil
}

func copyAssets(opts *options, pathsNames *pathsAndNames) error {
	placeholderMapping := map[string]string{
		"<PROJPATH>": pathsNames.projectPath,
		"<GOPATH>":   pathsNames.goPath,
	}
	if err := PrepareAssets(opts.assets, pathsNames.rootFSPath, placeholderMapping); err != nil {
		return err
	}
	return nil
}

func moveBinaryToRootFS(pathsNames *pathsAndNames, binary string) error {
	// Move the binary into the rootfs
	ep := filepath.Join(pathsNames.rootFSPath, binary)
	if err := os.Rename(filepath.Join(pathsNames.goBinPath, binary), ep); err != nil {
		return err
	}
	Debug("moved binary to: ", ep)
	return nil
}

func writeACI(pathsNames *pathsAndNames, manifest *schema.ImageManifest) error {
	mode := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
	of, err := os.OpenFile(pathsNames.imageFileName, mode, 0644)
	if err != nil {
		return fmt.Errorf("Error opening output file: %v", err)
	}
	defer of.Close()

	gw := gzip.NewWriter(of)
	defer gw.Close()

	tr := tar.NewWriter(gw)
	defer tr.Close()

	// FIXME: the files in the tar archive are added with the
	// wrong uid/gid. The uid/gid of the aci builder leaks in the
	// tar archive. See: #16
	iw := aci.NewImageWriter(*manifest, tr)
	if err := filepath.Walk(pathsNames.aciDirPath, aci.BuildWalker(pathsNames.aciDirPath, iw)); err != nil {
		return err
	}
	if err := iw.Close(); err != nil {
		return err
	}
	Info("Wrote ", of.Name())
	return nil
}

func mainWithError() error {
	InitDebug()

	opts, err := getOptions()
	if err != nil {
		return err
	}

	pathsNames, err := getPathsAndNames(opts)
	if err != nil {
		return err
	}

	if opts.keepTmp {
		Info(`Preserving temporary directory "`, pathsNames.tmpDirPath, `"`)
	} else {
		defer os.RemoveAll(pathsNames.tmpDirPath)
	}

	if err := makeDirectories(pathsNames); err != nil {
		return err
	}

	if err := runGoGet(opts, pathsNames); err != nil {
		return err
	}

	binary, err := getBinaryName(opts, pathsNames)
	if err != nil {
		return err
	}

	manifest, err := prepareManifest(opts, pathsNames, binary)
	if err != nil {
		return err
	}

	if err := copyAssets(opts, pathsNames); err != nil {
		return err
	}

	if err := moveBinaryToRootFS(pathsNames, binary); err != nil {
		return err
	}

	if err := writeACI(pathsNames, manifest); err != nil {
		return err
	}

	return nil
}

func main() {
	if err := mainWithError(); err != nil {
		Warn(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
)

var ARGV []string

func main() {
	ARGV = os.Args[1:]

	if len(ARGV) == 0 {
		// Read stdin only
		io.Copy(os.Stdout, os.Stdin)

	} else {
		// Read ARGV only
		for _, filename := range ARGV {
			// - means read stdin as a special case
			if filename == "-" {
				filename = "/dev/stdin"
			}

			// Otherwise we're after a file itself
			f, err := os.Open(filename)
			if err != nil {
				fmt.Fprintf(os.Stderr, "gocat: %s: No such file or directory\n", filename)
				continue
			}

			// Copy our output across!
			io.Copy(os.Stdout, f)
		}

	}
}
<commit_msg>Simplify processing with no arguments<commit_after>package main

import (
	"fmt"
	"io"
	"os"
)

var ARGV []string

func main() {
	ARGV = os.Args[1:]

	// If there are no arguments, then we read STDIN
	if len(ARGV) == 0 {
		ARGV = append(ARGV, "-")
	}

	// Iterate arguments to read filenames
	for _, filename := range ARGV {
		// - means read stdin as a special case, we just use /dev/stdin instead
		if filename == "-" {
			filename = "/dev/stdin"
		}

		// Time to try and read the file in question
		f, err := os.Open(filename)
		if err != nil {
			fmt.Fprintf(os.Stderr, "gocat: %s: No such file or directory\n", filename)
			continue
		}

		// Copy our output across!
		io.Copy(os.Stdout, f)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/musl/libgofr"
	"github.com/nfnt/resize"
	"image"
	"image/png"
	"log"
	"net/http"
	"runtime"
	"strconv"
	"time"
)

const Version = "0.0.2"

var id_chan = make(chan int, 1)

func finish(w http.ResponseWriter, status int, message string) {
	w.WriteHeader(status)
	fmt.Fprintf(w, message)
}

func logDuration(message string, start time.Time) {
	end := time.Now()
	log.Printf("%s: %v\n", message, end.Sub(start))
}

func route_png(w http.ResponseWriter, r *http.Request) {
	id := <-id_chan
	start := time.Now()
	defer logDuration(fmt.Sprintf("%08d %s", id, r.URL.Path), start)

	if r.Method != "GET" {
		finish(w, http.StatusMethodNotAllowed, "Method not allowed.")
		return
	}

	q := r.URL.Query()

	s, err := strconv.Atoi(q.Get("s"))
	if err != nil {
		s = 1
	}

	width, err := strconv.Atoi(q.Get("w"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid width")
		return
	}

	height, err := strconv.Atoi(q.Get("h"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid height")
		return
	}

	iterations, err := strconv.Atoi(q.Get("i"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid i")
		return
	}

	er, err := strconv.ParseFloat(q.Get("e"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid e")
		return
	}

	rmin, err := strconv.ParseFloat(q.Get("rmin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	imin, err := strconv.ParseFloat(q.Get("imin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	rmax, err := strconv.ParseFloat(q.Get("rmax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmax")
		return
	}

	imax, err := strconv.ParseFloat(q.Get("imax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	c := q.Get("c")
	hex := q.Get("m")

	p := gofr.Parameters{
		ImageWidth:   width * s,
		ImageHeight:  height * s,
		MaxI:         iterations,
		EscapeRadius: er,
		Min:          complex(rmin, imin),
		Max:          complex(rmax, imax),
		ColorFunc:    c,
		MemberColor:  hex,
	}

	log.Printf("%08d rendering\n", id)

	// TODO: Check parameters and set reasonable bounds on what we can
	// quickly calculate.

	img := image.NewNRGBA64(image.Rect(0, 0, p.ImageWidth, p.ImageHeight))
	n := runtime.NumCPU()
	contexts := gofr.MakeContexts(img, n, &p)
	gofr.Render(n, contexts, gofr.Mandelbrot)

	scaled_img := resize.Resize(uint(width), uint(height), image.Image(img), resize.Lanczos3)

	w.Header().Set("Content-Type", "image/png")
	png.Encode(w, scaled_img)
}

func main() {
	fs := http.FileServer(http.Dir("static"))
	bind_addr := "0.0.0.0:8000"

	go func() {
		for i := 0; ; i++ {
			id_chan <- i
		}
	}()

	http.Handle("/", fs)
	http.HandleFunc("/png", route_png)
	log.Printf("gofrd v%s", Version)
	log.Printf("libgofrd v%s", gofr.Version)
	log.Printf("Listening on: %s\n", bind_addr)
	log.Fatal(http.ListenAndServe(bind_addr, nil))
}
<commit_msg>Use UUIDs instead of counter, fix-up logging<commit_after>package main

import (
	"fmt"
	"github.com/google/uuid"
	"github.com/musl/libgofr"
	"github.com/nfnt/resize"
	"image"
	"image/png"
	"log"
	"net/http"
	"runtime"
	"strconv"
	"time"
)

const Version = "0.0.2"

var id_chan = make(chan uuid.UUID, 100)

func finish(w http.ResponseWriter, status int, message string) {
	w.WriteHeader(status)
	fmt.Fprintf(w, message)
}

func logDuration(message string, start time.Time) {
	end := time.Now()
	log.Printf("%s %v\n", message, end.Sub(start))
}

func route_png(w http.ResponseWriter, r *http.Request) {
	id := <-id_chan
	start := time.Now()
	log.Printf("%s %s %s %s", id, r.RemoteAddr, r.Method, r.URL.Path)
	defer logDuration(fmt.Sprintf("%s", id), start)

	if r.Method != "GET" {
		finish(w, http.StatusMethodNotAllowed, "Method not allowed.")
		return
	}

	q := r.URL.Query()

	s, err := strconv.Atoi(q.Get("s"))
	if err != nil {
		s = 1
	}

	width, err := strconv.Atoi(q.Get("w"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid width")
		return
	}

	height, err := strconv.Atoi(q.Get("h"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid height")
		return
	}

	iterations, err := strconv.Atoi(q.Get("i"))
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid i")
		return
	}

	er, err := strconv.ParseFloat(q.Get("e"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid e")
		return
	}

	rmin, err := strconv.ParseFloat(q.Get("rmin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	imin, err := strconv.ParseFloat(q.Get("imin"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	rmax, err := strconv.ParseFloat(q.Get("rmax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmax")
		return
	}

	imax, err := strconv.ParseFloat(q.Get("imax"), 64)
	if err != nil {
		finish(w, http.StatusUnprocessableEntity, "Invalid rmin")
		return
	}

	c := q.Get("c")
	hex := q.Get("m")

	p := gofr.Parameters{
		ImageWidth:   width * s,
		ImageHeight:  height * s,
		MaxI:         iterations,
		EscapeRadius: er,
		Min:          complex(rmin, imin),
		Max:          complex(rmax, imax),
		ColorFunc:    c,
		MemberColor:  hex,
	}

	// TODO: Check parameters and set reasonable bounds on what we can
	// quickly calculate.

	img := image.NewNRGBA64(image.Rect(0, 0, p.ImageWidth, p.ImageHeight))
	n := runtime.NumCPU()
	contexts := gofr.MakeContexts(img, n, &p)
	gofr.Render(n, contexts, gofr.Mandelbrot)

	scaled_img := resize.Resize(uint(width), uint(height), image.Image(img), resize.Lanczos3)

	w.Header().Set("Content-Type", "image/png")
	png.Encode(w, scaled_img)
}

func main() {
	fs := http.FileServer(http.Dir("static"))
	bind_addr := "0.0.0.0:8000"

	go func() {
		for i := 0; ; i++ {
			id_chan <- uuid.New()
		}
	}()

	http.Handle("/", fs)
	http.HandleFunc("/png", route_png)
	log.Printf("gofrd v%s", Version)
	log.Printf("libgofrd v%s", gofr.Version)
	log.Printf("Listening on: %s\n", bind_addr)
	log.Fatal(http.ListenAndServe(bind_addr, nil))
}
<|endoftext|>


<commit_before>package golog

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"

	"github.com/YueHonghui/rfw"
)

const (
	LevelTRC = iota
	LevelDBG
	LevelINF
	LevelWRN
	LevelERR
)

var (
	ErrLevelInvalid = errors.New("Level invalid. Level must be TRC|DBG|INF|WRN|ERR")
)

var (
	glock     sync.RWMutex
	glevel    int = LevelTRC
	gnewline  string
	gwriter   io.WriteCloser
	glogFatal *log.Logger
	glogERR   *log.Logger
	glogWRN   *log.Logger
	glogINF   *log.Logger
	glogDBG   *log.Logger
	glogTRC   *log.Logger
)

func ParseLevel(lv string) (int, bool) {
	if lv == "TRC" {
		return LevelTRC, true
	} else if lv == "DBG" {
		return LevelDBG, true
	} else if lv == "INF" {
		return LevelINF, true
	} else if lv == "WRN" {
		return LevelWRN, true
	} else if lv == "ERR" {
		return LevelERR, true
	} else {
		return LevelTRC, false
	}
}

func init() {
	glock.Lock()
	defer glock.Unlock()
	if runtime.GOOS == "windows" {
		gnewline = "\r\n"
	} else {
		gnewline = "\n"
	}
	glogFatal = log.New(os.Stderr, "[FAT] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogERR = log.New(os.Stdout, "[ERR] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogWRN = log.New(os.Stdout, "[WRN] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogINF = log.New(os.Stdout, "[INF] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogDBG = log.New(os.Stdout, "[DBG] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogTRC = log.New(os.Stdout, "[TRC] ", log.Ldate|log.Ltime|log.Lshortfile)
}

func parseLogUrl(url string) (schema, uri string, keyvalues map[string]string, err error) {
	keyvalues = make(map[string]string)
	itms := strings.Split(url, ",")
	schitms := strings.Split(itms[0], "://")
	if len(schitms) != 2 {
		err = errors.New(fmt.Sprintf("logurl invalid: %s", url))
		return
	}
	if schitms[0] != "file" {
		err = errors.New(fmt.Sprintf("schema %s in logurl %s not supported yet", schitms[0], url))
		return
	}
	schema = schitms[0]
	uri = schitms[1]
	for _, v := range itms[1:] {
		kvs := strings.Split(v, "=")
		if len(kvs) != 2 {
			err = errors.New(fmt.Sprintf("keyvalue %s in logurl %s invalid", v, url))
			return
		}
		keyvalues[kvs[0]] = kvs[1]
	}
	return
}

//@logurl
//    logurl used to determin to which place and how the log will be write, generate format is "[schema]://[uri],[key=value]...", for example
//	file:///home/logs/demo,[rotate=day|none],[level=TRC|DBG|INF|WRN|ERR]
func Init(logurl string) (err error) {
	var uri string
	var kvs map[string]string
	_, uri, kvs, err = parseLogUrl(logurl)
	if err != nil {
		return
	}
	glock.Lock()
	defer glock.Unlock()
	if gwriter != nil {
		glogERR.Fatalf("InitLogger must called only one time%s", gnewline)
		return
	}
	if rt, ok := kvs["rotate"]; ok && rt == "day" {
		gwriter, err = rfw.New(uri)
		if err != nil {
			return
		}
	} else {
		gwriter, err = os.OpenFile(uri, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0664)
		if err != nil {
			return
		}
	}
	if lv, ok := kvs["level"]; ok {
		level, valid := ParseLevel(lv)
		if !valid {
			gwriter.Close()
			return ErrLevelInvalid
		}
		SetLevel(level)
	}
	glogFatal = log.New(gwriter, "[FAT] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogERR = log.New(gwriter, "[ERR] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogWRN = log.New(gwriter, "[WRN] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogINF = log.New(gwriter, "[INF] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogDBG = log.New(gwriter, "[DBG] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogTRC = log.New(gwriter, "[TRC] ", log.Ldate|log.Ltime|log.Lshortfile)
	return
}

func SetLevel(level int) {
	glock.Lock()
	defer glock.Unlock()
	glevel = level
}

func GetLevel() int {
	return glevel
}

func TRC(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelTRC {
		glogTRC.Output(2, fmt.Sprintf(format+gnewline, v...))
	}
}

func DBG(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelDBG {
		glogDBG.Output(2, fmt.Sprintf(format+gnewline, v...))
	}
}

func INF(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelINF {
		glogINF.Output(2, fmt.Sprintf(format+gnewline, v...))
	}
}

func WRN(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelWRN {
		glogWRN.Output(2, fmt.Sprintf(format+gnewline, v...))
	}
}

func ERR(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelERR {
		glogERR.Output(2, fmt.Sprintf(format+gnewline, v...))
	}
}

func FAT(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	glogFatal.Output(2, fmt.Sprintf(format+gnewline, v...))
	os.Exit(1)
}

func TRCf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelTRC {
		glogTRC.Output(2, fmt.Sprintf(format, v...))
	}
}

func DBGf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelDBG {
		glogDBG.Output(2, fmt.Sprintf(format, v...))
	}
}

func INFf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelINF {
		glogINF.Output(2, fmt.Sprintf(format, v...))
	}
}

func WRNf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelWRN {
		glogWRN.Output(2, fmt.Sprintf(format, v...))
	}
}

func ERRf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelERR {
		glogERR.Output(2, fmt.Sprintf(format, v...))
	}
}
func Fatalf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	glogFatal.Output(2, fmt.Sprintf(format, v...))
	os.Exit(1)
}

func Fini() {
	glock.Lock()
	defer glock.Unlock()
	if gwriter != nil {
		gwriter.Close()
		gwriter = nil
	}
	glogFatal = log.New(os.Stderr, "[FAT] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogERR = log.New(os.Stdout, "[ERR] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogWRN = log.New(os.Stdout, "[WRN] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogINF = log.New(os.Stdout, "[INF] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogDBG = log.New(os.Stdout, "[DBG] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogTRC = log.New(os.Stdout, "[TRC] ", log.Ldate|log.Ltime|log.Lshortfile)
}
<commit_msg>fix dead lock<commit_after>package golog

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"

	"github.com/YueHonghui/rfw"
)

const (
	LevelTRC = iota
	LevelDBG
	LevelINF
	LevelWRN
	LevelERR
)

var (
	ErrLevelInvalid = errors.New("Level invalid. Level must be TRC|DBG|INF|WRN|ERR")
)

var (
	glock     sync.RWMutex
	glevel    int = LevelTRC
	gnewline  string
	gwriter   io.WriteCloser
	glogFatal *log.Logger
	glogERR   *log.Logger
	glogWRN   *log.Logger
	glogINF   *log.Logger
	glogDBG   *log.Logger
	glogTRC   *log.Logger
)

func ParseLevel(lv string) (int, bool) {
	if lv == "TRC" {
		return LevelTRC, true
	} else if lv == "DBG" {
		return LevelDBG, true
	} else if lv == "INF" {
		return LevelINF, true
	} else if lv == "WRN" {
		return LevelWRN, true
	} else if lv == "ERR" {
		return LevelERR, true
	} else {
		return LevelTRC, false
	}
}

func init() {
	glock.Lock()
	defer glock.Unlock()
	if runtime.GOOS == "windows" {
		gnewline = "\r\n"
	} else {
		gnewline = "\n"
	}
	glogFatal = log.New(os.Stderr, "[FAT] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogERR = log.New(os.Stdout, "[ERR] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogWRN = log.New(os.Stdout, "[WRN] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogINF = log.New(os.Stdout, "[INF] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogDBG = log.New(os.Stdout, "[DBG] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogTRC = log.New(os.Stdout, "[TRC] ", log.Ldate|log.Ltime|log.Lshortfile)
}

func parseLogUrl(url string) (schema, uri string, keyvalues map[string]string, err error) {
	keyvalues = make(map[string]string)
	itms := strings.Split(url, ",")
	schitms := strings.Split(itms[0], "://")
	if len(schitms) != 2 {
		err = errors.New(fmt.Sprintf("logurl invalid: %s", url))
		return
	}
	if schitms[0] != "file" {
		err = errors.New(fmt.Sprintf("schema %s in logurl %s not supported yet", schitms[0], url))
		return
	}
	schema = schitms[0]
	uri = schitms[1]
	for _, v := range itms[1:] {
		kvs := strings.Split(v, "=")
		if len(kvs) != 2 {
			err = errors.New(fmt.Sprintf("keyvalue %s in logurl %s invalid", v, url))
			return
		}
		keyvalues[kvs[0]] = kvs[1]
	}
	return
}

//@logurl
//    logurl used to determin to which place and how the log will be write, generate format is "[schema]://[uri],[key=value]...", for example
//	file:///home/logs/demo,[rotate=day|none],[level=TRC|DBG|INF|WRN|ERR]
func Init(logurl string) (err error) {
	var uri string
	var kvs map[string]string
	_, uri, kvs, err = parseLogUrl(logurl)
	if err != nil {
		return
	}
	glock.Lock()
	defer glock.Unlock()
	if gwriter != nil {
		glogERR.Fatalf("InitLogger must called only one time%s", gnewline)
		return
	}
	if rt, ok := kvs["rotate"]; ok && rt == "day" {
		gwriter, err = rfw.New(uri)
		if err != nil {
			return
		}
	} else {
		gwriter, err = os.OpenFile(uri, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0664)
		if err != nil {
			return
		}
	}
	if lv, ok := kvs["level"]; ok {
		level, valid := ParseLevel(lv)
		if !valid {
			gwriter.Close()
			return ErrLevelInvalid
		}
		glevel = level
	}
	glogFatal = log.New(gwriter, "[FAT] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogERR = log.New(gwriter, "[ERR] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogWRN = log.New(gwriter, "[WRN] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogINF = log.New(gwriter, "[INF] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogDBG = log.New(gwriter, "[DBG] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogTRC = log.New(gwriter, "[TRC] ", log.Ldate|log.Ltime|log.Lshortfile)
	return
}

func SetLevel(level int) {
	glock.Lock()
	defer glock.Unlock()
	glevel = level
}

func GetLevel() int {
	return glevel
}

func TRC(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelTRC {
		glogTRC.Output(2, fmt.Sprintf(format+gnewline, v...))
	}
}

func DBG(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelDBG {
		glogDBG.Output(2, fmt.Sprintf(format+gnewline, v...))
	}
}

func INF(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelINF {
		glogINF.Output(2, fmt.Sprintf(format+gnewline, v...))
	}
}

func WRN(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelWRN {
		glogWRN.Output(2, fmt.Sprintf(format+gnewline, v...))
	}
}

func ERR(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelERR {
		glogERR.Output(2, fmt.Sprintf(format+gnewline, v...))
	}
}

func FAT(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	glogFatal.Output(2, fmt.Sprintf(format+gnewline, v...))
	os.Exit(1)
}

func TRCf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelTRC {
		glogTRC.Output(2, fmt.Sprintf(format, v...))
	}
}

func DBGf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelDBG {
		glogDBG.Output(2, fmt.Sprintf(format, v...))
	}
}

func INFf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelINF {
		glogINF.Output(2, fmt.Sprintf(format, v...))
	}
}

func WRNf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelWRN {
		glogWRN.Output(2, fmt.Sprintf(format, v...))
	}
}

func ERRf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	if glevel <= LevelERR {
		glogERR.Output(2, fmt.Sprintf(format, v...))
	}
}
func Fatalf(format string, v ...interface{}) {
	glock.RLock()
	defer glock.RUnlock()
	glogFatal.Output(2, fmt.Sprintf(format, v...))
	os.Exit(1)
}

func Fini() {
	glock.Lock()
	defer glock.Unlock()
	if gwriter != nil {
		gwriter.Close()
		gwriter = nil
	}
	glogFatal = log.New(os.Stderr, "[FAT] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogERR = log.New(os.Stdout, "[ERR] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogWRN = log.New(os.Stdout, "[WRN] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogINF = log.New(os.Stdout, "[INF] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogDBG = log.New(os.Stdout, "[DBG] ", log.Ldate|log.Ltime|log.Lshortfile)
	glogTRC = log.New(os.Stdout, "[TRC] ", log.Ldate|log.Ltime|log.Lshortfile)
}
<|endoftext|>


<commit_before>package gorex

import (
	"bytes"
	"encoding/json"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"time"
)

func reqFormFileDisk(r Request, params map[string]string, paramName string, filePath string) (Request, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return r, err
	}
	defer file.Close()
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, filepath.Base(filePath))
	if err != nil {
		return r, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		return r, err
	}
	for key, val := range params {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return r, err
	}
	req, err := http.NewRequest(r.Method, r.URI, body)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	r.Req = req
	return r, err
}

func reqFormFile(r Request, params map[string]string, paramName string, fileName string, fileBuffer *bytes.Buffer) (Request, error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, fileName)
	if err != nil {
		return r, err
	}
	_, err = io.Copy(part, fileBuffer)
	if err != nil {
		return r, err
	}
	for key, val := range params {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return r, err
	}
	req, err := http.NewRequest(r.Method, r.URI, body)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	r.Req = req
	return r, err
}

func reqJSON(r Request, body interface{}) (Request, error) {
	marshalled, err := json.Marshal(body)
	if err != nil {
		return r, err
	}
	jsonBuffer := bytes.NewBuffer(marshalled)
	req, err := http.NewRequest(r.Method, r.URI, jsonBuffer)
	req.Header.Add("Content-Type", "application/json")
	r.Req = req
	return r, err
}

// Request - the request object
type Request struct {
	URI     string
	Method  string
	Req     *http.Request
	Timeout time.Duration
}

// Response - the response object
type Response struct {
	Res *http.Response
	URI string
}

// FormFileDisk - a request for a multipart upload with file path with optional params
func (r Request) FormFileDisk(params map[string]string, paramName string, fileName string, filePath string) (Request, error) {
	return reqFormFileDisk(r, params, paramName, filePath)
}

// FormFile - a request for a multipart upload with file buffer with optional params
func (r Request) FormFile(params map[string]string, paramName string, fileName string, fileBuffer *bytes.Buffer) (Request, error) {
	return reqFormFile(r, params, paramName, fileName, fileBuffer)
}

// JSON - a request to a JSON endpoint
func (r Request) JSON(body interface{}) (Request, error) {
	return reqJSON(r, body)
}

// Do - process the request with timeout
func (r Request) Do() (Response, error) {
	client := &http.Client{}
	client.Timeout = r.Timeout
	res, err := client.Do(r.Req)
	return Response{res, r.URI}, err
}

// JSON - decode JSON to interface
func (e Response) JSON(decoder interface{}) (*interface{}, error) {
	defer e.Res.Body.Close()
	resp := &decoder
	err := json.NewDecoder(e.Res.Body).Decode(resp)
	return resp, err
}
<commit_msg>remove extra param<commit_after>package gorex

import (
	"bytes"
	"encoding/json"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"time"
)

func reqFormFileDisk(r Request, params map[string]string, paramName string, filePath string) (Request, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return r, err
	}
	defer file.Close()
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, filepath.Base(filePath))
	if err != nil {
		return r, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		return r, err
	}
	for key, val := range params {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return r, err
	}
	req, err := http.NewRequest(r.Method, r.URI, body)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	r.Req = req
	return r, err
}

func reqFormFile(r Request, params map[string]string, paramName string, fileName string, fileBuffer *bytes.Buffer) (Request, error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, fileName)
	if err != nil {
		return r, err
	}
	_, err = io.Copy(part, fileBuffer)
	if err != nil {
		return r, err
	}
	for key, val := range params {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return r, err
	}
	req, err := http.NewRequest(r.Method, r.URI, body)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	r.Req = req
	return r, err
}

func reqJSON(r Request, body interface{}) (Request, error) {
	marshalled, err := json.Marshal(body)
	if err != nil {
		return r, err
	}
	jsonBuffer := bytes.NewBuffer(marshalled)
	req, err := http.NewRequest(r.Method, r.URI, jsonBuffer)
	req.Header.Add("Content-Type", "application/json")
	r.Req = req
	return r, err
}

// Request - the request object
type Request struct {
	URI     string
	Method  string
	Req     *http.Request
	Timeout time.Duration
}

// Response - the response object
type Response struct {
	Res *http.Response
	URI string
}

// FormFileDisk - a request for a multipart upload with file path with optional params
func (r Request) FormFileDisk(params map[string]string, paramName string, filePath string) (Request, error) {
	return reqFormFileDisk(r, params, paramName, filePath)
}

// FormFile - a request for a multipart upload with file buffer with optional params
func (r Request) FormFile(params map[string]string, paramName string, fileName string, fileBuffer *bytes.Buffer) (Request, error) {
	return reqFormFile(r, params, paramName, fileName, fileBuffer)
}

// JSON - a request to a JSON endpoint
func (r Request) JSON(body interface{}) (Request, error) {
	return reqJSON(r, body)
}

// Do - process the request with timeout
func (r Request) Do() (Response, error) {
	client := &http.Client{}
	client.Timeout = r.Timeout
	res, err := client.Do(r.Req)
	return Response{res, r.URI}, err
}

// JSON - decode JSON to interface
func (e Response) JSON(decoder interface{}) (*interface{}, error) {
	defer e.Res.Body.Close()
	resp := &decoder
	err := json.NewDecoder(e.Res.Body).Decode(resp)
	return resp, err
}
<|endoftext|>


<commit_before>package adder

func Add(n1, n2 int) int {
	return n1 + n2
}
<commit_msg>Add a Subtract function<commit_after>package adder

func Add(n1, n2 int) int {
	return n1 + n2
}

func Subtract(n1, n2 int) int {
	return n1 - n2
}
<|endoftext|>