<commit_before>package bitsmanager

import (
	"bytes"
	"code.cloudfoundry.org/cli/cf/api/resources"
	"code.cloudfoundry.org/cli/cf/configuration/coreconfig"
	. "code.cloudfoundry.org/cli/cf/i18n"
	"code.cloudfoundry.org/cli/cf/net"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/cloudfoundry/gofileutils/fileutils"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"runtime"
	"time"
)

const (
	DefaultAppUploadBitsTimeout = 15 * time.Minute
)

//go:generate counterfeiter . Repository
type Job struct {
	Metadata struct {
		GUID      string    `json:"guid"`
		CreatedAt time.Time `json:"created_at"`
		URL       string    `json:"url"`
	} `json:"metadata"`
	Entity struct {
		GUID         string `json:"guid"`
		Status       string `json:"status"`
		Error        string `json:"error"`
		ErrorDetails struct {
			Code        int    `json:"code"`
			Description string `json:"description"`
			ErrorCode   string `json:"error_code"`
		} `json:"error_details"`
	} `json:"entity"`
}
type ApplicationBitsRepository interface {
	GetApplicationSha1(appGUID string) (string, error)
	IsDiff(appGUID string, currentSha1 string) (bool, string, error)
	UploadBits(appGUID string, zipFile io.ReadCloser, fileSize int64) (apiErr error)
	CopyBits(origAppGuid string, newAppGuid string) error
}

type CloudControllerApplicationBitsRepository struct {
	config  coreconfig.Reader
	gateway net.Gateway
}

func NewCloudControllerApplicationBitsRepository(config coreconfig.Reader, gateway net.Gateway) (repo CloudControllerApplicationBitsRepository) {
	repo.config = config
	repo.gateway = gateway
	return
}
func (repo CloudControllerApplicationBitsRepository) IsDiff(appGUID string, currentSha1 string) (bool, string, error) {
	sha1Found, err := repo.GetApplicationSha1(appGUID)
	if err != nil {
		return true, "", err
	}
	return currentSha1 != sha1Found, sha1Found, nil
}
func (repo CloudControllerApplicationBitsRepository) GetApplicationSha1(appGUID string) (string, error) {
	// we are oblige to do the request by itself because cli is reading the full response body
	// to dump the response into a possible logger.
	// we need to read just few bytes to create the sha1
	apiURL := fmt.Sprintf("/v2/apps/%s/download", appGUID)
	request, err := http.NewRequest("GET", repo.config.APIEndpoint()+apiURL, nil)
	if err != nil {
		return "", fmt.Errorf("%s: %s", T("Error building request"), err.Error())
	}
	request.Header.Set("Authorization", repo.config.AccessToken())
	request.Header.Set("accept", "application/json")
	request.Header.Set("Connection", "close")
	request.Header.Set("content-type", "application/json")
	request.Header.Set("User-Agent", "go-cli "+repo.config.CLIVersion()+" / "+runtime.GOOS)

	tr := &http.Transport{
		Proxy:           http.ProxyFromEnvironment,
		TLSClientConfig: &tls.Config{InsecureSkipVerify: repo.config.IsSSLDisabled()},
	}
	client := &http.Client{
		Transport: tr,
		Timeout:   2 * time.Second,
	}

	resp, err := client.Do(request)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	sha1, err := GetSha1FromReader(resp.Body)
	if err != nil {
		return "", err
	}
	return sha1, nil
}
func (repo CloudControllerApplicationBitsRepository) CopyBits(origAppGuid string, newAppGuid string) error {
	apiURL := fmt.Sprintf("%s/v2/apps/%s/copy_bits", repo.config.APIEndpoint(), newAppGuid)
	data := bytes.NewReader([]byte(fmt.Sprintf(`{"source_app_guid":"%s"}`, origAppGuid)))
	req, err := repo.gateway.NewRequest("POST", apiURL, repo.config.AccessToken(), data)
	if err != nil {
		return err
	}
	var job Job
	_, err = repo.gateway.PerformRequestForJSONResponse(req, &job)
	if err != nil {
		return err
	}
	for {
		job, err := repo.getJob(job.Entity.GUID)
		if err != nil {
			return err
		}
		if job.Entity.Status == "finished" {
			return nil
		}
		if job.Entity.Status == "failed" {
			return fmt.Errorf(
				"Error %s, %s [code: %d]",
				job.Entity.ErrorDetails.ErrorCode,
				job.Entity.ErrorDetails.Description,
				job.Entity.ErrorDetails.Code,
			)
		}
		time.Sleep(2 * time.Second)
	}
	return nil
}
func (repo CloudControllerApplicationBitsRepository) getJob(jobGuid string) (Job, error) {
	apiURL := fmt.Sprintf("%s/v2/jobs/%s", repo.config.APIEndpoint(), jobGuid)
	req, err := repo.gateway.NewRequest("GET", apiURL, repo.config.AccessToken(), nil)
	if err != nil {
		return Job{}, err
	}
	var job Job
	_, err = repo.gateway.PerformRequestForJSONResponse(req, &job)
	if err != nil {
		return Job{}, err
	}
	return job, nil
}
func (repo CloudControllerApplicationBitsRepository) UploadBits(appGUID string, zipFile io.ReadCloser, fileSize int64) (apiErr error) {
	apiURL := fmt.Sprintf("/v2/apps/%s/bits", appGUID)
	fileutils.TempFile("requests", func(requestFile *os.File, err error) {
		if err != nil {
			apiErr = fmt.Errorf("%s: %s", T("Error creating tmp file: {{.Err}}", map[string]interface{}{"Err": err}), err.Error())
			return
		}

		presentFiles := []resources.AppFileResource{}

		presentFilesJSON, err := json.Marshal(presentFiles)
		if err != nil {
			apiErr = fmt.Errorf("%s: %s", T("Error marshaling JSON"), err.Error())
			return
		}

		boundary, err := repo.writeUploadBody(zipFile, fileSize, requestFile, presentFilesJSON)
		if err != nil {
			apiErr = fmt.Errorf("%s: %s", T("Error writing to tmp file: {{.Err}}", map[string]interface{}{"Err": err}), err.Error())
			return
		}

		var request *net.Request
		request, apiErr = repo.gateway.NewRequestForFile("PUT", repo.config.APIEndpoint()+apiURL, repo.config.AccessToken(), requestFile)
		if apiErr != nil {
			return
		}

		contentType := fmt.Sprintf("multipart/form-data; boundary=%s", boundary)
		request.HTTPReq.Header.Set("Content-Type", contentType)

		response := &resources.Resource{}
		_, apiErr = repo.gateway.PerformPollingRequestForJSONResponse(repo.config.APIEndpoint(), request, response, DefaultAppUploadBitsTimeout)
		if apiErr != nil {
			return
		}
	})

	return
}
func (repo CloudControllerApplicationBitsRepository) writeUploadBody(zipFile io.ReadCloser, fileSize int64, body *os.File, presentResourcesJSON []byte) (boundary string, err error) {
	writer := multipart.NewWriter(body)
	defer writer.Close()

	boundary = writer.Boundary()

	part, err := writer.CreateFormField("resources")
	if err != nil {
		return
	}
	_, err = io.Copy(part, bytes.NewBuffer(presentResourcesJSON))
	if err != nil {
		return
	}

	if zipFile != nil {

		part, zipErr := createZipPartWriter(fileSize, writer)
		if zipErr != nil {
			return
		}

		_, zipErr = io.Copy(part, zipFile)
		if zipErr != nil {
			return
		}
	}

	return
}

func createZipPartWriter(fileSize int64, writer *multipart.Writer) (io.Writer, error) {
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
	h.Set("Content-Type", "application/zip")
	h.Set("Content-Length", fmt.Sprintf("%d", fileSize))
	h.Set("Content-Transfer-Encoding", "binary")
	return writer.CreatePart(h)
}

//////////////////
// Not used for now, this is an intent to make an upload in full stream (no intermediate file)
func (repo CloudControllerApplicationBitsRepository) UploadBitsTmp(appGUID string, zipFile io.ReadCloser, fileSize int64) error {
	apiURL := fmt.Sprintf("/v2/apps/%s/bits", appGUID)
	buf := new(bytes.Buffer)
	io.Copy(buf, zipFile)
	panic(buf)
	r, w := io.Pipe()
	mpw := multipart.NewWriter(w)
	go func() {
		var err error
		defer mpw.Close()
		defer w.Close()
		part, err := mpw.CreateFormField("resources")
		if err != nil {
			panic(err)
		}
		_, err = io.Copy(part, bytes.NewBuffer([]byte("[]")))
		if err != nil {
			panic(err)
		}
		h := make(textproto.MIMEHeader)
		h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
		h.Set("Content-Type", "application/zip")
		h.Set("Content-Length", fmt.Sprintf("%d", fileSize))
		h.Set("Content-Transfer-Encoding", "binary")

		part, err = mpw.CreatePart(h)
		if err != nil {
			panic(err)
		}
		if _, err = io.Copy(part, zipFile); err != nil {
			panic(err)
		}
	}()
	var request *net.Request
	request, err := repo.gateway.NewRequest("PUT", repo.config.APIEndpoint()+apiURL, repo.config.AccessToken(), nil)
	if err != nil {
		return err
	}
	contentType := fmt.Sprintf("multipart/form-data; boundary=%s", mpw.Boundary())
	request.HTTPReq.Header.Set("Content-Type", contentType)
	request.HTTPReq.ContentLength = int64(repo.predictPart(int64(fileSize)))
	request.HTTPReq.Body = r

	response := &resources.Resource{}
	_, err = repo.gateway.PerformPollingRequestForJSONResponse(repo.config.APIEndpoint(), request, response, DefaultAppUploadBitsTimeout)
	if err != nil {
		panic(err)
	}
	return nil
}

func (repo CloudControllerApplicationBitsRepository) predictPart(filesize int64) int64 {
	buf := new(bytes.Buffer)
	mpw := multipart.NewWriter(buf)

	defer mpw.Close()
	part, err := mpw.CreateFormField("resources")
	if err != nil {
		panic(err)
	}
	_, err = io.Copy(part, bytes.NewBuffer([]byte("[]")))
	if err != nil {
		panic(err)
	}
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
	h.Set("Content-Type", "application/zip")
	h.Set("Content-Length", fmt.Sprintf("%d", filesize))
	h.Set("Content-Transfer-Encoding", "binary")

	part, err = mpw.CreatePart(h)
	b, _ := ioutil.ReadAll(buf)
	return int64(len(b)) + filesize
}

// end of the try
//////////////////
<commit_msg>app bits are now sent as a stream instead of creating intermediate file<commit_after>package bitsmanager

import (
	"bytes"
	"code.cloudfoundry.org/cli/cf/api/resources"
	"code.cloudfoundry.org/cli/cf/configuration/coreconfig"
	. "code.cloudfoundry.org/cli/cf/i18n"
	"code.cloudfoundry.org/cli/cf/net"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/cloudfoundry/gofileutils/fileutils"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"runtime"
	"time"
)

const (
	DefaultAppUploadBitsTimeout = 15 * time.Minute
)

//go:generate counterfeiter . Repository
type Job struct {
	Metadata struct {
		GUID      string    `json:"guid"`
		CreatedAt time.Time `json:"created_at"`
		URL       string    `json:"url"`
	} `json:"metadata"`
	Entity struct {
		GUID         string `json:"guid"`
		Status       string `json:"status"`
		Error        string `json:"error"`
		ErrorDetails struct {
			Code        int    `json:"code"`
			Description string `json:"description"`
			ErrorCode   string `json:"error_code"`
		} `json:"error_details"`
	} `json:"entity"`
}
type ApplicationBitsRepository interface {
	GetApplicationSha1(appGUID string) (string, error)
	IsDiff(appGUID string, currentSha1 string) (bool, string, error)
	UploadBits(appGUID string, zipFile io.ReadCloser, fileSize int64) (apiErr error)
	CopyBits(origAppGuid string, newAppGuid string) error
}

type CloudControllerApplicationBitsRepository struct {
	config  coreconfig.Reader
	gateway net.Gateway
}

func NewCloudControllerApplicationBitsRepository(config coreconfig.Reader, gateway net.Gateway) (repo CloudControllerApplicationBitsRepository) {
	repo.config = config
	repo.gateway = gateway
	return
}
func (repo CloudControllerApplicationBitsRepository) IsDiff(appGUID string, currentSha1 string) (bool, string, error) {
	sha1Found, err := repo.GetApplicationSha1(appGUID)
	if err != nil {
		return true, "", err
	}
	return currentSha1 != sha1Found, sha1Found, nil
}
func (repo CloudControllerApplicationBitsRepository) GetApplicationSha1(appGUID string) (string, error) {
	// we are oblige to do the request by itself because cli is reading the full response body
	// to dump the response into a possible logger.
	// we need to read just few bytes to create the sha1
	apiURL := fmt.Sprintf("/v2/apps/%s/download", appGUID)
	request, err := http.NewRequest("GET", repo.config.APIEndpoint()+apiURL, nil)
	if err != nil {
		return "", fmt.Errorf("%s: %s", T("Error building request"), err.Error())
	}
	request.Header.Set("Authorization", repo.config.AccessToken())
	request.Header.Set("accept", "application/json")
	request.Header.Set("Connection", "close")
	request.Header.Set("content-type", "application/json")
	request.Header.Set("User-Agent", "go-cli "+repo.config.CLIVersion()+" / "+runtime.GOOS)

	tr := &http.Transport{
		Proxy:           http.ProxyFromEnvironment,
		TLSClientConfig: &tls.Config{InsecureSkipVerify: repo.config.IsSSLDisabled()},
	}
	client := &http.Client{
		Transport: tr,
		Timeout:   2 * time.Second,
	}

	resp, err := client.Do(request)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	sha1, err := GetSha1FromReader(resp.Body)
	if err != nil {
		return "", err
	}
	return sha1, nil
}
func (repo CloudControllerApplicationBitsRepository) CopyBits(origAppGuid string, newAppGuid string) error {
	apiURL := fmt.Sprintf("%s/v2/apps/%s/copy_bits", repo.config.APIEndpoint(), newAppGuid)
	data := bytes.NewReader([]byte(fmt.Sprintf(`{"source_app_guid":"%s"}`, origAppGuid)))
	req, err := repo.gateway.NewRequest("POST", apiURL, repo.config.AccessToken(), data)
	if err != nil {
		return err
	}
	var job Job
	_, err = repo.gateway.PerformRequestForJSONResponse(req, &job)
	if err != nil {
		return err
	}
	for {
		job, err := repo.getJob(job.Entity.GUID)
		if err != nil {
			return err
		}
		if job.Entity.Status == "finished" {
			return nil
		}
		if job.Entity.Status == "failed" {
			return fmt.Errorf(
				"Error %s, %s [code: %d]",
				job.Entity.ErrorDetails.ErrorCode,
				job.Entity.ErrorDetails.Description,
				job.Entity.ErrorDetails.Code,
			)
		}
		time.Sleep(2 * time.Second)
	}
	return nil
}
func (repo CloudControllerApplicationBitsRepository) getJob(jobGuid string) (Job, error) {
	apiURL := fmt.Sprintf("%s/v2/jobs/%s", repo.config.APIEndpoint(), jobGuid)
	req, err := repo.gateway.NewRequest("GET", apiURL, repo.config.AccessToken(), nil)
	if err != nil {
		return Job{}, err
	}
	var job Job
	_, err = repo.gateway.PerformRequestForJSONResponse(req, &job)
	if err != nil {
		return Job{}, err
	}
	return job, nil
}

/////
//Old way to send bits, now do it as a stream
func (repo CloudControllerApplicationBitsRepository) UploadBitsTmp(appGUID string, zipFile io.ReadCloser, fileSize int64) (apiErr error) {
	apiURL := fmt.Sprintf("/v2/apps/%s/bits", appGUID)
	fileutils.TempFile("requests", func(requestFile *os.File, err error) {
		if err != nil {
			apiErr = fmt.Errorf("%s: %s", T("Error creating tmp file: {{.Err}}", map[string]interface{}{"Err": err}), err.Error())
			return
		}

		presentFiles := []resources.AppFileResource{}
		presentFilesJSON, _ := json.Marshal(presentFiles)

		boundary, err := repo.writeUploadBody(zipFile, fileSize, requestFile, presentFilesJSON)
		if err != nil {
			apiErr = fmt.Errorf("%s: %s", T("Error writing to tmp file: {{.Err}}", map[string]interface{}{"Err": err}), err.Error())
			return
		}

		var request *net.Request
		request, apiErr = repo.gateway.NewRequestForFile("PUT", repo.config.APIEndpoint()+apiURL, repo.config.AccessToken(), requestFile)
		if apiErr != nil {
			return
		}

		contentType := fmt.Sprintf("multipart/form-data; boundary=%s", boundary)
		request.HTTPReq.Header.Set("Content-Type", contentType)

		response := &resources.Resource{}
		_, apiErr = repo.gateway.PerformPollingRequestForJSONResponse(repo.config.APIEndpoint(), request, response, DefaultAppUploadBitsTimeout)
		if apiErr != nil {
			return
		}
	})

	return
}
func (repo CloudControllerApplicationBitsRepository) writeUploadBody(zipFile io.ReadCloser, fileSize int64, body *os.File, presentResourcesJSON []byte) (boundary string, err error) {
	writer := multipart.NewWriter(body)
	defer writer.Close()

	boundary = writer.Boundary()

	part, err := writer.CreateFormField("resources")
	if err != nil {
		return
	}
	_, err = io.Copy(part, bytes.NewBuffer(presentResourcesJSON))
	if err != nil {
		return
	}

	if zipFile == nil {
		return
	}
	part, zipErr := createZipPartWriter(fileSize, writer)
	if zipErr != nil {
		return
	}

	_, zipErr = io.Copy(part, zipFile)
	if zipErr != nil {
		return
	}
	return
}

func createZipPartWriter(fileSize int64, writer *multipart.Writer) (io.Writer, error) {
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
	h.Set("Content-Type", "application/zip")
	h.Set("Content-Length", fmt.Sprintf("%d", fileSize))
	h.Set("Content-Transfer-Encoding", "binary")
	return writer.CreatePart(h)
}

//Old way to send bits
/////
func (repo CloudControllerApplicationBitsRepository) UploadBits(appGUID string, zipFile io.ReadCloser, fileSize int64) error {
	apiURL := fmt.Sprintf("/v2/apps/%s/bits", appGUID)
	r, w := io.Pipe()
	mpw := multipart.NewWriter(w)
	go func() {
		var err error
		defer w.Close()
		part, err := mpw.CreateFormField("resources")
		if err != nil {
			panic(err)
		}
		_, err = io.Copy(part, bytes.NewBuffer([]byte("[]")))
		if err != nil {
			panic(err)
		}
		h := make(textproto.MIMEHeader)
		h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
		h.Set("Content-Type", "application/zip")
		h.Set("Content-Length", fmt.Sprintf("%d", fileSize))
		h.Set("Content-Transfer-Encoding", "binary")

		part, err = mpw.CreatePart(h)
		if err != nil {
			panic(err)
		}
		if _, err = io.Copy(part, zipFile); err != nil {
			panic(err)
		}
		mpw.Close()
	}()
	var request *net.Request
	request, err := repo.gateway.NewRequest("PUT", repo.config.APIEndpoint()+apiURL, repo.config.AccessToken(), nil)
	if err != nil {
		return err
	}
	contentType := fmt.Sprintf("multipart/form-data; boundary=%s", mpw.Boundary())
	request.HTTPReq.Header.Set("Content-Type", contentType)
	request.HTTPReq.ContentLength = int64(repo.predictPart(int64(fileSize), mpw.Boundary()))
	request.HTTPReq.Body = r

	response := &resources.Resource{}
	_, err = repo.gateway.PerformPollingRequestForJSONResponse(repo.config.APIEndpoint(), request, response, DefaultAppUploadBitsTimeout)
	if err != nil {
		panic(err)
	}
	return nil
}

func (repo CloudControllerApplicationBitsRepository) predictPart(filesize int64, boundary string) int64 {
	buf := new(bytes.Buffer)
	mpw := multipart.NewWriter(buf)

	mpw.SetBoundary(boundary)
	part, err := mpw.CreateFormField("resources")
	if err != nil {
		panic(err)
	}
	_, err = io.Copy(part, bytes.NewBuffer([]byte("[]")))
	if err != nil {
		panic(err)
	}
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
	h.Set("Content-Type", "application/zip")
	h.Set("Content-Length", fmt.Sprintf("%d", filesize))
	h.Set("Content-Transfer-Encoding", "binary")

	part, err = mpw.CreatePart(h)
	if err != nil {
		panic(err)
	}
	mpw.Close()
	b, _ := ioutil.ReadAll(buf)
	return int64(len(b)) + filesize
}
<|endoftext|>


<commit_before>/*
 * Copyright 2013 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package main

import (
	"github.com/gorilla/mux"
	"net/http"
	"fmt"
	"bufio"
)

func PrintData(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()
	fmt.Printf("%v:\n", r.URL.Path)

	reader := bufio.NewReader(r.Body)
	for {
		line, _, err := reader.ReadLine()
		if err != nil {
			return
		}
		fmt.Printf("\t%v\n", string(line))
	}
	w.WriteHeader(200)
	return
}

func main() {
	r := mux.NewRouter()
	r.HandleFunc("/auth", PrintData)
	r.HandleFunc("/msg", PrintData)
	r.HandleFunc("/fwd", PrintData)
	r.HandleFunc("/err", PrintData)
	r.HandleFunc("/login", PrintData)
	r.HandleFunc("/logout", PrintData)
	r.HandleFunc("/subscribe", PrintData)
	r.HandleFunc("/unsubscribe", PrintData)
	r.HandleFunc("/push", PrintData)
	http.Handle("/", r)
	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		fmt.Printf("Error: %v\n", err)
	}
	return
}

<commit_msg>A silly print and accept program.<commit_after>/*
 * Copyright 2013 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package main

import (
	"net/http"
	"fmt"
	"bufio"
)

func PrintData(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()
	fmt.Printf("%v:\n", r.URL.Path)

	reader := bufio.NewReader(r.Body)
	for {
		line, _, err := reader.ReadLine()
		if err != nil {
			return
		}
		fmt.Printf("\t%v\n", string(line))
	}
	w.WriteHeader(200)
	return
}

func main() {
	http.HandleFunc("/", PrintData)
	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		fmt.Printf("Error: %v\n", err)
	}
	return
}

<|endoftext|>


<commit_before>// RESTful API Controller Example.

package main

import (
	"github.com/headwindfly/clevergo"
	"html/template"
)

var (
	html = `

RESTful API Controller Example.

Requests

GET

POST

DELETE

PUT

HEAD

OPTIONS

PATCH

Result:

		<textarea rows="5" cols="100" id="result"></textarea>

	`
	tpl = template.Must(template.New("").Parse(html))
)

// accessControlMiddleware for setting Access-Control-Allow-* into response's header.
type accessControlMiddleware struct {
	origin  string
	methods string
}

// newAccessControlMiddleware returns a accessControlMiddleware's instance.
func newAccessControlMiddleware(origin, methods string) accessControlMiddleware {
	return accessControlMiddleware{
		origin:  origin,
		methods: methods,
	}
}

// Handle implemented the Middleware interface.
func (m accessControlMiddleware) Handle(next clevergo.Handler) clevergo.Handler {
	return clevergo.HandlerFunc(func(ctx *clevergo.Context) {
		// Set Access-Control-Allow-Origin and Access-Control-Allow-Methods for ajax request.
		ctx.Response.Header.Set("Access-Control-Allow-Origin", m.origin)
		ctx.Response.Header.Set("Access-Control-Allow-Methods", m.methods)

		next.Handle(ctx)
	})
}

type userController struct {
	middlewares []clevergo.Middleware // middlewares for this controller.
	handler     clevergo.Handler      // for cache the handler.
}

func newUserController(middlewares []clevergo.Middleware) userController {
	return userController{
		middlewares: middlewares,
	}
}

// getHandler is the most important method.
// It made the final handler be wrapped by the middlewares.
func (c userController) getHandler(next clevergo.Handler) clevergo.Handler {
	if c.handler == nil {
		c.handler = clevergo.HandlerFunc(c.handle(next))
		for i := len(c.middlewares) - 1; i >= 0; i-- {
			c.handler = c.middlewares[i].Handle(c.handler)
		}
	}

	return c.handler
}

// handle the final handler.
func (c userController) handle(next clevergo.Handler) clevergo.HandlerFunc {
	return func(ctx *clevergo.Context) {
		// Using param named '_method' to simulate the other request, such as PUT, DELETE etc.
		if !ctx.IsGet() {
			switch string(ctx.FormValue("_method")) {
			case "PUT":
				c.PUT(ctx)
				return
			case "DELETE":
				c.DELETE(ctx)
				return
			case "HEAD":
				c.HEAD(ctx)
				return
			case "OPTIONS":
				c.OPTIONS(ctx)
				return
			case "PATCH":
				c.PATCH(ctx)
				return
			}
		}

		next.Handle(ctx)
	}
}

// Handle implemented the Middleware interface.
func (c userController) Handle(next clevergo.Handler) clevergo.Handler {
	return c.getHandler(next)
}

func (c userController) GET(ctx *clevergo.Context) {
	ctx.Text("GET REQUEST.\n")
}

func (c userController) POST(ctx *clevergo.Context) {
	ctx.Text("POST REQUEST.\n")
}

func (c userController) DELETE(ctx *clevergo.Context) {
	ctx.Text("DELETE REQUEST.\n")
}

func (c userController) PUT(ctx *clevergo.Context) {
	ctx.Text("PUT REQUEST.\n")
}

func (c userController) OPTIONS(ctx *clevergo.Context) {
	ctx.Text("OPTIONS REQUEST.\n")
}

func (c userController) PATCH(ctx *clevergo.Context) {
	ctx.Text("PATCH REQUEST.\n")
}

func (c userController) HEAD(ctx *clevergo.Context) {
	ctx.Text("HEAD REQUEST.\n")
}

func index(ctx *clevergo.Context) {
	ctx.SetContentTypeToHTML()
	tpl.Execute(ctx, nil)
}

func main() {
	app := clevergo.NewApplication()

	// Create a router instance.
	router := app.NewRouter("")

	// Register route handler.
	router.GET("/", clevergo.HandlerFunc(index))
	router.RegisterController("/users", newUserController([]clevergo.Middleware{
		newAccessControlMiddleware("*", "GET, POST, DELETE, PUT"),
	}))

	// Start server.
	app.Run()
}
<commit_msg>Updated the restful example<commit_after>// RESTful API Controller Example.

package main

import (
	"github.com/headwindfly/clevergo"
	"html/template"
)

var (
	html = `

RESTful API Controller Example.

Requests

GET

POST

DELETE

PUT

HEAD (404 NOT FOUND)

OPTIONS (404 NOT FOUND)

PATCH (404 NOT FOUND)

Result:

		<textarea rows="5" cols="100" id="result"></textarea>

	`
	tpl = template.Must(template.New("").Parse(html))
)

// accessControlMiddleware for setting Access-Control-Allow-* into response's header.
type accessControlMiddleware struct {
	origin  string
	methods string
}

// newAccessControlMiddleware returns a accessControlMiddleware's instance.
func newAccessControlMiddleware(origin, methods string) accessControlMiddleware {
	return accessControlMiddleware{
		origin:  origin,
		methods: methods,
	}
}

// Handle implemented the Middleware interface.
func (m accessControlMiddleware) Handle(next clevergo.Handler) clevergo.Handler {
	return clevergo.HandlerFunc(func(ctx *clevergo.Context) {
		// Set Access-Control-Allow-Origin and Access-Control-Allow-Methods for ajax request.
		ctx.Response.Header.Set("Access-Control-Allow-Origin", m.origin)
		ctx.Response.Header.Set("Access-Control-Allow-Methods", m.methods)

		next.Handle(ctx)
	})
}

type userController struct {
	clevergo.Controller
}

func newUserController(middlewares []clevergo.Middleware) userController {
	return userController{
		Controller: clevergo.Controller{
			Middlewares: middlewares,
		},
	}
}

func simulate(c clevergo.ControllerInterface, next clevergo.Handler) clevergo.Handler {
	return clevergo.HandlerFunc(func(ctx *clevergo.Context) {
		// Using param named '_method' to simulate the other request, such as PUT, DELETE etc.
		if ctx.IsPost() {
			switch string(ctx.FormValue("_method")) {
			case "PUT":
				c.PUT(ctx)
				return
			case "DELETE":
				c.DELETE(ctx)
				return
			case "HEAD":
				c.HEAD(ctx)
				return
			case "OPTIONS":
				c.OPTIONS(ctx)
				return
			case "PATCH":
				c.PATCH(ctx)
				return
			}
		}

		next.Handle(ctx)
	})
}

// Handle implemented the Middleware interface.
//
// Important note： your controller have to implement the Middleware interface.
func (c userController) Handle(next clevergo.Handler) clevergo.Handler {
	return simulate(c, next)
}

func (c userController) GET(ctx *clevergo.Context) {
	ctx.Text("GET handler of userController.")
}

func (c userController) POST(ctx *clevergo.Context) {
	ctx.Text("POST handler of userController.")
}

func (c userController) DELETE(ctx *clevergo.Context) {
	ctx.Text("DELETE handler of userController.")
}

func (c userController) PUT(ctx *clevergo.Context) {
	ctx.Text("PUT handler of userController.")
}

func index(ctx *clevergo.Context) {
	ctx.SetContentTypeToHTML()
	tpl.Execute(ctx, nil)
}

func main() {
	app := clevergo.NewApplication()

	// Create a router instance.
	router := app.NewRouter("")

	// Register route handler.
	router.GET("/", clevergo.HandlerFunc(index))
	router.RegisterController("/users", newUserController([]clevergo.Middleware{
		newAccessControlMiddleware("*", "GET, POST, DELETE, PUT"),
	}))

	// Start server.
	app.Run()
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build example

package main

import (
	"fmt"
	_ "image/png"
	"log"
	"math/rand"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/ebitenutil"
)

const (
	screenWidth  = 320
	screenHeight = 240
)

var (
	ebitenImage       *ebiten.Image
	ebitenImageWidth  = 0
	ebitenImageHeight = 0
)

type Sprite struct {
	imageWidth  int
	imageHeight int
	x           int
	y           int
	vx          int
	vy          int
}

func (s *Sprite) Update() {
	s.x += s.vx
	s.y += s.vy
	if s.x < 0 {
		s.x = -s.x
		s.vx = -s.vx
	} else if screenWidth <= s.x+s.imageWidth {
		s.x = 2*(screenWidth-s.imageWidth) - s.x
		s.vx = -s.vx
	}
	if s.y < 0 {
		s.y = -s.y
		s.vy = -s.vy
	} else if screenHeight <= s.y+s.imageHeight {
		s.y = 2*(screenHeight-s.imageHeight) - s.y
		s.vy = -s.vy
	}
}

type Sprites struct {
	sprites []*Sprite
	num     int
}

func (s *Sprites) Update() {
	for _, sprite := range s.sprites {
		sprite.Update()
	}
}

const (
	MinSprites = 0
	MaxSprites = 50000
)

var sprites = &Sprites{make([]*Sprite, MaxSprites), 500}

var op *ebiten.DrawImageOptions

func init() {
	op = &ebiten.DrawImageOptions{}
	op.ColorM.Scale(1.0, 1.0, 1.0, 0.5)
}

func update(screen *ebiten.Image) error {
	if ebiten.IsKeyPressed(ebiten.KeyLeft) {
		sprites.num -= 20
		if sprites.num < MinSprites {
			sprites.num = MinSprites
		}
	}
	if ebiten.IsKeyPressed(ebiten.KeyRight) {
		sprites.num += 20
		if MaxSprites < sprites.num {
			sprites.num = MaxSprites
		}
	}
	sprites.Update()

	if ebiten.IsRunningSlowly() {
		return nil
	}
	for i := 0; i < sprites.num; i++ {
		s := sprites.sprites[i]
		op.GeoM.Reset()
		op.GeoM.Translate(float64(s.x), float64(s.y))
		screen.DrawImage(ebitenImage, op)
	}
	msg := fmt.Sprintf(`FPS: %0.2f
Num of sprites: %d
Press <- or -> to change the number of sprites`, ebiten.CurrentFPS(), sprites.num)
	if err := ebitenutil.DebugPrint(screen, msg); err != nil {
		return err
	}
	return nil
}

func main() {
	var err error
	ebitenImage, _, err = ebitenutil.NewImageFromFile("_resources/images/ebiten.png", ebiten.FilterNearest)
	if err != nil {
		log.Fatal(err)
	}
	ebitenImageWidth, ebitenImageHeight = ebitenImage.Size()
	for i := range sprites.sprites {
		w, h := ebitenImage.Size()
		x, y := rand.Intn(screenWidth-w), rand.Intn(screenHeight-h)
		vx, vy := 2*rand.Intn(2)-1, 2*rand.Intn(2)-1
		sprites.sprites[i] = &Sprite{
			imageWidth:  w,
			imageHeight: h,
			x:           x,
			y:           y,
			vx:          vx,
			vy:          vy,
		}
	}
	if err := ebiten.Run(update, screenWidth, screenHeight, 2, "Sprites (Ebiten Demo)"); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>examples/sprites: Apply ColorM change only once<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build example

package main

import (
	"fmt"
	_ "image/png"
	"log"
	"math/rand"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/ebitenutil"
)

const (
	screenWidth  = 320
	screenHeight = 240
)

var (
	ebitenImage *ebiten.Image
)

type Sprite struct {
	imageWidth  int
	imageHeight int
	x           int
	y           int
	vx          int
	vy          int
}

func (s *Sprite) Update() {
	s.x += s.vx
	s.y += s.vy
	if s.x < 0 {
		s.x = -s.x
		s.vx = -s.vx
	} else if screenWidth <= s.x+s.imageWidth {
		s.x = 2*(screenWidth-s.imageWidth) - s.x
		s.vx = -s.vx
	}
	if s.y < 0 {
		s.y = -s.y
		s.vy = -s.vy
	} else if screenHeight <= s.y+s.imageHeight {
		s.y = 2*(screenHeight-s.imageHeight) - s.y
		s.vy = -s.vy
	}
}

type Sprites struct {
	sprites []*Sprite
	num     int
}

func (s *Sprites) Update() {
	for _, sprite := range s.sprites {
		sprite.Update()
	}
}

const (
	MinSprites = 0
	MaxSprites = 50000
)

var (
	sprites = &Sprites{make([]*Sprite, MaxSprites), 500}
	op      = &ebiten.DrawImageOptions{}
)

func update(screen *ebiten.Image) error {
	if ebiten.IsKeyPressed(ebiten.KeyLeft) {
		sprites.num -= 20
		if sprites.num < MinSprites {
			sprites.num = MinSprites
		}
	}
	if ebiten.IsKeyPressed(ebiten.KeyRight) {
		sprites.num += 20
		if MaxSprites < sprites.num {
			sprites.num = MaxSprites
		}
	}
	sprites.Update()

	if ebiten.IsRunningSlowly() {
		return nil
	}
	for i := 0; i < sprites.num; i++ {
		s := sprites.sprites[i]
		op.GeoM.Reset()
		op.GeoM.Translate(float64(s.x), float64(s.y))
		screen.DrawImage(ebitenImage, op)
	}
	msg := fmt.Sprintf(`FPS: %0.2f
Num of sprites: %d
Press <- or -> to change the number of sprites`, ebiten.CurrentFPS(), sprites.num)
	if err := ebitenutil.DebugPrint(screen, msg); err != nil {
		return err
	}
	return nil
}

func main() {
	var err error
	img, _, err := ebitenutil.NewImageFromFile("_resources/images/ebiten.png", ebiten.FilterNearest)
	if err != nil {
		log.Fatal(err)
	}
	w, h := img.Size()
	ebitenImage, _ = ebiten.NewImage(w, h, ebiten.FilterNearest)
	op := &ebiten.DrawImageOptions{}
	op.ColorM.Scale(1, 1, 1, 0.5)
	ebitenImage.DrawImage(img, op)
	for i := range sprites.sprites {
		w, h := ebitenImage.Size()
		x, y := rand.Intn(screenWidth-w), rand.Intn(screenHeight-h)
		vx, vy := 2*rand.Intn(2)-1, 2*rand.Intn(2)-1
		sprites.sprites[i] = &Sprite{
			imageWidth:  w,
			imageHeight: h,
			x:           x,
			y:           y,
			vx:          vx,
			vy:          vy,
		}
	}
	if err := ebiten.Run(update, screenWidth, screenHeight, 2, "Sprites (Ebiten Demo)"); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(rsc): Document multi-change branch behavior.

// Command git-codereview provides a simple command-line user interface for
// working with git repositories and the Gerrit code review system.
// See "git-codereview help" for details.
package main // import "golang.org/x/review/git-codereview"

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var (
	flags   *flag.FlagSet
	verbose = new(count) // installed as -v below
	noRun   = new(bool)
)

func initFlags() {
	flags = flag.NewFlagSet("", flag.ExitOnError)
	flags.Usage = func() {
		fmt.Fprintf(stderr(), usage, os.Args[0], os.Args[0])
	}
	flags.Var(verbose, "v", "report commands")
	flags.BoolVar(noRun, "n", false, "print but do not run commands")
}

const globalFlags = "[-n] [-v]"

const usage = `Usage: %s <command> ` + globalFlags + `
Type "%s help" for more information.
`

const help = `Usage: %s <command> ` + globalFlags + `

The git-codereview command is a wrapper for the git command that provides a
simple interface to the "single-commit feature branch" development model.

See the docs for details: https://godoc.org/golang.org/x/review/git-codereview

The -v flag prints all commands that make changes.
The -n flag prints all commands that would be run, but does not run them.

Available commands:

	change [name]
		Create a change commit, or amend an existing change commit,
		with the staged changes. If a branch name is provided, check
		out that branch (creating it if it does not exist).
		Does not amend the existing commit when switching branches.
		If -q is specified, skip the editing of an extant pending
		change's commit message.
		If -a is specified, automatically add any unstaged changes in
		tracked files during commit.

	change NNNN[/PP]
		Checkout the commit corresponding to CL number NNNN and
		patch set PP from Gerrit.
		If the patch set is omitted, use the current patch set.

	gofmt [-l]
		Run gofmt on all tracked files in the staging area and the
		working tree.
		If -l is specified, list files that need formatting.
		Otherwise, reformat files in place.

	help
		Show this help text.

	hooks
		Install Git commit hooks for Gerrit and gofmt.
		Every other operation except help also does this,
		if they are not already installed.

	mail [-f] [-r reviewer,...] [-cc mail,...] [commit]
		Upload change commit to the code review server and send mail
		requesting a code review.
		If there are multiple commits on this branch, upload commits
		up to and including the named commit.
		If -f is specified, upload even if there are staged changes.
		The -r and -cc flags identify the email addresses of people to
		do the code review and to be CC'ed about the code review.
		Multiple addresses are given as a comma-separated list.

	mail -diff
		Show the changes but do not send mail or upload.

	pending [-c] [-l] [-s]
		Show the status of all pending changes and staged, unstaged,
		and untracked files in the local repository.
		If -c is specified, show only changes on the current branch.
		If -l is specified, only use locally available information.
		If -s is specified, show short output.

	submit [-i | commit...]
		Push the pending change to the Gerrit server and tell Gerrit to
		submit it to the master branch.

	sync
		Fetch changes from the remote repository and merge them into
		the current branch, rebasing the change commit on top of them.

`

func main() {
	initFlags()

	if len(os.Args) < 2 {
		flags.Usage()
		if dieTrap != nil {
			dieTrap()
		}
		os.Exit(2)
	}
	command, args := os.Args[1], os.Args[2:]

	if command == "help" {
		fmt.Fprintf(stdout(), help, os.Args[0])
		return
	}

	// Install hooks automatically, but only if this is a Gerrit repo.
	if haveGerrit() {
		// Don't pass installHook args directly,
		// since args might contain args meant for other commands.
		// Filter down to just global flags.
		var hookArgs []string
		for _, arg := range args {
			switch arg {
			case "-n", "-v":
				hookArgs = append(hookArgs, arg)
			}
		}
		installHook(hookArgs)
	}

	switch command {
	case "branchpoint":
		cmdBranchpoint(args)
	case "change":
		cmdChange(args)
	case "gofmt":
		cmdGofmt(args)
	case "hook-invoke":
		cmdHookInvoke(args)
	case "hooks":
		installHook(args) // in case above was bypassed
	case "mail", "m":
		cmdMail(args)
	case "pending":
		cmdPending(args)
	case "rebase-work":
		cmdRebaseWork(args)
	case "submit":
		cmdSubmit(args)
	case "sync":
		cmdSync(args)
	case "test-loadAuth": // for testing only
		loadAuth()
	default:
		flags.Usage()
	}
}

func expectZeroArgs(args []string, command string) {
	flags.Parse(args)
	if len(flags.Args()) > 0 {
		fmt.Fprintf(stderr(), "Usage: %s %s %s\n", os.Args[0], command, globalFlags)
		os.Exit(2)
	}
}

func run(command string, args ...string) {
	if err := runErr(command, args...); err != nil {
		if *verbose == 0 {
			// If we're not in verbose mode, print the command
			// before dying to give context to the failure.
			fmt.Fprintf(stderr(), "(running: %s)\n", commandString(command, args))
		}
		dief("%v", err)
	}
}

func runErr(command string, args ...string) error {
	return runDirErr("", command, args...)
}

var runLogTrap []string

func runDirErr(dir, command string, args ...string) error {
	if *verbose > 0 || *noRun {
		fmt.Fprintln(stderr(), commandString(command, args))
	}
	if *noRun {
		return nil
	}
	if runLogTrap != nil {
		runLogTrap = append(runLogTrap, strings.TrimSpace(command+" "+strings.Join(args, " ")))
	}
	cmd := exec.Command(command, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = stdout()
	cmd.Stderr = stderr()
	return cmd.Run()
}

// cmdOutput runs the command line, returning its output.
// If the command cannot be run or does not exit successfully,
// cmdOutput dies.
//
// NOTE: cmdOutput must be used only to run commands that read state,
// not for commands that make changes. Commands that make changes
// should be run using runDirErr so that the -v and -n flags apply to them.
func cmdOutput(command string, args ...string) string {
	return cmdOutputDir(".", command, args...)
}

// cmdOutputDir runs the command line in dir, returning its output.
// If the command cannot be run or does not exit successfully,
// cmdOutput dies.
//
// NOTE: cmdOutput must be used only to run commands that read state,
// not for commands that make changes. Commands that make changes
// should be run using runDirErr so that the -v and -n flags apply to them.
func cmdOutputDir(dir, command string, args ...string) string {
	s, err := cmdOutputDirErr(dir, command, args...)
	if err != nil {
		fmt.Fprintf(stderr(), "%v\n%s\n", commandString(command, args), s)
		dief("%v", err)
	}
	return s
}

// cmdOutputErr runs the command line in dir, returning its output
// and any error results.
//
// NOTE: cmdOutputErr must be used only to run commands that read state,
// not for commands that make changes. Commands that make changes
// should be run using runDirErr so that the -v and -n flags apply to them.
func cmdOutputErr(command string, args ...string) (string, error) {
	return cmdOutputDirErr(".", command, args...)
}

// cmdOutputDirErr runs the command line in dir, returning its output
// and any error results.
//
// NOTE: cmdOutputDirErr must be used only to run commands that read state,
// not for commands that make changes. Commands that make changes
// should be run using runDirErr so that the -v and -n flags apply to them.
func cmdOutputDirErr(dir, command string, args ...string) (string, error) {
	// NOTE: We only show these non-state-modifying commands with -v -v.
	// Otherwise things like 'git sync -v' show all our internal "find out about
	// the git repo" commands, which is confusing if you are just trying to find
	// out what git sync means.
	if *verbose > 1 {
		fmt.Fprintln(stderr(), commandString(command, args))
	}
	cmd := exec.Command(command, args...)
	if dir != "." {
		cmd.Dir = dir
	}
	b, err := cmd.CombinedOutput()
	return string(b), err
}

// trim is shorthand for strings.TrimSpace.
func trim(text string) string {
	return strings.TrimSpace(text)
}

// trimErr applies strings.TrimSpace to the result of cmdOutput(Dir)Err,
// passing the error along unmodified.
func trimErr(text string, err error) (string, error) {
	return strings.TrimSpace(text), err
}

// lines returns the lines in text.
func lines(text string) []string {
	out := strings.Split(text, "\n")
	// Split will include a "" after the last line. Remove it.
	if n := len(out) - 1; n >= 0 && out[n] == "" {
		out = out[:n]
	}
	return out
}

// nonBlankLines returns the non-blank lines in text.
func nonBlankLines(text string) []string {
	var out []string
	for _, s := range lines(text) {
		if strings.TrimSpace(s) != "" {
			out = append(out, s)
		}
	}
	return out
}

func commandString(command string, args []string) string {
	return strings.Join(append([]string{command}, args...), " ")
}

var dieTrap func()

func dief(format string, args ...interface{}) {
	printf(format, args...)
	die()
}

func die() {
	if dieTrap != nil {
		dieTrap()
	}
	os.Exit(1)
}

func verbosef(format string, args ...interface{}) {
	if *verbose > 0 {
		printf(format, args...)
	}
}

var stdoutTrap, stderrTrap *bytes.Buffer

func stdout() io.Writer {
	if stdoutTrap != nil {
		return stdoutTrap
	}
	return os.Stdout
}

func stderr() io.Writer {
	if stderrTrap != nil {
		return stderrTrap
	}
	return os.Stderr
}

func printf(format string, args ...interface{}) {
	fmt.Fprintf(stderr(), "%s: %s\n", os.Args[0], fmt.Sprintf(format, args...))
}

// count is a flag.Value that is like a flag.Bool and a flag.Int.
// If used as -name, it increments the count, but -name=x sets the count.
// Used for verbose flag -v.
type count int

func (c *count) String() string {
	return fmt.Sprint(int(*c))
}

func (c *count) Set(s string) error {
	switch s {
	case "true":
		*c++
	case "false":
		*c = 0
	default:
		n, err := strconv.Atoi(s)
		if err != nil {
			return fmt.Errorf("invalid count %q", s)
		}
		*c = count(n)
	}
	return nil
}

func (c *count) IsBoolFlag() bool {
	return true
}
<commit_msg>codereview: mention rebase-work in usage message<commit_after>// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(rsc): Document multi-change branch behavior.

// Command git-codereview provides a simple command-line user interface for
// working with git repositories and the Gerrit code review system.
// See "git-codereview help" for details.
package main // import "golang.org/x/review/git-codereview"

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

var (
	flags   *flag.FlagSet
	verbose = new(count) // installed as -v below
	noRun   = new(bool)
)

func initFlags() {
	flags = flag.NewFlagSet("", flag.ExitOnError)
	flags.Usage = func() {
		fmt.Fprintf(stderr(), usage, os.Args[0], os.Args[0])
	}
	flags.Var(verbose, "v", "report commands")
	flags.BoolVar(noRun, "n", false, "print but do not run commands")
}

const globalFlags = "[-n] [-v]"

const usage = `Usage: %s <command> ` + globalFlags + `
Type "%s help" for more information.
`

const help = `Usage: %s <command> ` + globalFlags + `

The git-codereview command is a wrapper for the git command that provides a
simple interface to the "single-commit feature branch" development model.

See the docs for details: https://godoc.org/golang.org/x/review/git-codereview

The -v flag prints all commands that make changes.
The -n flag prints all commands that would be run, but does not run them.

Available commands:

	change [name]
		Create a change commit, or amend an existing change commit,
		with the staged changes. If a branch name is provided, check
		out that branch (creating it if it does not exist).
		Does not amend the existing commit when switching branches.
		If -q is specified, skip the editing of an extant pending
		change's commit message.
		If -a is specified, automatically add any unstaged changes in
		tracked files during commit.

	change NNNN[/PP]
		Checkout the commit corresponding to CL number NNNN and
		patch set PP from Gerrit.
		If the patch set is omitted, use the current patch set.

	gofmt [-l]
		Run gofmt on all tracked files in the staging area and the
		working tree.
		If -l is specified, list files that need formatting.
		Otherwise, reformat files in place.

	help
		Show this help text.

	hooks
		Install Git commit hooks for Gerrit and gofmt.
		Every other operation except help also does this,
		if they are not already installed.

	mail [-f] [-r reviewer,...] [-cc mail,...] [commit]
		Upload change commit to the code review server and send mail
		requesting a code review.
		If there are multiple commits on this branch, upload commits
		up to and including the named commit.
		If -f is specified, upload even if there are staged changes.
		The -r and -cc flags identify the email addresses of people to
		do the code review and to be CC'ed about the code review.
		Multiple addresses are given as a comma-separated list.

	mail -diff
		Show the changes but do not send mail or upload.

	pending [-c] [-l] [-s]
		Show the status of all pending changes and staged, unstaged,
		and untracked files in the local repository.
		If -c is specified, show only changes on the current branch.
		If -l is specified, only use locally available information.
		If -s is specified, show short output.

	rebase-work
		Run git rebase in interactive mode over pending changes
		(shorthand for "git rebase -i $(git codereview branchpoint)").
		This rebase does not incorporate any new changes from the origin
		branch, in contrast with a normal "git rebase -i".

	submit [-i | commit...]
		Push the pending change to the Gerrit server and tell Gerrit to
		submit it to the master branch.

	sync
		Fetch changes from the remote repository and merge them into
		the current branch, rebasing the change commit on top of them.

`

func main() {
	initFlags()

	if len(os.Args) < 2 {
		flags.Usage()
		if dieTrap != nil {
			dieTrap()
		}
		os.Exit(2)
	}
	command, args := os.Args[1], os.Args[2:]

	if command == "help" {
		fmt.Fprintf(stdout(), help, os.Args[0])
		return
	}

	// Install hooks automatically, but only if this is a Gerrit repo.
	if haveGerrit() {
		// Don't pass installHook args directly,
		// since args might contain args meant for other commands.
		// Filter down to just global flags.
		var hookArgs []string
		for _, arg := range args {
			switch arg {
			case "-n", "-v":
				hookArgs = append(hookArgs, arg)
			}
		}
		installHook(hookArgs)
	}

	switch command {
	case "branchpoint":
		cmdBranchpoint(args)
	case "change":
		cmdChange(args)
	case "gofmt":
		cmdGofmt(args)
	case "hook-invoke":
		cmdHookInvoke(args)
	case "hooks":
		installHook(args) // in case above was bypassed
	case "mail", "m":
		cmdMail(args)
	case "pending":
		cmdPending(args)
	case "rebase-work":
		cmdRebaseWork(args)
	case "submit":
		cmdSubmit(args)
	case "sync":
		cmdSync(args)
	case "test-loadAuth": // for testing only
		loadAuth()
	default:
		flags.Usage()
	}
}

func expectZeroArgs(args []string, command string) {
	flags.Parse(args)
	if len(flags.Args()) > 0 {
		fmt.Fprintf(stderr(), "Usage: %s %s %s\n", os.Args[0], command, globalFlags)
		os.Exit(2)
	}
}

func run(command string, args ...string) {
	if err := runErr(command, args...); err != nil {
		if *verbose == 0 {
			// If we're not in verbose mode, print the command
			// before dying to give context to the failure.
			fmt.Fprintf(stderr(), "(running: %s)\n", commandString(command, args))
		}
		dief("%v", err)
	}
}

func runErr(command string, args ...string) error {
	return runDirErr("", command, args...)
}

var runLogTrap []string

func runDirErr(dir, command string, args ...string) error {
	if *verbose > 0 || *noRun {
		fmt.Fprintln(stderr(), commandString(command, args))
	}
	if *noRun {
		return nil
	}
	if runLogTrap != nil {
		runLogTrap = append(runLogTrap, strings.TrimSpace(command+" "+strings.Join(args, " ")))
	}
	cmd := exec.Command(command, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = stdout()
	cmd.Stderr = stderr()
	return cmd.Run()
}

// cmdOutput runs the command line, returning its output.
// If the command cannot be run or does not exit successfully,
// cmdOutput dies.
//
// NOTE: cmdOutput must be used only to run commands that read state,
// not for commands that make changes. Commands that make changes
// should be run using runDirErr so that the -v and -n flags apply to them.
func cmdOutput(command string, args ...string) string {
	return cmdOutputDir(".", command, args...)
}

// cmdOutputDir runs the command line in dir, returning its output.
// If the command cannot be run or does not exit successfully,
// cmdOutput dies.
//
// NOTE: cmdOutput must be used only to run commands that read state,
// not for commands that make changes. Commands that make changes
// should be run using runDirErr so that the -v and -n flags apply to them.
func cmdOutputDir(dir, command string, args ...string) string {
	s, err := cmdOutputDirErr(dir, command, args...)
	if err != nil {
		fmt.Fprintf(stderr(), "%v\n%s\n", commandString(command, args), s)
		dief("%v", err)
	}
	return s
}

// cmdOutputErr runs the command line in dir, returning its output
// and any error results.
//
// NOTE: cmdOutputErr must be used only to run commands that read state,
// not for commands that make changes. Commands that make changes
// should be run using runDirErr so that the -v and -n flags apply to them.
func cmdOutputErr(command string, args ...string) (string, error) {
	return cmdOutputDirErr(".", command, args...)
}

// cmdOutputDirErr runs the command line in dir, returning its output
// and any error results.
//
// NOTE: cmdOutputDirErr must be used only to run commands that read state,
// not for commands that make changes. Commands that make changes
// should be run using runDirErr so that the -v and -n flags apply to them.
func cmdOutputDirErr(dir, command string, args ...string) (string, error) {
	// NOTE: We only show these non-state-modifying commands with -v -v.
	// Otherwise things like 'git sync -v' show all our internal "find out about
	// the git repo" commands, which is confusing if you are just trying to find
	// out what git sync means.
	if *verbose > 1 {
		fmt.Fprintln(stderr(), commandString(command, args))
	}
	cmd := exec.Command(command, args...)
	if dir != "." {
		cmd.Dir = dir
	}
	b, err := cmd.CombinedOutput()
	return string(b), err
}

// trim is shorthand for strings.TrimSpace.
func trim(text string) string {
	return strings.TrimSpace(text)
}

// trimErr applies strings.TrimSpace to the result of cmdOutput(Dir)Err,
// passing the error along unmodified.
func trimErr(text string, err error) (string, error) {
	return strings.TrimSpace(text), err
}

// lines returns the lines in text.
func lines(text string) []string {
	out := strings.Split(text, "\n")
	// Split will include a "" after the last line. Remove it.
	if n := len(out) - 1; n >= 0 && out[n] == "" {
		out = out[:n]
	}
	return out
}

// nonBlankLines returns the non-blank lines in text.
func nonBlankLines(text string) []string {
	var out []string
	for _, s := range lines(text) {
		if strings.TrimSpace(s) != "" {
			out = append(out, s)
		}
	}
	return out
}

func commandString(command string, args []string) string {
	return strings.Join(append([]string{command}, args...), " ")
}

var dieTrap func()

func dief(format string, args ...interface{}) {
	printf(format, args...)
	die()
}

func die() {
	if dieTrap != nil {
		dieTrap()
	}
	os.Exit(1)
}

func verbosef(format string, args ...interface{}) {
	if *verbose > 0 {
		printf(format, args...)
	}
}

var stdoutTrap, stderrTrap *bytes.Buffer

func stdout() io.Writer {
	if stdoutTrap != nil {
		return stdoutTrap
	}
	return os.Stdout
}

func stderr() io.Writer {
	if stderrTrap != nil {
		return stderrTrap
	}
	return os.Stderr
}

func printf(format string, args ...interface{}) {
	fmt.Fprintf(stderr(), "%s: %s\n", os.Args[0], fmt.Sprintf(format, args...))
}

// count is a flag.Value that is like a flag.Bool and a flag.Int.
// If used as -name, it increments the count, but -name=x sets the count.
// Used for verbose flag -v.
type count int

func (c *count) String() string {
	return fmt.Sprint(int(*c))
}

func (c *count) Set(s string) error {
	switch s {
	case "true":
		*c++
	case "false":
		*c = 0
	default:
		n, err := strconv.Atoi(s)
		if err != nil {
			return fmt.Errorf("invalid count %q", s)
		}
		*c = count(n)
	}
	return nil
}

func (c *count) IsBoolFlag() bool {
	return true
}
<|endoftext|>


<commit_before>package glacier

import "testing"

type thTestCase struct {
	treeHash    string
	linearHash  string
	iterations  int
	dataPerIter byte
}

type mthTestCase struct {
	perPartHash string
	parts       int
	treeHash    string
}

func TestMultiTreeHasher(t *testing.T) {
	testCases := []mthTestCase{
		mthTestCase{
			perPartHash: "7d10631c4d690a8dadf9ef848c7d04307e3c1dfcbb8e0d8443dcd03f480ee9c4",
			treeHash:    "7d10631c4d690a8dadf9ef848c7d04307e3c1dfcbb8e0d8443dcd03f480ee9c4",
			parts:       1,
		},
		mthTestCase{
			perPartHash: "7d10631c4d690a8dadf9ef848c7d04307e3c1dfcbb8e0d8443dcd03f480ee9c4",
			treeHash:    "d6d88996813a8a871bc0b0a4c067bdb806bc20beeb81f365b0b7307d7dd8f103",
			parts:       2,
		},
		mthTestCase{
			perPartHash: "7d10631c4d690a8dadf9ef848c7d04307e3c1dfcbb8e0d8443dcd03f480ee9c4",
			treeHash:    "45fd17a746d04f8ade481994600f29320e181144b2aee43899cff89196fbbb6b",
			parts:       3,
		},
		mthTestCase{
			perPartHash: "7d10631c4d690a8dadf9ef848c7d04307e3c1dfcbb8e0d8443dcd03f480ee9c4",
			treeHash:    "de28773aaed0cc4ff1e41d8808f129c3ab1c98c6c8078d55fdb9ff4963cd9cad",
			parts:       4,
		},
	}

	for _, testCase := range testCases {
		mth := MultiTreeHasher{}
		for i := 0; i < testCase.parts; i++ {
			mth.Add(testCase.perPartHash)
		}
		computed := mth.CreateHash()
		if testCase.treeHash != computed {
			t.Errorf("Expected tree hash %s; got %s", testCase.treeHash, computed)
		}
	}
}

func TestTreeHash(t *testing.T) {
	th := NewTreeHash()
	th.Write([]byte("Hello World"))
	th.Close()
	treeHash := "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e"
	linearHash := "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e"
	if treeHash != toHex(th.TreeHash()) {
		t.Fatalf("Expected treehash %s; got %s", treeHash, toHex(th.TreeHash()))
	}
	if linearHash != toHex(th.Hash()) {
		t.Fatalf("Expected linear hash %s; got %s", linearHash, toHex(th.Hash()))
	}

	testCases := []thTestCase{
		thTestCase{
			treeHash:    "9bc1b2a288b26af7257a36277ae3816a7d4f16e89c1e7e77d0a5c48bad62b360",
			linearHash:  "9bc1b2a288b26af7257a36277ae3816a7d4f16e89c1e7e77d0a5c48bad62b360",
			iterations:  1 << 20,
			dataPerIter: 'a',
		},
		thTestCase{
			treeHash:    "560c2c9333c719cb00cfdffee3ba293db17f58743cdd1f7e4055373ae6300afa",
			linearHash:  "5256ec18f11624025905d057d6befb03d77b243511ac5f77ed5e0221ce6d84b5",
			iterations:  2 << 20,
			dataPerIter: 'a',
		},
		thTestCase{
			treeHash:    "70239f4f2ead7561f69d48b956b547edef52a1280a93c262c0b582190be7db17",
			linearHash:  "6f850bc94ae6f7de14297c01616c36d712d22864497b28a63b81d776b035e656",
			iterations:  3 << 20,
			dataPerIter: 'a',
		},
		thTestCase{
			treeHash:    "9491cb2ed1d4e7cd53215f4017c23ec4ad21d7050a1e6bb636c4f67e8cddb844",
			linearHash:  "299285fc41a44cdb038b9fdaf494c76ca9d0c866672b2b266c1a0c17dda60a05",
			iterations:  4 << 20,
			dataPerIter: 'a',
		},
		thTestCase{
			treeHash:    "daede4eb580f914dacd5e0bdf7015c937fd615c1e6c6552d25cb04a8b7219828",
			linearHash:  "34c8bdd269f89a091cf17d5d23503940e0abf61c4b6544e42854b9af437f31bb",
			iterations:  3<<20 + 1<<19,
			dataPerIter: 'a',
		},
	}

	th = NewTreeHash()
	for _, testCase := range testCases {
		th.Reset()
		data := make([]byte, testCase.iterations)
		for i := range data {
			data[i] = testCase.dataPerIter
		}
		n, err := th.Write(data)
		if err != nil {
			t.Fatal(err)
		}
		if n != testCase.iterations {
			t.Fatalf("treehash wrote %d, should have written %d", n, testCase.iterations)
		}
		th.Close()
		if result := toHex(th.TreeHash()); testCase.treeHash != result {
			t.Fatal("tree hash, wanted:", testCase.treeHash, "got:", result)
		}
		if result := toHex(th.Hash()); testCase.linearHash != result {
			t.Fatal("hash of entire file, wanted:", testCase.linearHash, "got:", result)
		}
	}
}

func TestTreeHashCloseEmpty(t *testing.T) {
	th := NewTreeHash()
	err := th.Close()
	if err != nil {
		t.Error(err)
	}
}

func BenchmarkTreeHash(b *testing.B) {
	b.StopTimer()
	data := make([]byte, 1024)
	for i := range data {
		data[i] = 'a'
	}
	th := NewTreeHash()
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		n, err := th.Write(data)
		if err != nil {
			b.Fatal(err)
		}
		b.SetBytes(int64(n))
	}
	th.Close()
}
<commit_msg>Add benchmark for closing tree hash.<commit_after>package glacier

import (
	"crypto/sha256"
	"testing"
)

type thTestCase struct {
	treeHash    string
	linearHash  string
	iterations  int
	dataPerIter byte
}

type mthTestCase struct {
	perPartHash string
	parts       int
	treeHash    string
}

func TestMultiTreeHasher(t *testing.T) {
	testCases := []mthTestCase{
		mthTestCase{
			perPartHash: "7d10631c4d690a8dadf9ef848c7d04307e3c1dfcbb8e0d8443dcd03f480ee9c4",
			treeHash:    "7d10631c4d690a8dadf9ef848c7d04307e3c1dfcbb8e0d8443dcd03f480ee9c4",
			parts:       1,
		},
		mthTestCase{
			perPartHash: "7d10631c4d690a8dadf9ef848c7d04307e3c1dfcbb8e0d8443dcd03f480ee9c4",
			treeHash:    "d6d88996813a8a871bc0b0a4c067bdb806bc20beeb81f365b0b7307d7dd8f103",
			parts:       2,
		},
		mthTestCase{
			perPartHash: "7d10631c4d690a8dadf9ef848c7d04307e3c1dfcbb8e0d8443dcd03f480ee9c4",
			treeHash:    "45fd17a746d04f8ade481994600f29320e181144b2aee43899cff89196fbbb6b",
			parts:       3,
		},
		mthTestCase{
			perPartHash: "7d10631c4d690a8dadf9ef848c7d04307e3c1dfcbb8e0d8443dcd03f480ee9c4",
			treeHash:    "de28773aaed0cc4ff1e41d8808f129c3ab1c98c6c8078d55fdb9ff4963cd9cad",
			parts:       4,
		},
	}

	for _, testCase := range testCases {
		mth := MultiTreeHasher{}
		for i := 0; i < testCase.parts; i++ {
			mth.Add(testCase.perPartHash)
		}
		computed := mth.CreateHash()
		if testCase.treeHash != computed {
			t.Errorf("Expected tree hash %s; got %s", testCase.treeHash, computed)
		}
	}
}

func TestTreeHash(t *testing.T) {
	th := NewTreeHash()
	th.Write([]byte("Hello World"))
	th.Close()
	treeHash := "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e"
	linearHash := "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e"
	if treeHash != toHex(th.TreeHash()) {
		t.Fatalf("Expected treehash %s; got %s", treeHash, toHex(th.TreeHash()))
	}
	if linearHash != toHex(th.Hash()) {
		t.Fatalf("Expected linear hash %s; got %s", linearHash, toHex(th.Hash()))
	}

	testCases := []thTestCase{
		thTestCase{
			treeHash:    "9bc1b2a288b26af7257a36277ae3816a7d4f16e89c1e7e77d0a5c48bad62b360",
			linearHash:  "9bc1b2a288b26af7257a36277ae3816a7d4f16e89c1e7e77d0a5c48bad62b360",
			iterations:  1 << 20,
			dataPerIter: 'a',
		},
		thTestCase{
			treeHash:    "560c2c9333c719cb00cfdffee3ba293db17f58743cdd1f7e4055373ae6300afa",
			linearHash:  "5256ec18f11624025905d057d6befb03d77b243511ac5f77ed5e0221ce6d84b5",
			iterations:  2 << 20,
			dataPerIter: 'a',
		},
		thTestCase{
			treeHash:    "70239f4f2ead7561f69d48b956b547edef52a1280a93c262c0b582190be7db17",
			linearHash:  "6f850bc94ae6f7de14297c01616c36d712d22864497b28a63b81d776b035e656",
			iterations:  3 << 20,
			dataPerIter: 'a',
		},
		thTestCase{
			treeHash:    "9491cb2ed1d4e7cd53215f4017c23ec4ad21d7050a1e6bb636c4f67e8cddb844",
			linearHash:  "299285fc41a44cdb038b9fdaf494c76ca9d0c866672b2b266c1a0c17dda60a05",
			iterations:  4 << 20,
			dataPerIter: 'a',
		},
		thTestCase{
			treeHash:    "daede4eb580f914dacd5e0bdf7015c937fd615c1e6c6552d25cb04a8b7219828",
			linearHash:  "34c8bdd269f89a091cf17d5d23503940e0abf61c4b6544e42854b9af437f31bb",
			iterations:  3<<20 + 1<<19,
			dataPerIter: 'a',
		},
	}

	th = NewTreeHash()
	for _, testCase := range testCases {
		th.Reset()
		data := make([]byte, testCase.iterations)
		for i := range data {
			data[i] = testCase.dataPerIter
		}
		n, err := th.Write(data)
		if err != nil {
			t.Fatal(err)
		}
		if n != testCase.iterations {
			t.Fatalf("treehash wrote %d, should have written %d", n, testCase.iterations)
		}
		th.Close()
		if result := toHex(th.TreeHash()); testCase.treeHash != result {
			t.Fatal("tree hash, wanted:", testCase.treeHash, "got:", result)
		}
		if result := toHex(th.Hash()); testCase.linearHash != result {
			t.Fatal("hash of entire file, wanted:", testCase.linearHash, "got:", result)
		}
	}
}

func TestTreeHashCloseEmpty(t *testing.T) {
	th := NewTreeHash()
	err := th.Close()
	if err != nil {
		t.Error(err)
	}
}

func BenchmarkTreeHash(b *testing.B) {
	b.StopTimer()
	data := make([]byte, 1024)
	for i := range data {
		data[i] = 'a'
	}
	th := NewTreeHash()
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		n, err := th.Write(data)
		if err != nil {
			b.Fatal(err)
		}
		b.SetBytes(int64(n))
	}
	th.Close()
}

func BenchmarkTreeHashClose(b *testing.B) {
	nodes := make([][sha256.Size]byte, 4)
	for i := range nodes {
		nodes[i] = sha256.Sum256([]byte{byte(i)})
	}
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		treeHash(nodes)
	}
}
<|endoftext|>


<commit_before>package _2_sorts

func QuickSort(arr []int) {
	arrLen := len(arr)
	if arrLen <= 1 {
		return
	}
	quickSort(arr, 0, arrLen-1)
}

func quickSort(arr []int, start, end int) {
	if start >= end {
		return
	}

	pivot := partition(arr, start, end)
	quickSort(arr, start, pivot)
	quickSort(arr, pivot+1, end)
}

func partition(arr []int, low, high int) int {
	pivotV := arr[low]
	for low < high {
		for low < high && arr[high] > pivotV { //指针从右边开始向右找到一个比pivot小的数
			high--
		}
		arr[low] = arr[high] //将这个数放到low位，注意第一次这个位置放的是pivot值，所以不会丢

		for low < high && arr[low] < pivotV { //指针从左边开始向右找到第一个比pivot大的数
			low++
		}
		arr[high] = arr[low] //将这个数赋值给之前的high指针，因为之前high指针指向的数已经被一定，所以不会丢
	}

	//最后将pivot的值放入合适位置，此时low与high相等
	arr[low] = pivotV
	return low
}
<commit_msg>fix (golang-quicksort) : Fix the quicksort algorithm for golang version<commit_after>package _2_sorts

// QuickSort is quicksort methods for golang
func QuickSort(arr []int) {
	separateSort(arr, 0, len(arr)-1)
}

func separateSort(arr []int, start, end int) {
	if start >= end {
		return
	}
	i := partition(arr, start, end)
	separateSort(arr, start, i-1)
	separateSort(arr, i+1, end)
}

func partition(arr []int, start, end int) int {
	// 选取最后一位当对比数字
	pivot := arr[end]

	var i = start
	for j := start; j < end; j++ {
		if arr[j] < pivot {
			if !(i == j) {
				// 交换位置
				arr[i], arr[j] = arr[j], arr[i]
			}
			i++
		}
	}

	arr[i], arr[end] = arr[end], arr[i]

	return i
}
<|endoftext|>


<commit_before>package engine

import (
	"testing"

	"golang.org/x/crypto/nacl/box"

	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
)

// Test that SignED25519() signs the given message with the device
// signing key, and that the signature is verifiable by the returned
// public key.
//
// (For general tests that valid signatures are accepted and invalid
// signatures are rejected, see naclwrap_test.go.)
func TestCryptoSignED25519(t *testing.T) {
	tc := SetupEngineTest(t, "crypto")
	defer tc.Cleanup()

	u := CreateAndSignupFakeUser(tc, "fu")
	secretUI := &libkb.TestSecretUI{Passphrase: u.Passphrase}

	msg := []byte("test message")
	ret, err := SignED25519(tc.G, secretUI, keybase1.SignED25519Arg{
		Msg: msg,
	})
	if err != nil {
		t.Fatal(err)
	}

	publicKey := libkb.NaclSigningKeyPublic(ret.PublicKey)
	if !publicKey.Verify(msg, (*libkb.NaclSignature)(&ret.Sig)) {
		t.Error(libkb.VerificationError{})
	}
}

// Test that CryptoHandler.SignED25519() propagates any error
// encountered when getting the device signing key.
func TestCryptoSignED25519NoSigningKey(t *testing.T) {
	tc := SetupEngineTest(t, "crypto")
	defer tc.Cleanup()

	secretUI := &libkb.TestSecretUI{}
	_, err := SignED25519(tc.G, secretUI, keybase1.SignED25519Arg{
		Msg: []byte("test message"),
	})

	if _, ok := err.(libkb.SelfNotFoundError); !ok {
		t.Errorf("expected SelfNotFoundError, got %v", err)
	}
}

func BenchmarkCryptoSignED25519(b *testing.B) {
	tc := SetupEngineTest(b, "crypto")
	defer tc.Cleanup()

	u := CreateAndSignupFakeUser(tc, "fu")
	secretUI := &libkb.TestSecretUI{Passphrase: u.Passphrase}

	for i := 0; i < b.N; i++ {
		msg := []byte("test message")
		_, err := SignED25519(tc.G, secretUI, keybase1.SignED25519Arg{
			Msg: msg,
		})
		if err != nil {
			b.Fatal(err)
		}
	}
}

// Test that CryptoHandler.UnboxBytes32() decrypts a boxed 32-byte
// array correctly.
func TestCryptoUnboxBytes32(t *testing.T) {
	tc := SetupEngineTest(t, "crypto")
	defer tc.Cleanup()

	u := CreateAndSignupFakeUser(tc, "fu")
	secretUI := &libkb.TestSecretUI{Passphrase: u.Passphrase}

	key, err := getMySecretKey(
		tc.G, secretUI, libkb.DeviceEncryptionKeyType, "test")
	if err != nil {
		t.Fatal(err)
	}
	kp, ok := key.(libkb.NaclDHKeyPair)
	if !ok || kp.Private == nil {
		t.Fatalf("unexpected key %v", key)
	}

	peerKp, err := libkb.GenerateNaclDHKeyPair()
	if err != nil {
		t.Fatal(err)
	}

	expectedBytes32 := keybase1.Bytes32{0, 1, 2, 3, 4, 5}
	nonce := [24]byte{6, 7, 8, 9, 10}
	peersPublicKey := keybase1.BoxPublicKey(peerKp.Public)

	encryptedData := box.Seal(nil, expectedBytes32[:], &nonce, (*[32]byte)(&kp.Public), (*[32]byte)(peerKp.Private))

	var encryptedBytes32 keybase1.EncryptedBytes32
	if len(encryptedBytes32) != len(encryptedData) {
		t.Fatalf("Expected %d bytes, got %d", len(encryptedBytes32), len(encryptedData))
	}

	copy(encryptedBytes32[:], encryptedData)

	bytes32, err := UnboxBytes32(tc.G, secretUI, keybase1.UnboxBytes32Arg{
		EncryptedBytes32: encryptedBytes32,
		Nonce:            nonce,
		PeersPublicKey:   peersPublicKey,
	})

	if err != nil {
		t.Fatal(err)
	}

	if bytes32 != expectedBytes32 {
		t.Errorf("expected %s, got %s", expectedBytes32, bytes32)
	}
}

// Test that CryptoHandler.UnboxBytes32() propagates any decryption
// errors correctly.
//
// For now, we're assuming that nacl/box works correctly (i.e., we're
// not testing the ways in which decryption can fail).
func TestCryptoUnboxBytes32DecryptionError(t *testing.T) {
	tc := SetupEngineTest(t, "crypto")
	defer tc.Cleanup()

	u := CreateAndSignupFakeUser(tc, "fu")
	secretUI := &libkb.TestSecretUI{Passphrase: u.Passphrase}

	_, err := UnboxBytes32(tc.G, secretUI, keybase1.UnboxBytes32Arg{})
	if err != (libkb.DecryptionError{}) {
		t.Errorf("expected nil, got %v", err)
	}
}

// Test that CryptoHandler.UnboxBytes32() propagates any error
// encountered when getting the device encryption key.
func TestCryptoUnboxBytes32NoEncryptionKey(t *testing.T) {
	tc := SetupEngineTest(t, "crypto")
	defer tc.Cleanup()

	secretUI := &libkb.TestSecretUI{}
	_, err := UnboxBytes32(tc.G, secretUI, keybase1.UnboxBytes32Arg{})

	if _, ok := err.(libkb.SelfNotFoundError); !ok {
		t.Errorf("expected SelfNotFoundError, got %v", err)
	}
}

func cachedSecretKey(tc libkb.TestContext, ktype libkb.SecretKeyType) (key libkb.GenericKey, err error) {
	aerr := tc.G.LoginState().Account(func(a *libkb.Account) {
		key, err = a.CachedSecretKey(libkb.SecretKeyArg{KeyType: ktype})
	}, "cachedSecretKey")

	if aerr != nil {
		return nil, aerr
	}
	return key, err
}

func assertCachedSecretKey(tc libkb.TestContext, ktype libkb.SecretKeyType) {
	skey, err := cachedSecretKey(tc, ktype)
	if err != nil {
		tc.T.Fatalf("error getting cached secret key: %s", err)
	}
	if skey == nil {
		tc.T.Fatalf("expected cached key, got nil")
	}
}

func assertNotCachedSecretKey(tc libkb.TestContext, ktype libkb.SecretKeyType) {
	skey, err := cachedSecretKey(tc, ktype)
	if err == nil {
		tc.T.Fatal("expected err getting cached secret key, got nil")
	}
	if _, notFound := err.(libkb.NotFoundError); !notFound {
		tc.T.Fatalf("expected not found error, got %s (%T)", err, err)
	}
	if skey != nil {
		tc.T.Fatalf("expected nil cached key, got %v", skey)
	}
}

// TestCachedSecretKey tests that secret device keys are cached
// properly.
func TestCachedSecretKey(t *testing.T) {
	tc := SetupEngineTest(t, "login")
	defer tc.Cleanup()

	u := CreateAndSignupFakeUser(tc, "login")

	assertNotCachedSecretKey(tc, libkb.DeviceSigningKeyType)
	assertNotCachedSecretKey(tc, libkb.DeviceEncryptionKeyType)

	msg := []byte("test message")
	_, err := SignED25519(tc.G, u.NewSecretUI(), keybase1.SignED25519Arg{
		Msg: msg,
	})
	if err != nil {
		t.Fatal(err)
	}

	assertCachedSecretKey(tc, libkb.DeviceSigningKeyType)
	assertNotCachedSecretKey(tc, libkb.DeviceEncryptionKeyType)

	Logout(tc)
	u.LoginOrBust(tc)

	// login caches this...
	assertCachedSecretKey(tc, libkb.DeviceSigningKeyType)
	assertNotCachedSecretKey(tc, libkb.DeviceEncryptionKeyType)
}
<commit_msg>Reset benchmark timer<commit_after>package engine

import (
	"testing"

	"golang.org/x/crypto/nacl/box"

	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
)

// Test that SignED25519() signs the given message with the device
// signing key, and that the signature is verifiable by the returned
// public key.
//
// (For general tests that valid signatures are accepted and invalid
// signatures are rejected, see naclwrap_test.go.)
func TestCryptoSignED25519(t *testing.T) {
	tc := SetupEngineTest(t, "crypto")
	defer tc.Cleanup()

	u := CreateAndSignupFakeUser(tc, "fu")
	secretUI := &libkb.TestSecretUI{Passphrase: u.Passphrase}

	msg := []byte("test message")
	ret, err := SignED25519(tc.G, secretUI, keybase1.SignED25519Arg{
		Msg: msg,
	})
	if err != nil {
		t.Fatal(err)
	}

	publicKey := libkb.NaclSigningKeyPublic(ret.PublicKey)
	if !publicKey.Verify(msg, (*libkb.NaclSignature)(&ret.Sig)) {
		t.Error(libkb.VerificationError{})
	}
}

// Test that CryptoHandler.SignED25519() propagates any error
// encountered when getting the device signing key.
func TestCryptoSignED25519NoSigningKey(t *testing.T) {
	tc := SetupEngineTest(t, "crypto")
	defer tc.Cleanup()

	secretUI := &libkb.TestSecretUI{}
	_, err := SignED25519(tc.G, secretUI, keybase1.SignED25519Arg{
		Msg: []byte("test message"),
	})

	if _, ok := err.(libkb.SelfNotFoundError); !ok {
		t.Errorf("expected SelfNotFoundError, got %v", err)
	}
}

func BenchmarkCryptoSignED25519(b *testing.B) {
	tc := SetupEngineTest(b, "crypto")
	defer tc.Cleanup()

	u := CreateAndSignupFakeUser(tc, "fu")
	secretUI := u.NewSecretUI()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		msg := []byte("test message")
		_, err := SignED25519(tc.G, secretUI, keybase1.SignED25519Arg{
			Msg: msg,
		})
		if err != nil {
			b.Fatal(err)
		}
	}
}

// Test that CryptoHandler.UnboxBytes32() decrypts a boxed 32-byte
// array correctly.
func TestCryptoUnboxBytes32(t *testing.T) {
	tc := SetupEngineTest(t, "crypto")
	defer tc.Cleanup()

	u := CreateAndSignupFakeUser(tc, "fu")
	secretUI := &libkb.TestSecretUI{Passphrase: u.Passphrase}

	key, err := getMySecretKey(
		tc.G, secretUI, libkb.DeviceEncryptionKeyType, "test")
	if err != nil {
		t.Fatal(err)
	}
	kp, ok := key.(libkb.NaclDHKeyPair)
	if !ok || kp.Private == nil {
		t.Fatalf("unexpected key %v", key)
	}

	peerKp, err := libkb.GenerateNaclDHKeyPair()
	if err != nil {
		t.Fatal(err)
	}

	expectedBytes32 := keybase1.Bytes32{0, 1, 2, 3, 4, 5}
	nonce := [24]byte{6, 7, 8, 9, 10}
	peersPublicKey := keybase1.BoxPublicKey(peerKp.Public)

	encryptedData := box.Seal(nil, expectedBytes32[:], &nonce, (*[32]byte)(&kp.Public), (*[32]byte)(peerKp.Private))

	var encryptedBytes32 keybase1.EncryptedBytes32
	if len(encryptedBytes32) != len(encryptedData) {
		t.Fatalf("Expected %d bytes, got %d", len(encryptedBytes32), len(encryptedData))
	}

	copy(encryptedBytes32[:], encryptedData)

	bytes32, err := UnboxBytes32(tc.G, secretUI, keybase1.UnboxBytes32Arg{
		EncryptedBytes32: encryptedBytes32,
		Nonce:            nonce,
		PeersPublicKey:   peersPublicKey,
	})

	if err != nil {
		t.Fatal(err)
	}

	if bytes32 != expectedBytes32 {
		t.Errorf("expected %s, got %s", expectedBytes32, bytes32)
	}
}

// Test that CryptoHandler.UnboxBytes32() propagates any decryption
// errors correctly.
//
// For now, we're assuming that nacl/box works correctly (i.e., we're
// not testing the ways in which decryption can fail).
func TestCryptoUnboxBytes32DecryptionError(t *testing.T) {
	tc := SetupEngineTest(t, "crypto")
	defer tc.Cleanup()

	u := CreateAndSignupFakeUser(tc, "fu")
	secretUI := &libkb.TestSecretUI{Passphrase: u.Passphrase}

	_, err := UnboxBytes32(tc.G, secretUI, keybase1.UnboxBytes32Arg{})
	if err != (libkb.DecryptionError{}) {
		t.Errorf("expected nil, got %v", err)
	}
}

// Test that CryptoHandler.UnboxBytes32() propagates any error
// encountered when getting the device encryption key.
func TestCryptoUnboxBytes32NoEncryptionKey(t *testing.T) {
	tc := SetupEngineTest(t, "crypto")
	defer tc.Cleanup()

	secretUI := &libkb.TestSecretUI{}
	_, err := UnboxBytes32(tc.G, secretUI, keybase1.UnboxBytes32Arg{})

	if _, ok := err.(libkb.SelfNotFoundError); !ok {
		t.Errorf("expected SelfNotFoundError, got %v", err)
	}
}

func cachedSecretKey(tc libkb.TestContext, ktype libkb.SecretKeyType) (key libkb.GenericKey, err error) {
	aerr := tc.G.LoginState().Account(func(a *libkb.Account) {
		key, err = a.CachedSecretKey(libkb.SecretKeyArg{KeyType: ktype})
	}, "cachedSecretKey")

	if aerr != nil {
		return nil, aerr
	}
	return key, err
}

func assertCachedSecretKey(tc libkb.TestContext, ktype libkb.SecretKeyType) {
	skey, err := cachedSecretKey(tc, ktype)
	if err != nil {
		tc.T.Fatalf("error getting cached secret key: %s", err)
	}
	if skey == nil {
		tc.T.Fatalf("expected cached key, got nil")
	}
}

func assertNotCachedSecretKey(tc libkb.TestContext, ktype libkb.SecretKeyType) {
	skey, err := cachedSecretKey(tc, ktype)
	if err == nil {
		tc.T.Fatal("expected err getting cached secret key, got nil")
	}
	if _, notFound := err.(libkb.NotFoundError); !notFound {
		tc.T.Fatalf("expected not found error, got %s (%T)", err, err)
	}
	if skey != nil {
		tc.T.Fatalf("expected nil cached key, got %v", skey)
	}
}

// TestCachedSecretKey tests that secret device keys are cached
// properly.
func TestCachedSecretKey(t *testing.T) {
	tc := SetupEngineTest(t, "login")
	defer tc.Cleanup()

	u := CreateAndSignupFakeUser(tc, "login")

	assertNotCachedSecretKey(tc, libkb.DeviceSigningKeyType)
	assertNotCachedSecretKey(tc, libkb.DeviceEncryptionKeyType)

	msg := []byte("test message")
	_, err := SignED25519(tc.G, u.NewSecretUI(), keybase1.SignED25519Arg{
		Msg: msg,
	})
	if err != nil {
		t.Fatal(err)
	}

	assertCachedSecretKey(tc, libkb.DeviceSigningKeyType)
	assertNotCachedSecretKey(tc, libkb.DeviceEncryptionKeyType)

	Logout(tc)
	u.LoginOrBust(tc)

	// login caches this...
	assertCachedSecretKey(tc, libkb.DeviceSigningKeyType)
	assertNotCachedSecretKey(tc, libkb.DeviceEncryptionKeyType)
}
<|endoftext|>


<commit_before>/**
* @see https://github.com/Shopify/sarama/blob/master/tools/kafka-console-consumer/kafka-console-consumer.go
 */
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"

	"bytes"
	"encoding/json"
	"github.com/Shopify/sarama"
	"github.com/hidu/go-speed"
	"github.com/hidu/goutils/object"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

var (
	brokerList = flag.String("brokers", os.Getenv("KAFKA_PEERS"), "The comma separated list of brokers in the Kafka cluster")
	topic      = flag.String("topic", "", "REQUIRED: the topic to consume")
	partitions = flag.String("partitions", "all", "The partitions to consume, can be 'all' or comma-separated numbers")
	offset     = flag.String("offset", "newest", "The offset to start with. Can be `oldest`, `newest`")
	verbose    = flag.Bool("verbose", false, "Whether to turn on sarama logging")
	//	bufferSize = flag.Int("buffer-size", 256, "The buffer size of the message channel.")
	httpConsumerUrl      = flag.String("http-con-url", "", "http consumer url")
	httpConsumerTimeout  = flag.Int("http-con-timeout", 10000, "http consumer timeout,ms")
	httpConsumerReTryNum = flag.Int("rt", 3, "http consumer retry times")
	httpConsumerJsonSuc  = flag.Bool("http-check-json", true, `check http consumer response json {"errno":0}`)

	con = flag.Int("con", 10, "Concurrent Num")

	logger = log.New(os.Stderr, "", log.LstdFlags)
)

const User_Agent = "go-kafka-pusher/0.1/hidu"

var speedData *speed.Speed

func main() {
	flag.Parse()

	if *brokerList == "" {
		printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.")
	}

	if *topic == "" {
		printUsageErrorAndExit("-topic is required")
	}
	if *httpConsumerUrl == "" {
		printUsageErrorAndExit("-http-con-url is required")
	}

	_, err := url.Parse(*httpConsumerUrl)
	if err != nil {
		printUsageErrorAndExit("invalid consumer url")
	}

	if *verbose {
		sarama.Logger = logger
	}

	speedData = speed.NewSpeed("call", 5, func(msg string, sp *speed.Speed) {
		logger.Println("[speed]", msg)
	})

	var initialOffset int64
	switch *offset {
	case "oldest":
		initialOffset = sarama.OffsetOldest
	case "newest":
		initialOffset = sarama.OffsetNewest
	default:
		printUsageErrorAndExit("-offset should be `oldest` or `newest`")
	}

	c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), nil)
	if err != nil {
		printErrorAndExit(69, "Failed to start consumer: %s", err)
	}

	partitionList, err := getPartitions(c)
	if err != nil {
		printErrorAndExit(69, "Failed to get the list of partitions: %s", err)
	}

	var (
		messages = make(chan *sarama.ConsumerMessage, *con)
		closing  = make(chan struct{})
		wg       sync.WaitGroup
	)

	go func() {
		signals := make(chan os.Signal, 1)
		signal.Notify(signals, os.Kill, os.Interrupt)
		<-signals
		logger.Println("Initiating shutdown of consumer...")
		close(closing)
	}()

	for _, partition := range partitionList {
		pc, err := c.ConsumePartition(*topic, partition, initialOffset)
		if err != nil {
			printErrorAndExit(69, "Failed to start consumer for partition %d: %s", partition, err)
		}

		go func(pc sarama.PartitionConsumer) {
			<-closing
			pc.AsyncClose()
		}(pc)

		wg.Add(1)
		go func(pc sarama.PartitionConsumer) {
			defer wg.Done()
			for message := range pc.Messages() {
				messages <- message
			}
		}(pc)
	}

	for num := 0; num < *con; num++ {
		go func() {
			client := &http.Client{
				Timeout: time.Duration(*httpConsumerTimeout) * time.Millisecond,
			}
			for msg := range messages {
				subNum := 0
				for i := 0; i < *httpConsumerReTryNum; i++ {
					if dealMessage(client, msg, i) {
						subNum = 1
						break
					}
				}
				speedData.Inc(1, len(msg.Value), subNum)
			}
		}()
	}

	wg.Wait()
	logger.Println("Done consuming topic", *topic)
	close(messages)
	speedData.Stop()

	if err := c.Close(); err != nil {
		logger.Println("Failed to close consumer: ", err)
	}
}

func dealMessage(client *http.Client, msg *sarama.ConsumerMessage, try int) bool {
	val := string(msg.Value)
	key := string(msg.Key)
	logData := []string{
		"info",
		fmt.Sprintf("try=%d", try),
		"is_suc=failed",
		fmt.Sprintf("offset=%d", msg.Offset),
		fmt.Sprintf("partition=%d", msg.Partition),
		fmt.Sprintf("key=%s", url.QueryEscape(key)),
		fmt.Sprintf("value_len=%d", len(msg.Value)),
	}
	defer (func() {
		logger.Println(logData)
	})()

	if *verbose {
		fmt.Printf("Key:\t%s\n", key)
		fmt.Printf("Value:\t%s\n", val)
		fmt.Println()
	}
	vs := make(url.Values)
	vs.Add("kafka_key", key)
	vs.Add("kafka_value", val)
	vs.Add("kafka_partition", fmt.Sprintf("%d", msg.Partition))
	vs.Add("kafka_offset", fmt.Sprintf("%d", msg.Offset))

	start := time.Now()

	req, reqErr := http.NewRequest("POST", *httpConsumerUrl, bytes.NewReader([]byte(vs.Encode())))
	if reqErr != nil {
		logData[0] = "error"
		logData = append(logData, "http_build_req_error:", reqErr.Error())
		return false
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("User-Agent", User_Agent)

	resp, err := client.Do(req)

	used := time.Now().Sub(start)

	logData = append(logData, fmt.Sprintf("used_ms=%d", used.Nanoseconds()/1e6))
	if err != nil {
		logData[0] = "error"
		logData = append(logData, "http_client_error:", err.Error())
		return false
	}
	defer resp.Body.Close()
	bd, respErr := ioutil.ReadAll(resp.Body)

	if respErr != nil {
		logData[0] = "error"
		logData = append(logData, "http_read_resp_error:", respErr.Error())
		return false
	}
	logData = append(logData, fmt.Sprintf("http_code=%d", resp.StatusCode))

	isSuc := true
	if resp.StatusCode != http.StatusOK {
		isSuc = false
	}

	if isSuc && *httpConsumerJsonSuc {
		var obj interface{}
		jsonErr := json.Unmarshal(bd, &obj)
		if jsonErr != nil {
			logData = append(logData, "resp_errno=unknow")
			logData = append(logData, "resp_json_err:", jsonErr.Error())
			isSuc = false
		} else {
			errno, _ := object.NewInterfaceWalker(obj).GetString("/errno")
			logData = append(logData, "resp_errno="+errno)
			isSuc = errno == "0"
		}
	}

	if isSuc {
		logData[2] = "is_suc=suc"
	}

	logData = append(logData, fmt.Sprintf("resp_len=%d", len(bd)), fmt.Sprintf("resp=%s", url.QueryEscape(string(bd))))

	return isSuc
}

func getPartitions(c sarama.Consumer) ([]int32, error) {
	if *partitions == "all" {
		return c.Partitions(*topic)
	}

	tmp := strings.Split(*partitions, ",")
	var pList []int32
	for i := range tmp {
		val, err := strconv.ParseInt(tmp[i], 10, 32)
		if err != nil {
			return nil, err
		}
		pList = append(pList, int32(val))
	}

	return pList, nil
}

func printErrorAndExit(code int, format string, values ...interface{}) {
	fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
	fmt.Fprintln(os.Stderr)
	os.Exit(code)
}

func printUsageErrorAndExit(format string, values ...interface{}) {
	fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
	fmt.Fprintln(os.Stderr)
	fmt.Fprintln(os.Stderr, "Available command line options:")
	flag.PrintDefaults()
	os.Exit(64)
}
<commit_msg>update pusher<commit_after>/**
* @see https://github.com/Shopify/sarama/blob/master/tools/kafka-console-consumer/kafka-console-consumer.go
 */
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"

	"bytes"
	"encoding/json"
	"github.com/Shopify/sarama"
	"github.com/hidu/go-speed"
	"github.com/hidu/goutils/object"
	"io/ioutil"
	"net/http"
	"net/url"
	"sync/atomic"
	"time"
)

var (
	brokerList = flag.String("brokers", os.Getenv("PUSHER_BROKERS"), "The comma separated list of brokers in the Kafka cluster")
	topic      = flag.String("topic", os.Getenv("PUSHER_TOPIC"), "REQUIRED: the topic to consume")
	partitions = flag.String("partitions", "all", "The partitions to consume, can be 'all' or comma-separated numbers")
	offset     = flag.Int64("offset", -1, "The offset to start with. Can be -2:oldest, -1:newest")
	verbose    = flag.Bool("verbose", false, "Whether to turn on sarama logging")
	//	bufferSize = flag.Int("buffer-size", 256, "The buffer size of the message channel.")
	httpConsumerUrl      = flag.String("http-con-url", os.Getenv("PUSHER_HTTP_CON_URL"), "http consumer url")
	httpConsumerTimeout  = flag.Int("http-con-timeout", 10000, "http consumer timeout,ms")
	httpConsumerReTryNum = flag.Int("rt", 3, "http consumer retry times")
	httpConsumerJsonSuc  = flag.String("http-check-json", "/errno=0", `check http consumer response json {"errno":0}`)

	con = flag.Int("con", 10, "Concurrent Num")

	logger = log.New(os.Stderr, "", log.LstdFlags)
)

const User_Agent = "go-kafka-pusher/0.1/hidu"

var speedData *speed.Speed

var HttpConsumerUrls []string

var dealId uint64

var currentOffset int64

func main() {
	flag.Parse()

	if *brokerList == "" {
		printUsageErrorAndExit("You have to provide -brokers as a comma-separated list, or set the KAFKA_PEERS environment variable.")
	}

	if *topic == "" {
		printUsageErrorAndExit("-topic is required")
	}

	HttpConsumerUrls = parseConUrlFlag()

	if *verbose {
		sarama.Logger = logger
	}

	if *httpConsumerJsonSuc != "" && !strings.Contains(*httpConsumerJsonSuc, "=") {
		printUsageErrorAndExit(`-http-check-json must contains "="`)
	}

	speedData = speed.NewSpeed("call", 5, func(msg string, sp *speed.Speed) {
		logger.Println("[speed]", msg)
	})

	currentOffset=*offset

start:
	c, err := sarama.NewConsumer(strings.Split(*brokerList, ","), nil)
	if err != nil {
		logger.Printf("Failed to start consumer: %s\n", err)
		time.Sleep(500*time.Millisecond)
		goto start
	}
	logger.Println("start consumer success")
	partitionList, err := getPartitions(c)
	if err != nil {
		logger.Printf("Failed to get the list of partitions: %s\n", err)

		c.Close()
		time.Sleep(500*time.Millisecond)
		goto start
	}

	var (
		messages = make(chan *sarama.ConsumerMessage, *con)
		closing  = make(chan struct{})
		wg       sync.WaitGroup
	)

	go func() {
		signals := make(chan os.Signal, 1)
		signal.Notify(signals, os.Kill, os.Interrupt)
		<-signals
		logger.Println("Initiating shutdown of consumer...")
		close(closing)
	}()

	for _, partition := range partitionList {
		pc, err := c.ConsumePartition(*topic, partition, currentOffset)
		if err != nil {
			logger.Printf("Failed to start consumer for partition %d: %s\n", partition, err)
			c.Close()
			time.Sleep(500*time.Millisecond)
			goto start
		}

		go func(pc sarama.PartitionConsumer) {
			<-closing
			pc.AsyncClose()
		}(pc)

		wg.Add(1)
		go func(pc sarama.PartitionConsumer) {
			defer wg.Done()
			for message := range pc.Messages() {
				messages <- message
			}
		}(pc)
	}

	for num := 0; num < *con; num++ {
		go func() {
			client := &http.Client{
				Timeout: time.Duration(*httpConsumerTimeout) * time.Millisecond,
			}
			for msg := range messages {
				currentOffset=msg.Offset
				subNum := 0
				for i := 0; i < *httpConsumerReTryNum; i++ {
					if dealMessage(client, msg, i) {
						subNum = 1
						break
					}
				}
				speedData.Inc(1, len(msg.Value), subNum)
			}
		}()
	}

	wg.Wait()
	logger.Println("Done consuming topic", *topic)
	close(messages)
	speedData.Stop()

	if err := c.Close(); err != nil {
		logger.Println("Failed to close consumer: ", err)
	}
}

func parseConUrlFlag() []string{
	if *httpConsumerUrl == "" {
		printUsageErrorAndExit("-http-con-url is required")
	}
	str := strings.Replace(strings.TrimSpace(*httpConsumerUrl), "\n", ";", -1)
	arr := strings.Split(str, ";")
	var urls []string
	for _, line := range arr {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		_, err := url.Parse(line)
		if err != nil {
			printUsageErrorAndExit("invalid consumer url:" + line)
		}
		urls = append(urls, line)
	}

	if len(urls) < 1 {
		printUsageErrorAndExit("-http-con-url is empty")
	}

	return urls
}

func dealMessage(client *http.Client, msg *sarama.ConsumerMessage, try int) bool {
	curDealId := atomic.AddUint64(&dealId, 1)
	val := string(msg.Value)
	key := string(msg.Key)
	logData := []string{
		fmt.Sprintf("id=%d", curDealId),
		"info",
		fmt.Sprintf("try=%d", try),
		"is_suc=failed",
		fmt.Sprintf("offset=%d", msg.Offset),
		fmt.Sprintf("partition=%d", msg.Partition),
		fmt.Sprintf("key=%s", url.QueryEscape(key)),
		fmt.Sprintf("value_len=%d", len(msg.Value)),
	}
	defer (func() {
		logger.Println(logData)
	})()

	if *verbose {
		fmt.Printf("Partition:\t%d\n", msg.Partition)
		fmt.Printf("Offset:\t%d\n", msg.Offset)
		fmt.Printf("Key:\t%s\n", key)
		fmt.Printf("Value:\t%s\n", val)
		fmt.Println()
	}
	vs := make(url.Values)
	vs.Add("kafka_key", key)
	vs.Add("kafka_partition", fmt.Sprintf("%d", msg.Partition))
	vs.Add("kafka_offset", fmt.Sprintf("%d", msg.Offset))
	qs := vs.Encode()
	vs.Add("kafka_value", val)

	start := time.Now()
	urlNew := HttpConsumerUrls[int(curDealId%uint64(len(HttpConsumerUrls)))]
	if strings.Contains(urlNew, "?") {
		urlNew = urlNew + "&" + qs
	} else {
		urlNew = urlNew + "?" + qs
	}

	req, reqErr := http.NewRequest("POST", urlNew, bytes.NewReader([]byte(vs.Encode())))

	if reqErr != nil {
		logData[1] = "error"
		logData = append(logData, "http_build_req_error:", reqErr.Error())
		return false
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("User-Agent", User_Agent)

	resp, err := client.Do(req)

	used := time.Now().Sub(start)

	logData = append(logData, fmt.Sprintf("used_ms=%d", used.Nanoseconds()/1e6))
	if err != nil {
		logData[1] = "error"
		logData = append(logData, "http_client_error:", err.Error())
		return false
	}
	defer resp.Body.Close()
	bd, respErr := ioutil.ReadAll(resp.Body)

	if respErr != nil {
		logData[1] = "error"
		logData = append(logData, "http_read_resp_error:", respErr.Error())
		return false
	}
	logData = append(logData, fmt.Sprintf("http_code=%d", resp.StatusCode))

	isSuc := true
	if resp.StatusCode != http.StatusOK {
		isSuc = false
	}

	if isSuc && *httpConsumerJsonSuc != "" {
		var obj interface{}
		jsonErr := json.Unmarshal(bd, &obj)
		if jsonErr != nil {
			logData = append(logData, "resp_errno=unknow")
			logData = append(logData, "resp_json_err:", jsonErr.Error())
			isSuc = false
		} else {
			_info := strings.SplitN(*httpConsumerJsonSuc, "=", 2)
			errno, _ := object.NewInterfaceWalker(obj).GetString(_info[0])
			logData = append(logData, "resp_errno="+errno)
			isSuc = errno == _info[1]
		}
	}

	if isSuc {
		logData[3] = "is_suc=suc"
	}

	logData = append(logData, fmt.Sprintf("resp_len=%d", len(bd)), fmt.Sprintf("resp=%s", url.QueryEscape(string(bd))))

	return isSuc
}

func getPartitions(c sarama.Consumer) ([]int32, error) {
	if *partitions == "all" {
		return c.Partitions(*topic)
	}

	tmp := strings.Split(*partitions, ",")
	var pList []int32
	for i := range tmp {
		val, err := strconv.ParseInt(tmp[i], 10, 32)
		if err != nil {
			return nil, err
		}
		pList = append(pList, int32(val))
	}

	return pList, nil
}

func printErrorAndExit(code int, format string, values ...interface{}) {
	fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
	fmt.Fprintln(os.Stderr)
	logger.Printf("ERROR: %s\n", fmt.Sprintf(format, values...))
	os.Exit(code)
}

func printUsageErrorAndExit(format string, values ...interface{}) {
	fmt.Fprintf(os.Stderr, "ERROR: %s\n", fmt.Sprintf(format, values...))
	fmt.Fprintln(os.Stderr)
	fmt.Fprintln(os.Stderr, "Available command line options:")
	flag.PrintDefaults()
	os.Exit(64)
}
<|endoftext|>