<commit_before>package request

import (
	"crypto/md5"
	"errors"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"math/rand"
	"mime/multipart"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"

	"github.com/MG-RAST/Shock/shock-server/auth"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node/archive"
	"github.com/MG-RAST/Shock/shock-server/node/file"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/Shock/shock-server/util"
	"github.com/MG-RAST/golib/httpclient"
	"github.com/MG-RAST/golib/stretchr/goweb/context"
	"github.com/jum/tinyftp"
)

type checkSumCom struct {
	buf      []byte
	n        int
	checksum string
}

func Log(req *http.Request) {
	host, _, _ := net.SplitHostPort(req.RemoteAddr)
	// failed attempt to get the host in ipv4
	//addrs, _ := net.LookupIP(host)
	//fmt.Println(addrs)
	suffix := ""
	if _, auth := req.Header["Authorization"]; auth {
		suffix += " AUTH"
	}

	if l, has := req.Header["Content-Length"]; has {
		suffix += " Content-Length: " + l[0]
	}
	url := ""
	if req.URL.RawQuery != "" {
		url = fmt.Sprintf("%s %s?%s", req.Method, req.URL.Path, req.URL.RawQuery)
	} else {
		url = fmt.Sprintf("%s %s", req.Method, req.URL.Path)
	}
	logger.Infof("%s \"%s%s\"", host, url, suffix)
}

// Authenticate _
func Authenticate(req *http.Request) (u *user.User, err error) {
	if _, ok := req.Header["Authorization"]; !ok {
		err = errors.New(e.NoAuth)
		return
	}
	header := req.Header.Get("Authorization")
	u, err = auth.Authenticate(header)
	return
}

// AuthError _
func AuthError(err error, ctx context.Context) error {

	if conf.DEBUG_AUTH {
		return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
	}

	if err.Error() == e.InvalidAuth {
		return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid authorization header or content")
	}
	err_msg := "Error at Auth: " + err.Error()
	logger.Error(err_msg)
	return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}

// helper function to create a node from an http data post
func DataUpload(r *http.Request) (params map[string]string, files file.FormFiles, err error) {
	params = make(map[string]string)
	files = make(file.FormFiles)
	tmpPath := fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())

	files["upload"] = file.FormFile{Name: "filename", Path: tmpPath, Checksum: make(map[string]string)}
	if tmpFile, err := os.Create(tmpPath); err == nil {
		defer tmpFile.Close()
		md5h := md5.New()
		dst := io.MultiWriter(tmpFile, md5h)
		if _, err = io.Copy(dst, r.Body); err != nil {
			return nil, nil, err
		}
		files["upload"].Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
	} else {
		return nil, nil, err
	}

	return
}

// helper function for create & update
func ParseMultipartForm(r *http.Request) (params map[string]string, files file.FormFiles, err error) {
	params = make(map[string]string)
	files = make(file.FormFiles)
	reader, err := r.MultipartReader()
	if err != nil {
		return
	}

	tmpPath := ""
	for {
		var part *multipart.Part
		part, err = reader.NextPart()
		if err != nil {
			if err.Error() != "EOF" {
				return
			} else {
				break
			}

		}

		// params don't have a FileName()
		// files must have FormName() of either "upload", "gzip", "bzip2", "attributes", "subset_indices", or an integer
		if part.FileName() == "" {
			if !util.IsValidParamName(part.FormName()) {
				return nil, files, errors.New("invalid param: " + part.FormName())
			}
			buffer := make([]byte, 32*1024)
			var n int
			n, err = part.Read(buffer)
			if n == 0 || err != nil {
				break
			}
			formValue := fmt.Sprintf("%s", buffer[0:n])
			if part.FormName() == "upload_url" {
				tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
				files[part.FormName()] = file.FormFile{Name: "", Path: tmpPath, Checksum: make(map[string]string)}
				// download from url
				var tmpFile *os.File
				tmpFile, err = os.Create(tmpPath)
				if err != nil {
					err = fmt.Errorf("(ParseMultipartForm) os.Create returned: %s", err.Error())
					return
				}
				defer tmpFile.Close()
				var tmpform = files[part.FormName()]
				md5h := md5.New()
				dst := io.MultiWriter(tmpFile, md5h)
				var fileName string
				var body io.ReadCloser
				fileName, body, err = fetchFileStream(formValue)
				if err != nil {
					return nil, files, errors.New("unable to stream url: " + err.Error())
				}
				defer body.Close()
				if _, err = io.Copy(dst, body); err != nil {
					return nil, files, err
				}
				tmpform.Name = fileName
				tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
				files[part.FormName()] = tmpform

			} else {
				// regular form field
				params[part.FormName()] = formValue
			}
		} else {
			// determine file type
			isSubsetFile := false
			if part.FormName() == "subset_indices" {
				isSubsetFile = true
			}
			isPartsFile := false
			if _, er := strconv.Atoi(part.FormName()); er == nil {
				isPartsFile = true
			}
			if !isPartsFile && !util.IsValidFileName(part.FormName()) {
				return nil, files, errors.New("invalid file param: " + part.FormName())
			}
			// download it
			tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
			files[part.FormName()] = file.FormFile{Name: part.FileName(), Path: tmpPath, Checksum: make(map[string]string)}
			var tmpFile *os.File
			tmpFile, err = os.Create(tmpPath)
			if err != nil {
				err = fmt.Errorf("(ParseMultipartForm) os.Create returned: %s", err.Error())
				return
			}
			defer tmpFile.Close()
			if util.IsValidUploadFile(part.FormName()) || isPartsFile || isSubsetFile {
				// handle upload or parts files
				var tmpform = files[part.FormName()]
				md5h := md5.New()
				dst := io.MultiWriter(tmpFile, md5h)
				ucReader, ucErr := archive.UncompressReader(part.FormName(), part)
				if ucErr != nil {
					return nil, files, ucErr
				}
				if _, err = io.Copy(dst, ucReader); err != nil {
					return nil, files, err
				}
				if archive.IsValidUncompress(part.FormName()) {
					tmpform.Name = util.StripSuffix(part.FileName())
				}
				tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
				files[part.FormName()] = tmpform
			} else {
				// handle file where md5 not needed
				if _, err = io.Copy(tmpFile, part); err != nil {
					return nil, files, err
				}
			}

		}

	}
	return
}

func fetchFileStream(urlStr string) (f string, r io.ReadCloser, err error) {
	u, _ := url.Parse(urlStr)
	if (u.Scheme == "") || (u.Host == "") || (u.Path == "") {
		return "", nil, errors.New("Not a valid url: " + urlStr)
	}
	pathParts := strings.Split(strings.TrimRight(u.Path, "/"), "/")
	fileName := pathParts[len(pathParts)-1]
	cleanPath := strings.Join(pathParts, "/")

	if (u.Scheme == "http") || (u.Scheme == "https") {
		res, err := httpclient.Get(u.String(), httpclient.Header{}, nil, nil)
		if err != nil {
			return "", nil, errors.New("httpclient returned: " + err.Error())
		}
		if res.StatusCode != 200 { //err in fetching data
			resbody, _ := ioutil.ReadAll(res.Body)
			return "", nil, errors.New(fmt.Sprintf("url=%s, res=%s", u.String(), resbody))
		}
		return fileName, res.Body, err
	} else if u.Scheme == "ftp" {
		// set port if missing
		ftpHost := u.Host
		hostParts := strings.Split(u.Host, ":")
		if len(hostParts) == 1 {
			ftpHost = u.Host + ":21"
		}
		c, _, _, err := tinyftp.Dial("tcp", ftpHost)
		if err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		defer c.Close()
		if _, _, err = c.Login("", ""); err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		dconn, _, _, err := c.RetrieveFrom(cleanPath)
		if err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		return fileName, dconn, err
	}
	return "", nil, errors.New("unsupported protocol scheme: " + u.Scheme)
}

func writeChecksum(f func() hash.Hash, c chan checkSumCom) {
	go func() {
		h := f()
		for {
			select {
			case b := <-c:
				if b.n == 0 {
					c <- checkSumCom{checksum: fmt.Sprintf("%x", h.Sum(nil))}
					return
				} else {
					h.Write(b.buf[0:b.n])
				}
			}
		}
	}()
}
<commit_msg>use ioutil.TempFile<commit_after>package request

import (
	"crypto/md5"
	"errors"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"math/rand"
	"mime/multipart"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/MG-RAST/Shock/shock-server/auth"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node/archive"
	"github.com/MG-RAST/Shock/shock-server/node/file"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/Shock/shock-server/util"
	"github.com/MG-RAST/golib/httpclient"
	"github.com/MG-RAST/golib/stretchr/goweb/context"
	"github.com/jum/tinyftp"
)

type checkSumCom struct {
	buf      []byte
	n        int
	checksum string
}

func Log(req *http.Request) {
	host, _, _ := net.SplitHostPort(req.RemoteAddr)
	// failed attempt to get the host in ipv4
	//addrs, _ := net.LookupIP(host)
	//fmt.Println(addrs)
	suffix := ""
	if _, auth := req.Header["Authorization"]; auth {
		suffix += " AUTH"
	}

	if l, has := req.Header["Content-Length"]; has {
		suffix += " Content-Length: " + l[0]
	}
	url := ""
	if req.URL.RawQuery != "" {
		url = fmt.Sprintf("%s %s?%s", req.Method, req.URL.Path, req.URL.RawQuery)
	} else {
		url = fmt.Sprintf("%s %s", req.Method, req.URL.Path)
	}
	logger.Infof("%s \"%s%s\"", host, url, suffix)
}

// Authenticate _
func Authenticate(req *http.Request) (u *user.User, err error) {
	if _, ok := req.Header["Authorization"]; !ok {
		err = errors.New(e.NoAuth)
		return
	}
	header := req.Header.Get("Authorization")
	u, err = auth.Authenticate(header)
	return
}

// AuthError _
func AuthError(err error, ctx context.Context) error {

	if conf.DEBUG_AUTH {
		return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
	}

	if err.Error() == e.InvalidAuth {
		return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid authorization header or content")
	}
	err_msg := "Error at Auth: " + err.Error()
	logger.Error(err_msg)
	return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}

// helper function to create a node from an http data post
func DataUpload(r *http.Request) (params map[string]string, files file.FormFiles, err error) {
	params = make(map[string]string)
	files = make(file.FormFiles)
	tmpPath := fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())

	files["upload"] = file.FormFile{Name: "filename", Path: tmpPath, Checksum: make(map[string]string)}
	if tmpFile, err := os.Create(tmpPath); err == nil {
		defer tmpFile.Close()
		md5h := md5.New()
		dst := io.MultiWriter(tmpFile, md5h)
		if _, err = io.Copy(dst, r.Body); err != nil {
			return nil, nil, err
		}
		files["upload"].Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
	} else {
		return nil, nil, err
	}

	return
}

// helper function for create & update
func ParseMultipartForm(r *http.Request) (params map[string]string, files file.FormFiles, err error) {
	params = make(map[string]string)
	files = make(file.FormFiles)
	reader, err := r.MultipartReader()
	if err != nil {
		return
	}

	tmpPath := ""
	for {
		var part *multipart.Part
		part, err = reader.NextPart()
		if err != nil {
			if err.Error() != "EOF" {
				return
			} else {
				break
			}

		}

		// params don't have a FileName()
		// files must have FormName() of either "upload", "gzip", "bzip2", "attributes", "subset_indices", or an integer
		if part.FileName() == "" {
			if !util.IsValidParamName(part.FormName()) {
				return nil, files, errors.New("invalid param: " + part.FormName())
			}
			buffer := make([]byte, 32*1024)
			var n int
			n, err = part.Read(buffer)
			if n == 0 || err != nil {
				break
			}
			formValue := fmt.Sprintf("%s", buffer[0:n])
			if part.FormName() == "upload_url" {
				tempDir := path.Join(conf.PATH_DATA, "temp")
				var tmpFile *os.File
				tmpFile, err = ioutil.TempFile(tempDir, "")
				if err != nil {
					err = fmt.Errorf("(ParseMultipartForm) ioutil.TempFile returned: %s", err.Error())
					return
				}
				defer tmpFile.Close()
				tmpPath = tmpFile.Name()
				//tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
				files[part.FormName()] = file.FormFile{Name: "", Path: tmpPath, Checksum: make(map[string]string)}
				// download from url
				//var tmpFile *os.File
				// tmpFile, err = os.Create(tmpPath)
				// if err != nil {
				// 	err = fmt.Errorf("(ParseMultipartForm) os.Create returned: %s", err.Error())
				// 	return
				// }

				var tmpform = files[part.FormName()]
				md5h := md5.New()
				dst := io.MultiWriter(tmpFile, md5h)
				var fileName string
				var body io.ReadCloser
				fileName, body, err = fetchFileStream(formValue)
				if err != nil {
					return nil, files, errors.New("unable to stream url: " + err.Error())
				}
				defer body.Close()
				if _, err = io.Copy(dst, body); err != nil {
					return nil, files, err
				}
				tmpform.Name = fileName
				tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
				files[part.FormName()] = tmpform

			} else {
				// regular form field
				params[part.FormName()] = formValue
			}
		} else {
			// determine file type
			isSubsetFile := false
			if part.FormName() == "subset_indices" {
				isSubsetFile = true
			}
			isPartsFile := false
			if _, er := strconv.Atoi(part.FormName()); er == nil {
				isPartsFile = true
			}
			if !isPartsFile && !util.IsValidFileName(part.FormName()) {
				return nil, files, errors.New("invalid file param: " + part.FormName())
			}
			// download it
			tempDir := path.Join(conf.PATH_DATA, "temp")
			var tmpFile *os.File
			tmpFile, err = ioutil.TempFile(tempDir, "")
			if err != nil {
				err = fmt.Errorf("(ParseMultipartForm) ioutil.TempFile returned: %s", err.Error())
				return
			}
			defer tmpFile.Close()

			tmpPath = tmpFile.Name()
			//tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
			files[part.FormName()] = file.FormFile{Name: part.FileName(), Path: tmpPath, Checksum: make(map[string]string)}
			//var tmpFile *os.File
			// tmpFile, err = os.Create(tmpPath)
			// if err != nil {
			// 	err = fmt.Errorf("(ParseMultipartForm) os.Create returned: %s", err.Error())
			// 	return
			// }

			if util.IsValidUploadFile(part.FormName()) || isPartsFile || isSubsetFile {
				// handle upload or parts files
				var tmpform = files[part.FormName()]
				md5h := md5.New()
				dst := io.MultiWriter(tmpFile, md5h)
				ucReader, ucErr := archive.UncompressReader(part.FormName(), part)
				if ucErr != nil {
					return nil, files, ucErr
				}
				if _, err = io.Copy(dst, ucReader); err != nil {
					return nil, files, err
				}
				if archive.IsValidUncompress(part.FormName()) {
					tmpform.Name = util.StripSuffix(part.FileName())
				}
				tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
				files[part.FormName()] = tmpform
			} else {
				// handle file where md5 not needed
				if _, err = io.Copy(tmpFile, part); err != nil {
					return nil, files, err
				}
			}

		}

	}
	return
}

func fetchFileStream(urlStr string) (f string, r io.ReadCloser, err error) {
	u, _ := url.Parse(urlStr)
	if (u.Scheme == "") || (u.Host == "") || (u.Path == "") {
		return "", nil, errors.New("Not a valid url: " + urlStr)
	}
	pathParts := strings.Split(strings.TrimRight(u.Path, "/"), "/")
	fileName := pathParts[len(pathParts)-1]
	cleanPath := strings.Join(pathParts, "/")

	if (u.Scheme == "http") || (u.Scheme == "https") {
		res, err := httpclient.Get(u.String(), httpclient.Header{}, nil, nil)
		if err != nil {
			return "", nil, errors.New("httpclient returned: " + err.Error())
		}
		if res.StatusCode != 200 { //err in fetching data
			resbody, _ := ioutil.ReadAll(res.Body)
			return "", nil, errors.New(fmt.Sprintf("url=%s, res=%s", u.String(), resbody))
		}
		return fileName, res.Body, err
	} else if u.Scheme == "ftp" {
		// set port if missing
		ftpHost := u.Host
		hostParts := strings.Split(u.Host, ":")
		if len(hostParts) == 1 {
			ftpHost = u.Host + ":21"
		}
		c, _, _, err := tinyftp.Dial("tcp", ftpHost)
		if err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		defer c.Close()
		if _, _, err = c.Login("", ""); err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		dconn, _, _, err := c.RetrieveFrom(cleanPath)
		if err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		return fileName, dconn, err
	}
	return "", nil, errors.New("unsupported protocol scheme: " + u.Scheme)
}

func writeChecksum(f func() hash.Hash, c chan checkSumCom) {
	go func() {
		h := f()
		for {
			select {
			case b := <-c:
				if b.n == 0 {
					c <- checkSumCom{checksum: fmt.Sprintf("%x", h.Sum(nil))}
					return
				} else {
					h.Write(b.buf[0:b.n])
				}
			}
		}
	}()
}
<|endoftext|>


<commit_before>package magick

// #include <magick/api.h>
import "C"

// ChannelStatistics includes several statistics about
// a color channel. Magick normalizes All fields in this
// structure to the interval [0, 1].
type ChannelStatistics struct {
	Minimum  float64
	Maximum  float64
	Mean     float64
	StdDev   float64
	Variance float64
	// The following fields are only filled in when building
	// against ImageMagick
	Kurtosis float64
	Skewness float64
}

type Statistics struct {
	Red     *ChannelStatistics
	Green   *ChannelStatistics
	Blue    *ChannelStatistics
	Opacity *ChannelStatistics
}

func (im *Image) Statistics() (*Statistics, error) {
	return im.statistics()
}
<commit_msg>Fix typo in comment<commit_after>package magick

// #include <magick/api.h>
import "C"

// ChannelStatistics includes several statistics about
// a color channel. Magick normalizes all fields in this
// structure to the interval [0, 1].
type ChannelStatistics struct {
	Minimum  float64
	Maximum  float64
	Mean     float64
	StdDev   float64
	Variance float64
	// The following fields are only filled in when building
	// against ImageMagick
	Kurtosis float64
	Skewness float64
}

type Statistics struct {
	Red     *ChannelStatistics
	Green   *ChannelStatistics
	Blue    *ChannelStatistics
	Opacity *ChannelStatistics
}

func (im *Image) Statistics() (*Statistics, error) {
	return im.statistics()
}
<|endoftext|>


<commit_before>package diff_test

import (
	"bytes"
	"testing"

	"github.com/schemalex/schemalex/diff"
	"github.com/stretchr/testify/assert"
)

func TestDiff(t *testing.T) {
	type Spec struct {
		Before string
		After  string
		Expect string
	}

	specs := []Spec{
		// drop table
		{
			Before: "CREATE TABLE `hoge` ( `id` integer not null ); CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "DROP TABLE `hoge`;",
		},
		// create table
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `hoge` ( `id` INTEGER NOT NULL ) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COMMENT 'table comment'; CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "CREATE TABLE `hoge` (\n`id` INT (11) NOT NULL\n) ENGINE = InnoDB, DEFAULT CHARACTER SET = utf8mb4, COMMENT = 'table comment';",
		},
		// drop column
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL, `c` VARCHAR (20) NOT NULL DEFAULT 'xxx' );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "ALTER TABLE `fuga` DROP COLUMN `c`;",
		},
		// add column (after)
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL, `c` VARCHAR (20) NOT NULL DEFAULT 'xxx');",
			Expect: "ALTER TABLE `fuga` ADD COLUMN `c` VARCHAR (20) NOT NULL DEFAULT 'xxx' AFTER `id`;",
		},
		// add column (first)
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `c` VARCHAR (20) NOT NULL DEFAULT 'xxx', `id` INTEGER NOT NULL);",
			Expect: "ALTER TABLE `fuga` ADD COLUMN `c` VARCHAR (20) NOT NULL DEFAULT 'xxx' FIRST;",
		},
		// change column
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` BIGINT NOT NULL );",
			Expect: "ALTER TABLE `fuga` CHANGE COLUMN `id` `id` BIGINT (20) NOT NULL;",
		},
		// change column with comment
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL COMMENT 'fuga is good' );",
			Expect: "ALTER TABLE `fuga` CHANGE COLUMN `id` `id` INT (11) NOT NULL COMMENT 'fuga is good';",
		},
		// drop primary key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, PRIMARY KEY (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			Expect: "ALTER TABLE `fuga` DROP INDEX PRIMARY KEY;",
		},
		// add primary key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, PRIMARY KEY (`id`) );",
			Expect: "ALTER TABLE `fuga` ADD PRIMARY KEY (`id`);",
		},
		// drop unique key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			Expect: "ALTER TABLE `fuga` DROP INDEX `uniq_id`;",
		},
		// add unique key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			Expect: "ALTER TABLE `fuga` ADD CONSTRAINT `symbol` UNIQUE INDEX `uniq_id` USING BTREE (`id`);",
		},
		// not change index
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			Expect: "",
		},
		// not change FOREIGN KEY
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			Expect: "",
		},
		// change CONSTRAINT symbol naml
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, CONSTRAINT `fsym` FOREIGN KEY (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, CONSTRAINT `ksym` FOREIGN KEY (fid) REFERENCES f (id) );",
			Expect: "ALTER TABLE `fuga` DROP FOREIGN KEY `fsym`;\nALTER TABLE `fuga` DROP INDEX `fsym`;\nALTER TABLE `fuga` ADD INDEX `ksym` (`fid`);\nALTER TABLE `fuga` ADD CONSTRAINT `ksym` FOREIGN KEY (`fid`) REFERENCES `f` (`id`);",
		},
		// remove FOREIGN KEY
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, INDEX fid (fid) );",
			Expect: "ALTER TABLE `fuga` DROP FOREIGN KEY `fk`;\nALTER TABLE `fuga` ADD INDEX `fid` (`fid`);",
		},
		// multi modify
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `aid` INTEGER NOT NULL, `bid` INTEGER NOT NULL, INDEX `ab` (`aid`, `bid`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `aid` INTEGER NOT NULL, `cid` INTEGER NOT NULL, INDEX `ac` (`aid`, `cid`) );",
			Expect: "ALTER TABLE `fuga` DROP INDEX `ab`;\nALTER TABLE `fuga` DROP COLUMN `bid`;\nALTER TABLE `fuga` ADD COLUMN `cid` INT (11) NOT NULL AFTER `aid`;\nALTER TABLE `fuga` ADD INDEX `ac` (`aid`, `cid`);",
		},
		// not change to query what generated by show create table
		{
			// human input
			Before: `
create table foo (
   id int not null AUTO_INCREMENT PRIMARY KEY,
   tinyints tinyint,
   tinyintu tinyint unsigned,
   smallints smallint,
   smallintu smallint unsigned,
   mediumints mediumint,
   mediumintu mediumint unsigned,
   ints int comment 'this is sined int nullable',
   intu int unsigned,
   integers integer null default null,
   integeru integer unsigned null,
   bigins bigint UNIQUE KEY,
   bigintu bigint unsigned,
   floats float,
   floaru float unsigned,
   doubles double,
   doubleu double unsigned,
   decimals decimal,
   decimalu decimal unsigned,
   varcharn varchar (10) null,
   varcharnn varchar (10) not null,
   textn text,
   textnn text not null,
   blobn blob,
   blobnn blob,
   intsd int default 0,
   intud int unsigned default 0,
   CONSTRAINT bar_fk FOREIGN KEY (integers) REFERENCES bar (id),
   INDEX foo_idx (ints)
);
			`,
			// show create table foo
			After: `
CREATE TABLE foo (
  id int(11) NOT NULL AUTO_INCREMENT,
  tinyints tinyint(4) DEFAULT NULL,
  tinyintu tinyint(3) unsigned DEFAULT NULL,
  smallints smallint(6) DEFAULT NULL,
  smallintu smallint(5) unsigned DEFAULT NULL,
  mediumints mediumint(9) DEFAULT NULL,
  mediumintu mediumint(8) unsigned DEFAULT NULL,
  ints int(11) DEFAULT NULL COMMENT 'this is sined int nullable',
  intu int(10) unsigned DEFAULT NULL,
  integers int(11) DEFAULT NULL,
  integeru int(10) unsigned DEFAULT NULL,
  bigins bigint(20) DEFAULT NULL,
  bigintu bigint(20) unsigned DEFAULT NULL,
  floats float DEFAULT NULL,
  floaru float unsigned DEFAULT NULL,
  doubles double DEFAULT NULL,
  doubleu double unsigned DEFAULT NULL,
  decimals decimal(10,0) DEFAULT NULL,
  decimalu decimal(10,0) unsigned DEFAULT NULL,
  varcharn varchar(10) DEFAULT NULL,
  varcharnn varchar(10) NOT NULL,
  textn text,
  textnn text NOT NULL,
  blobn blob,
  blobnn blob,
  intsd int(11) DEFAULT '0',
  intud int(10) unsigned DEFAULT '0',
  PRIMARY KEY (id),
  UNIQUE KEY bigins (bigins),
  KEY bar_fk (integers),
  KEY foo_idx (ints),
  CONSTRAINT bar_fk FOREIGN KEY (integers) REFERENCES bar (id)
);
			`,
			Expect: "",
		},
	}

	var buf bytes.Buffer
	for _, spec := range specs {
		buf.Reset()
		if !assert.NoError(t, diff.Strings(&buf, spec.Before, spec.After), "diff.String should succeed") {
			return
		}

		if !assert.Equal(t, spec.Expect, buf.String(), "result SQL should match") {
			t.Logf("before = %s", spec.Before)
			t.Logf("after = %s", spec.After)
			return
		}
	}
}
<commit_msg>Changed "ADD COLUMN" test cases.<commit_after>package diff_test

import (
	"bytes"
	"testing"

	"github.com/schemalex/schemalex/diff"
	"github.com/stretchr/testify/assert"
)

func TestDiff(t *testing.T) {
	type Spec struct {
		Before string
		After  string
		Expect string
	}

	specs := []Spec{
		// drop table
		{
			Before: "CREATE TABLE `hoge` ( `id` integer not null ); CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "DROP TABLE `hoge`;",
		},
		// create table
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `hoge` ( `id` INTEGER NOT NULL ) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COMMENT 'table comment'; CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "CREATE TABLE `hoge` (\n`id` INT (11) NOT NULL\n) ENGINE = InnoDB, DEFAULT CHARACTER SET = utf8mb4, COMMENT = 'table comment';",
		},
		// drop column
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL, `c` VARCHAR (20) NOT NULL DEFAULT 'xxx' );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "ALTER TABLE `fuga` DROP COLUMN `c`;",
		},
		// add column (after)
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL, `a` INTEGER NOT NULL, `b` INTEGER NOT NULL, `c` INTEGER NOT NULL );",
			Expect: "ALTER TABLE `fuga` ADD COLUMN `a` INT (11) NOT NULL AFTER `id`;\nALTER TABLE `fuga` ADD COLUMN `b` INT (11) NOT NULL AFTER `a`;\nALTER TABLE `fuga` ADD COLUMN `c` INT (11) NOT NULL AFTER `b`;",
		},
		// add column (first)
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `a` INTEGER NOT NULL, `b` INTEGER NOT NULL, `c` INTEGER NOT NULL, `id` INTEGER NOT NULL);",
			Expect: "ALTER TABLE `fuga` ADD COLUMN `a` INT (11) NOT NULL FIRST;\nALTER TABLE `fuga` ADD COLUMN `b` INT (11) NOT NULL AFTER `a`;\nALTER TABLE `fuga` ADD COLUMN `c` INT (11) NOT NULL AFTER `b`;",
		},
		// change column
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` BIGINT NOT NULL );",
			Expect: "ALTER TABLE `fuga` CHANGE COLUMN `id` `id` BIGINT (20) NOT NULL;",
		},
		// change column with comment
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL COMMENT 'fuga is good' );",
			Expect: "ALTER TABLE `fuga` CHANGE COLUMN `id` `id` INT (11) NOT NULL COMMENT 'fuga is good';",
		},
		// drop primary key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, PRIMARY KEY (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			Expect: "ALTER TABLE `fuga` DROP INDEX PRIMARY KEY;",
		},
		// add primary key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, PRIMARY KEY (`id`) );",
			Expect: "ALTER TABLE `fuga` ADD PRIMARY KEY (`id`);",
		},
		// drop unique key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			Expect: "ALTER TABLE `fuga` DROP INDEX `uniq_id`;",
		},
		// add unique key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			Expect: "ALTER TABLE `fuga` ADD CONSTRAINT `symbol` UNIQUE INDEX `uniq_id` USING BTREE (`id`);",
		},
		// not change index
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			Expect: "",
		},
		// not change FOREIGN KEY
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			Expect: "",
		},
		// change CONSTRAINT symbol naml
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, CONSTRAINT `fsym` FOREIGN KEY (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, CONSTRAINT `ksym` FOREIGN KEY (fid) REFERENCES f (id) );",
			Expect: "ALTER TABLE `fuga` DROP FOREIGN KEY `fsym`;\nALTER TABLE `fuga` DROP INDEX `fsym`;\nALTER TABLE `fuga` ADD INDEX `ksym` (`fid`);\nALTER TABLE `fuga` ADD CONSTRAINT `ksym` FOREIGN KEY (`fid`) REFERENCES `f` (`id`);",
		},
		// remove FOREIGN KEY
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, INDEX fid (fid) );",
			Expect: "ALTER TABLE `fuga` DROP FOREIGN KEY `fk`;\nALTER TABLE `fuga` ADD INDEX `fid` (`fid`);",
		},
		// multi modify
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `aid` INTEGER NOT NULL, `bid` INTEGER NOT NULL, INDEX `ab` (`aid`, `bid`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `aid` INTEGER NOT NULL, `cid` INTEGER NOT NULL, INDEX `ac` (`aid`, `cid`) );",
			Expect: "ALTER TABLE `fuga` DROP INDEX `ab`;\nALTER TABLE `fuga` DROP COLUMN `bid`;\nALTER TABLE `fuga` ADD COLUMN `cid` INT (11) NOT NULL AFTER `aid`;\nALTER TABLE `fuga` ADD INDEX `ac` (`aid`, `cid`);",
		},
		// not change to query what generated by show create table
		{
			// human input
			Before: `
create table foo (
   id int not null AUTO_INCREMENT PRIMARY KEY,
   tinyints tinyint,
   tinyintu tinyint unsigned,
   smallints smallint,
   smallintu smallint unsigned,
   mediumints mediumint,
   mediumintu mediumint unsigned,
   ints int comment 'this is sined int nullable',
   intu int unsigned,
   integers integer null default null,
   integeru integer unsigned null,
   bigins bigint UNIQUE KEY,
   bigintu bigint unsigned,
   floats float,
   floaru float unsigned,
   doubles double,
   doubleu double unsigned,
   decimals decimal,
   decimalu decimal unsigned,
   varcharn varchar (10) null,
   varcharnn varchar (10) not null,
   textn text,
   textnn text not null,
   blobn blob,
   blobnn blob,
   intsd int default 0,
   intud int unsigned default 0,
   CONSTRAINT bar_fk FOREIGN KEY (integers) REFERENCES bar (id),
   INDEX foo_idx (ints)
);
			`,
			// show create table foo
			After: `
CREATE TABLE foo (
  id int(11) NOT NULL AUTO_INCREMENT,
  tinyints tinyint(4) DEFAULT NULL,
  tinyintu tinyint(3) unsigned DEFAULT NULL,
  smallints smallint(6) DEFAULT NULL,
  smallintu smallint(5) unsigned DEFAULT NULL,
  mediumints mediumint(9) DEFAULT NULL,
  mediumintu mediumint(8) unsigned DEFAULT NULL,
  ints int(11) DEFAULT NULL COMMENT 'this is sined int nullable',
  intu int(10) unsigned DEFAULT NULL,
  integers int(11) DEFAULT NULL,
  integeru int(10) unsigned DEFAULT NULL,
  bigins bigint(20) DEFAULT NULL,
  bigintu bigint(20) unsigned DEFAULT NULL,
  floats float DEFAULT NULL,
  floaru float unsigned DEFAULT NULL,
  doubles double DEFAULT NULL,
  doubleu double unsigned DEFAULT NULL,
  decimals decimal(10,0) DEFAULT NULL,
  decimalu decimal(10,0) unsigned DEFAULT NULL,
  varcharn varchar(10) DEFAULT NULL,
  varcharnn varchar(10) NOT NULL,
  textn text,
  textnn text NOT NULL,
  blobn blob,
  blobnn blob,
  intsd int(11) DEFAULT '0',
  intud int(10) unsigned DEFAULT '0',
  PRIMARY KEY (id),
  UNIQUE KEY bigins (bigins),
  KEY bar_fk (integers),
  KEY foo_idx (ints),
  CONSTRAINT bar_fk FOREIGN KEY (integers) REFERENCES bar (id)
);
			`,
			Expect: "",
		},
	}

	var buf bytes.Buffer
	for _, spec := range specs {
		buf.Reset()
		if !assert.NoError(t, diff.Strings(&buf, spec.Before, spec.After), "diff.String should succeed") {
			return
		}

		if !assert.Equal(t, spec.Expect, buf.String(), "result SQL should match") {
			t.Logf("before = %s", spec.Before)
			t.Logf("after = %s", spec.After)
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package compiler

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
)

var (
	versionRegexp = regexp.MustCompile("[0-9]+\\.[0-9]+\\.[0-9]+")
	legacyRegexp  = regexp.MustCompile("0\\.(9\\..*|1\\.[01])")
	paramsLegacy  = []string{
		"--binary",       // Request to output the contract in binary (hexadecimal).
		"file",           //
		"--json-abi",     // Request to output the contract's JSON ABI interface.
		"file",           //
		"--natspec-user", // Request to output the contract's Natspec user documentation.
		"file",           //
		"--natspec-dev",  // Request to output the contract's Natspec developer documentation.
		"file",
		"--add-std",
		"1",
	}
	paramsNew = []string{
		"--bin",      // Request to output the contract in binary (hexadecimal).
		"--abi",      // Request to output the contract's JSON ABI interface.
		"--userdoc",  // Request to output the contract's Natspec user documentation.
		"--devdoc",   // Request to output the contract's Natspec developer documentation.
		"--add-std",  // include standard lib contracts
		"--optimize", // code optimizer switched on
		"-o",         // output directory
	}
)

type Contract struct {
	Code string       `json:"code"`
	Info ContractInfo `json:"info"`
}

type ContractInfo struct {
	Source          string      `json:"source"`
	Language        string      `json:"language"`
	LanguageVersion string      `json:"languageVersion"`
	CompilerVersion string      `json:"compilerVersion"`
	CompilerOptions string      `json:"compilerOptions"`
	AbiDefinition   interface{} `json:"abiDefinition"`
	UserDoc         interface{} `json:"userDoc"`
	DeveloperDoc    interface{} `json:"developerDoc"`
}

type Solidity struct {
	solcPath    string
	version     string
	fullVersion string
	legacy      bool
}

func New(solcPath string) (sol *Solidity, err error) {
	// set default solc
	if len(solcPath) == 0 {
		solcPath = "solc"
	}
	solcPath, err = exec.LookPath(solcPath)
	if err != nil {
		return
	}

	cmd := exec.Command(solcPath, "--version")
	var out bytes.Buffer
	cmd.Stdout = &out
	err = cmd.Run()
	if err != nil {
		return
	}

	fullVersion := out.String()
	version := versionRegexp.FindString(fullVersion)
	legacy := legacyRegexp.MatchString(version)

	sol = &Solidity{
		solcPath:    solcPath,
		version:     version,
		fullVersion: fullVersion,
		legacy:      legacy,
	}
	glog.V(logger.Info).Infoln(sol.Info())
	return
}

func (sol *Solidity) Info() string {
	return fmt.Sprintf("%s\npath: %s", sol.fullVersion, sol.solcPath)
}

func (sol *Solidity) Version() string {
	return sol.version
}

// Compile builds and returns all the contracts contained within a source string.
func (sol *Solidity) Compile(source string) (map[string]*Contract, error) {
	// Short circuit if no source code was specified
	if len(source) == 0 {
		return nil, errors.New("solc: empty source string")
	}
	// Create a safe place to dump compilation output
	wd, err := ioutil.TempDir("", "solc")
	if err != nil {
		return nil, fmt.Errorf("solc: failed to create temporary build folder: %v", err)
	}
	defer os.RemoveAll(wd)

	// Assemble the compiler command, change to the temp folder and capture any errors
	stderr := new(bytes.Buffer)

	var params []string
	if sol.legacy {
		params = paramsLegacy
	} else {
		params = paramsNew
		params = append(params, wd)
	}
	compilerOptions := strings.Join(params, " ")

	cmd := exec.Command(sol.solcPath, params...)
	cmd.Dir = wd
	cmd.Stdin = strings.NewReader(source)
	cmd.Stderr = stderr

	if err := cmd.Run(); err != nil {
		return nil, fmt.Errorf("solc: %v\n%s", err, string(stderr.Bytes()))
	}
	// Sanity check that something was actually built
	matches, _ := filepath.Glob(filepath.Join(wd, "*.bin*"))
	if len(matches) < 1 {
		return nil, fmt.Errorf("solc: no build results found")
	}
	// Compilation succeeded, assemble and return the contracts
	contracts := make(map[string]*Contract)
	for _, path := range matches {
		_, file := filepath.Split(path)
		base := strings.Split(file, ".")[0]

		// Parse the individual compilation results (code binary, ABI definitions, user and dev docs)
		var binary []byte
		binext := ".bin"
		if sol.legacy {
			binext = ".binary"
		}
		if binary, err = ioutil.ReadFile(filepath.Join(wd, base+binext)); err != nil {
			return nil, fmt.Errorf("solc: error reading compiler output for code: %v", err)
		}

		var abi interface{}
		if blob, err := ioutil.ReadFile(filepath.Join(wd, base+".abi")); err != nil {
			return nil, fmt.Errorf("solc: error reading abi definition: %v", err)
		} else if err = json.Unmarshal(blob, &abi); err != nil {
			return nil, fmt.Errorf("solc: error parsing abi definition: %v", err)
		}

		var userdoc interface{}
		if blob, err := ioutil.ReadFile(filepath.Join(wd, base+".docuser")); err != nil {
			return nil, fmt.Errorf("solc: error reading user doc: %v", err)
		} else if err = json.Unmarshal(blob, &userdoc); err != nil {
			return nil, fmt.Errorf("solc: error parsing user doc: %v", err)
		}

		var devdoc interface{}
		if blob, err := ioutil.ReadFile(filepath.Join(wd, base+".docdev")); err != nil {
			return nil, fmt.Errorf("solc: error reading dev doc: %v", err)
		} else if err = json.Unmarshal(blob, &devdoc); err != nil {
			return nil, fmt.Errorf("solc: error parsing dev doc: %v", err)
		}
		// Assemble the final contract
		contracts[base] = &Contract{
			Code: "0x" + string(binary),
			Info: ContractInfo{
				Source:          source,
				Language:        "Solidity",
				LanguageVersion: sol.version,
				CompilerVersion: sol.version,
				CompilerOptions: compilerOptions,
				AbiDefinition:   abi,
				UserDoc:         userdoc,
				DeveloperDoc:    devdoc,
			},
		}
	}
	return contracts, nil
}

func SaveInfo(info *ContractInfo, filename string) (contenthash common.Hash, err error) {
	infojson, err := json.Marshal(info)
	if err != nil {
		return
	}
	contenthash = common.BytesToHash(crypto.Keccak256(infojson))
	err = ioutil.WriteFile(filename, infojson, 0600)
	return
}
<commit_msg>common/compiler: support relative path to solc<commit_after>// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package compiler

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
)

var (
	versionRegexp = regexp.MustCompile("[0-9]+\\.[0-9]+\\.[0-9]+")
	legacyRegexp  = regexp.MustCompile("0\\.(9\\..*|1\\.[01])")
	paramsLegacy  = []string{
		"--binary",       // Request to output the contract in binary (hexadecimal).
		"file",           //
		"--json-abi",     // Request to output the contract's JSON ABI interface.
		"file",           //
		"--natspec-user", // Request to output the contract's Natspec user documentation.
		"file",           //
		"--natspec-dev",  // Request to output the contract's Natspec developer documentation.
		"file",
		"--add-std",
		"1",
	}
	paramsNew = []string{
		"--bin",      // Request to output the contract in binary (hexadecimal).
		"--abi",      // Request to output the contract's JSON ABI interface.
		"--userdoc",  // Request to output the contract's Natspec user documentation.
		"--devdoc",   // Request to output the contract's Natspec developer documentation.
		"--add-std",  // include standard lib contracts
		"--optimize", // code optimizer switched on
		"-o",         // output directory
	}
)

type Contract struct {
	Code string       `json:"code"`
	Info ContractInfo `json:"info"`
}

type ContractInfo struct {
	Source          string      `json:"source"`
	Language        string      `json:"language"`
	LanguageVersion string      `json:"languageVersion"`
	CompilerVersion string      `json:"compilerVersion"`
	CompilerOptions string      `json:"compilerOptions"`
	AbiDefinition   interface{} `json:"abiDefinition"`
	UserDoc         interface{} `json:"userDoc"`
	DeveloperDoc    interface{} `json:"developerDoc"`
}

type Solidity struct {
	solcPath    string
	version     string
	fullVersion string
	legacy      bool
}

func New(solcPath string) (sol *Solidity, err error) {
	// set default solc
	if len(solcPath) == 0 {
		solcPath = "solc"
	}
	solcPath, err = exec.LookPath(solcPath)
	if err != nil {
		return
	}

	cmd := exec.Command(solcPath, "--version")
	var out bytes.Buffer
	cmd.Stdout = &out
	err = cmd.Run()
	if err != nil {
		return
	}

	fullVersion := out.String()
	version := versionRegexp.FindString(fullVersion)
	legacy := legacyRegexp.MatchString(version)

	sol = &Solidity{
		solcPath:    solcPath,
		version:     version,
		fullVersion: fullVersion,
		legacy:      legacy,
	}
	glog.V(logger.Info).Infoln(sol.Info())
	return
}

func (sol *Solidity) Info() string {
	return fmt.Sprintf("%s\npath: %s", sol.fullVersion, sol.solcPath)
}

func (sol *Solidity) Version() string {
	return sol.version
}

// Compile builds and returns all the contracts contained within a source string.
func (sol *Solidity) Compile(source string) (map[string]*Contract, error) {
	// Short circuit if no source code was specified
	if len(source) == 0 {
		return nil, errors.New("solc: empty source string")
	}
	// Create a safe place to dump compilation output
	wd, err := ioutil.TempDir("", "solc")
	if err != nil {
		return nil, fmt.Errorf("solc: failed to create temporary build folder: %v", err)
	}
	defer os.RemoveAll(wd)

	// Assemble the compiler command, change to the temp folder and capture any errors
	stderr := new(bytes.Buffer)

	var params []string
	if sol.legacy {
		params = paramsLegacy
	} else {
		params = paramsNew
		params = append(params, wd)
	}
	compilerOptions := strings.Join(params, " ")

	cmd := exec.Command(sol.solcPath, params...)
	cmd.Stdin = strings.NewReader(source)
	cmd.Stderr = stderr

	if err := cmd.Run(); err != nil {
		return nil, fmt.Errorf("solc: %v\n%s", err, string(stderr.Bytes()))
	}
	// Sanity check that something was actually built
	matches, _ := filepath.Glob(filepath.Join(wd, "*.bin*"))
	if len(matches) < 1 {
		return nil, fmt.Errorf("solc: no build results found")
	}
	// Compilation succeeded, assemble and return the contracts
	contracts := make(map[string]*Contract)
	for _, path := range matches {
		_, file := filepath.Split(path)
		base := strings.Split(file, ".")[0]

		// Parse the individual compilation results (code binary, ABI definitions, user and dev docs)
		var binary []byte
		binext := ".bin"
		if sol.legacy {
			binext = ".binary"
		}
		if binary, err = ioutil.ReadFile(filepath.Join(wd, base+binext)); err != nil {
			return nil, fmt.Errorf("solc: error reading compiler output for code: %v", err)
		}

		var abi interface{}
		if blob, err := ioutil.ReadFile(filepath.Join(wd, base+".abi")); err != nil {
			return nil, fmt.Errorf("solc: error reading abi definition: %v", err)
		} else if err = json.Unmarshal(blob, &abi); err != nil {
			return nil, fmt.Errorf("solc: error parsing abi definition: %v", err)
		}

		var userdoc interface{}
		if blob, err := ioutil.ReadFile(filepath.Join(wd, base+".docuser")); err != nil {
			return nil, fmt.Errorf("solc: error reading user doc: %v", err)
		} else if err = json.Unmarshal(blob, &userdoc); err != nil {
			return nil, fmt.Errorf("solc: error parsing user doc: %v", err)
		}

		var devdoc interface{}
		if blob, err := ioutil.ReadFile(filepath.Join(wd, base+".docdev")); err != nil {
			return nil, fmt.Errorf("solc: error reading dev doc: %v", err)
		} else if err = json.Unmarshal(blob, &devdoc); err != nil {
			return nil, fmt.Errorf("solc: error parsing dev doc: %v", err)
		}
		// Assemble the final contract
		contracts[base] = &Contract{
			Code: "0x" + string(binary),
			Info: ContractInfo{
				Source:          source,
				Language:        "Solidity",
				LanguageVersion: sol.version,
				CompilerVersion: sol.version,
				CompilerOptions: compilerOptions,
				AbiDefinition:   abi,
				UserDoc:         userdoc,
				DeveloperDoc:    devdoc,
			},
		}
	}
	return contracts, nil
}

func SaveInfo(info *ContractInfo, filename string) (contenthash common.Hash, err error) {
	infojson, err := json.Marshal(info)
	if err != nil {
		return
	}
	contenthash = common.BytesToHash(crypto.Keccak256(infojson))
	err = ioutil.WriteFile(filename, infojson, 0600)
	return
}
<|endoftext|>


<commit_before>package v8

/*
#include "v8_wrap.h"
#include <stdlib.h>
*/
import "C"
import "unsafe"
import "runtime"

//import "reflect"

// A sandboxed execution context with its own set of built-in objects
// and functions.
type Context struct {
	embedable
	self   unsafe.Pointer
	engine *Engine
}

type ContextScope struct {
	context *Context
}

type EscapableScope struct {
	ContextScope
}

func (cs ContextScope) GetEngine() *Engine {
	return cs.context.engine
}

func (cs ContextScope) GetPrivateData() interface{} {
	return cs.context.GetPrivateData()
}

func (cs ContextScope) SetPrivateData(data interface{}) {
	cs.context.SetPrivateData(data)
}

func (e *Engine) NewContext(globalTemplate *ObjectTemplate) *Context {
	var globalTemplatePtr unsafe.Pointer
	if globalTemplate != nil {
		globalTemplatePtr = globalTemplate.self
	}
	self := C.V8_NewContext(e.self, globalTemplatePtr)

	if self == nil {
		return nil
	}

	result := &Context{
		self:   self,
		engine: e,
	}

	runtime.SetFinalizer(result, func(c *Context) {
		if traceDispose {
			println("v8.Context.Dispose()", c.self)
		}
		C.V8_DisposeContext(c.self)
	})

	return result
}

//export go_context_scope_callback
func go_context_scope_callback(c unsafe.Pointer, callback unsafe.Pointer) {
	(*(*func(ContextScope))(callback))(ContextScope{(*Context)(c)})
}

//export go_escapable_scope_callback
func go_escapable_scope_callback(c unsafe.Pointer, callback unsafe.Pointer){
	(*(*func(EscapableScope))(callback))(EscapableScope{ContextScope{(*Context)(c)}})
}

func (c *Context) Scope(callback func(ContextScope)) {
	C.V8_Context_Scope(c.self, unsafe.Pointer(c), unsafe.Pointer(&callback))
}

func (c *Context) EscapableScope(callback func(EscapableScope)){
	C.V8_Escapable_Scope(c.self, unsafe.Pointer(c), unsafe.Pointer(&callback))
}

//export go_try_catch_callback
func go_try_catch_callback(callback unsafe.Pointer) {
	(*(*func())(callback))()
}

func escape(s string) string {
	output := ""
	for _, r := range s {
		switch r {
		case '"':
			output += "\\\""
		case '\\':
			output += "\\\\"
		case '/':
			output += "\\/"
		case '\n':
			output += "\\n"
		case '\r':
			output += "\\r"
		case '\t':
			output += "\\t"
		case '\b':
			output += "\\b"
		case '\f':
			output += "\\f"
		default:
			output += string(r)
		}
	}
	return output
}

func (es EscapableScope) Escape(escontext *Context) *Context {
	self := C.V8_Escapable_Escape(escontext.self)
	if self == nil {
		return nil
	}
	e := es.GetEngine()
	result := &Context{
		self:   self,
		engine: e,
	}

	runtime.SetFinalizer(result, func(c *Context) {
		if traceDispose {
			println("v8.Context.Dispose()", c.self)
		}
		C.V8_DisposeContext(c.self)
	})

	return result
	//return newValue(es.GetEngine(), C.V8_Escapable_Escape())
}

func (cs ContextScope) ThrowException(err string) {
	cs.Eval(`throw "` + escape(err) + `"`)
	//
	// TODO: use Isolate::ThrowException() will make FunctionTemplate::GetFunction() returns NULL, why?
	//
	//errPtr := unsafe.Pointer((*reflect.StringHeader)(unsafe.Pointer(&err)).Data)
	//C.V8_Context_ThrowException(cs.context.self, (*C.char)(errPtr), C.int(len(err)))
}

func (cs ContextScope) ThrowException2(value *Value) {
	C.V8_Context_ThrowException2(value.self)
}

func (cs ContextScope) TryCatch(callback func()) *Message {
	msg := C.V8_Context_TryCatch(cs.context.self, unsafe.Pointer(&callback))
	if msg == nil {
		return nil
	}
	return (*Message)(msg)
}

type Exception struct {
	*Value
	*Message
}

func (cs ContextScope) TryCatchException(callback func()) *Exception {
	e := C.V8_Context_TryCatchException(cs.context.self, unsafe.Pointer(&callback))
	if e == nil {
		return nil
	}

	excep := (*exception)(e)
	val := newValue(cs.GetEngine(), excep.Pointer)
	if val == nil {
		return nil
	}

	return &Exception{val, excep.Message}
}

func (cs ContextScope) Global() *Object {
	return newValue(cs.GetEngine(), C.V8_Context_Global(cs.context.self)).ToObject()
}
<commit_msg>Add function <SetSecurityToken,GetSecurityToken,UseDefaultSecurityToken,GetEmbedderData,SetEmbedderData><commit_after>package v8

/*
#include "v8_wrap.h"
#include <stdlib.h>
*/
import "C"
import "unsafe"
import "runtime"

//import "reflect"

// A sandboxed execution context with its own set of built-in objects
// and functions.
type Context struct {
	embedable
	self   unsafe.Pointer
	engine *Engine
}

type ContextScope struct {
	context *Context
}

type EscapableScope struct {
	ContextScope
}

func (cs ContextScope) GetEngine() *Engine {
	return cs.context.engine
}

func (cs ContextScope) GetPrivateData() interface{} {
	return cs.context.GetPrivateData()
}

func (cs ContextScope) SetPrivateData(data interface{}) {
	cs.context.SetPrivateData(data)
}

func (e *Engine) NewContext(globalTemplate *ObjectTemplate) *Context {
	var globalTemplatePtr unsafe.Pointer
	if globalTemplate != nil {
		globalTemplatePtr = globalTemplate.self
	}
	self := C.V8_NewContext(e.self, globalTemplatePtr)

	if self == nil {
		return nil
	}

	result := &Context{
		self:   self,
		engine: e,
	}

	runtime.SetFinalizer(result, func(c *Context) {
		if traceDispose {
			println("v8.Context.Dispose()", c.self)
		}
		C.V8_DisposeContext(c.self)
	})

	return result
}

//export go_context_scope_callback
func go_context_scope_callback(c unsafe.Pointer, callback unsafe.Pointer) {
	(*(*func(ContextScope))(callback))(ContextScope{(*Context)(c)})
}

//export go_escapable_scope_callback
func go_escapable_scope_callback(c unsafe.Pointer, callback unsafe.Pointer){
	(*(*func(EscapableScope))(callback))(EscapableScope{ContextScope{(*Context)(c)}})
}

func (c *Context) Scope(callback func(ContextScope)) {
	C.V8_Context_Scope(c.self, unsafe.Pointer(c), unsafe.Pointer(&callback))
}

func (c *Context) GetEngine() *Engine{
	return c.engine
}

func (c *Context) EscapableScope(callback func(EscapableScope)){
	C.V8_Escapable_Scope(c.self, unsafe.Pointer(c), unsafe.Pointer(&callback))
}

func (c *Context) SetSecurityToken(value *Value){
	C.V8_Context_SetSecurityToken(c.self, value.self)
}

func (c *Context) GetSecurityToken() *Value {
	return newValue(c.GetEngine(), C.V8_Context_GetSecurityToken(c.self))
}

func (c *Context) UseDefaultSecurityToken() {
	C.V8_Context_UseDefaultSecurityToken(c.self)
}

func (c *Context) GetEmbedderData(index int) *Value {
	return newValue(c.GetEngine(), C.V8_Context_GetEmbedderData(c.self,C.int(index)))
}

func (c *Context) SetEmbedderData(index int, value *Value) {
	C.V8_Context_SetEmbedderData(c.self, C.int(index), value.self)
}

//export go_try_catch_callback
func go_try_catch_callback(callback unsafe.Pointer) {
	(*(*func())(callback))()
}

func escape(s string) string {
	output := ""
	for _, r := range s {
		switch r {
		case '"':
			output += "\\\""
		case '\\':
			output += "\\\\"
		case '/':
			output += "\\/"
		case '\n':
			output += "\\n"
		case '\r':
			output += "\\r"
		case '\t':
			output += "\\t"
		case '\b':
			output += "\\b"
		case '\f':
			output += "\\f"
		default:
			output += string(r)
		}
	}
	return output
}

func (es EscapableScope) Escape(escontext *Context) *Context {
	self := C.V8_Escapable_Escape(escontext.self)
	if self == nil {
		return nil
	}
	e := es.GetEngine()
	result := &Context{
		self:   self,
		engine: e,
	}

	runtime.SetFinalizer(result, func(c *Context) {
		if traceDispose {
			println("v8.Context.Dispose()", c.self)
		}
		C.V8_DisposeContext(c.self)
	})

	return result
	//return newValue(es.GetEngine(), C.V8_Escapable_Escape())
}

func (cs ContextScope) ThrowException(err string) {
	cs.Eval(`throw "` + escape(err) + `"`)
	//
	// TODO: use Isolate::ThrowException() will make FunctionTemplate::GetFunction() returns NULL, why?
	//
	//errPtr := unsafe.Pointer((*reflect.StringHeader)(unsafe.Pointer(&err)).Data)
	//C.V8_Context_ThrowException(cs.context.self, (*C.char)(errPtr), C.int(len(err)))
}

func (cs ContextScope) ThrowException2(value *Value) {
	C.V8_Context_ThrowException2(value.self)
}

func (cs ContextScope) TryCatch(callback func()) *Message {
	msg := C.V8_Context_TryCatch(cs.context.self, unsafe.Pointer(&callback))
	if msg == nil {
		return nil
	}
	return (*Message)(msg)
}

type Exception struct {
	*Value
	*Message
}

func (cs ContextScope) TryCatchException(callback func()) *Exception {
	e := C.V8_Context_TryCatchException(cs.context.self, unsafe.Pointer(&callback))
	if e == nil {
		return nil
	}

	excep := (*exception)(e)
	val := newValue(cs.GetEngine(), excep.Pointer)
	if val == nil {
		return nil
	}

	return &Exception{val, excep.Message}
}

func (cs ContextScope) Global() *Object {
	return newValue(cs.GetEngine(), C.V8_Context_Global(cs.context.self)).ToObject()
}
<|endoftext|>


<commit_before>package jsoniter

import (
	"strconv"
)

var intDigits []int8

const uint32SafeToMultiply10 = uint32(0xffffffff) / 10 - 1
const uint64SafeToMultiple10 = uint64(0xffffffffffffffff) / 10 - 1
const int64Max = uint64(0x7fffffffffffffff)
const int32Max = uint32(0x7fffffff)
const int16Max = uint32(0x7fff)
const uint16Max = uint32(0xffff)
const int8Max = uint32(0x7fff)
const uint8Max = uint32(0xffff)

func init() {
	intDigits = make([]int8, 256)
	for i := 0; i < len(floatDigits); i++ {
		intDigits[i] = invalidCharForNumber
	}
	for i := int8('0'); i <= int8('9'); i++ {
		intDigits[i] = i - int8('0')
	}
}

func (iter *Iterator) ReadUint() uint {
	return uint(iter.ReadUint64())
}

func (iter *Iterator) ReadInt() int {
	return int(iter.ReadInt64())
}

func (iter *Iterator) ReadInt8() (ret int8) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > int8Max + 1 {
			iter.reportError("ReadInt8", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return -int8(val)
	} else {
		val := iter.readUint32(c)
		if val > int8Max {
			iter.reportError("ReadInt8", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return int8(val)
	}
}

func (iter *Iterator) ReadUint8() (ret uint8) {
	val := iter.readUint32(iter.nextToken())
	if val > uint8Max {
		iter.reportError("ReadUint8", "overflow: " + strconv.FormatInt(int64(val), 10))
		return
	}
	return uint8(val)
}

func (iter *Iterator) ReadInt16() (ret int16) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > int16Max + 1 {
			iter.reportError("ReadInt16", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return -int16(val)
	} else {
		val := iter.readUint32(c)
		if val > int16Max {
			iter.reportError("ReadInt16", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return int16(val)
	}
}

func (iter *Iterator) ReadUint16() (ret uint16) {
	val := iter.readUint32(iter.nextToken())
	if val > uint16Max {
		iter.reportError("ReadUint16", "overflow: " + strconv.FormatInt(int64(val), 10))
		return
	}
	return uint16(val)
}

func (iter *Iterator) ReadInt32() (ret int32) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > int32Max + 1 {
			iter.reportError("ReadInt32", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return -int32(val)
	} else {
		val := iter.readUint32(c)
		if val > int32Max {
			iter.reportError("ReadInt32", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return int32(val)
	}
}

func (iter *Iterator) ReadUint32() (ret uint32) {
	return iter.readUint32(iter.nextToken())
}

func (iter *Iterator) readUint32(c byte) (ret uint32) {
	ind := intDigits[c]
	if ind == 0 {
		return 0 // single zero
	}
	if ind == invalidCharForNumber {
		iter.reportError("readUint32", "unexpected character: " + string([]byte{byte(ind)}))
		return
	}
	value := uint32(ind)
	if iter.tail - iter.head > 10 {
		i := iter.head
		ind2 := intDigits[iter.buf[i]]
		if ind2 == invalidCharForNumber {
			iter.head = i
			return value
		}
		i++
		ind3 := intDigits[iter.buf[i]]
		if ind3 == invalidCharForNumber {
			iter.head = i
			return value * 10 + uint32(ind2)
		}
		//iter.head = i + 1
		//value = value * 100 + uint32(ind2) * 10 + uint32(ind3)
		i++
		ind4 := intDigits[iter.buf[i]]
		if ind4 == invalidCharForNumber {
			iter.head = i
			return value * 100 + uint32(ind2) * 10 + uint32(ind3)
		}
		i++
		ind5 := intDigits[iter.buf[i]]
		if ind5 == invalidCharForNumber {
			iter.head = i
			return value * 1000 + uint32(ind2) * 100 + uint32(ind3) * 10 + uint32(ind4)
		}
		i++
		ind6 := intDigits[iter.buf[i]]
		if ind6 == invalidCharForNumber {
			iter.head = i
			return value * 10000 + uint32(ind2) * 1000 + uint32(ind3) * 100 + uint32(ind4) * 10 + uint32(ind5)
		}
		i++
		ind7 := intDigits[iter.buf[i]]
		if ind7 == invalidCharForNumber {
			iter.head = i
			return value * 100000 + uint32(ind2) * 10000 + uint32(ind3) * 1000 + uint32(ind4) * 100 + uint32(ind5) * 10 + uint32(ind6)
		}
		i++
		ind8 := intDigits[iter.buf[i]]
		if ind8 == invalidCharForNumber {
			iter.head = i
			return value * 1000000 + uint32(ind2) * 100000 + uint32(ind3) * 10000 + uint32(ind4) * 1000 + uint32(ind5) * 100 + uint32(ind6) * 10 + uint32(ind7)
		}
		i++
		ind9 := intDigits[iter.buf[i]]
		value = value * 10000000 + uint32(ind2) * 1000000 + uint32(ind3) * 100000 + uint32(ind4) * 10000 + uint32(ind5) * 1000 + uint32(ind6) * 100 + uint32(ind7) * 10 + uint32(ind8)
		iter.head = i
		if ind9 == invalidCharForNumber {
			return value
		}
	}
	for {
		for i := iter.head; i < iter.tail; i++ {
			ind = intDigits[iter.buf[i]]
			if ind == invalidCharForNumber {
				iter.head = i
				return value
			}
			if value > uint32SafeToMultiply10 {
				value2 := (value << 3) + (value << 1) + uint32(ind)
				if value2 < value {
					iter.reportError("readUint32", "overflow")
					return
				} else {
					value = value2
					continue
				}
			}
			value = (value << 3) + (value << 1) + uint32(ind)
		}
		if (!iter.loadMore()) {
			return value
		}
	}
}

func (iter *Iterator) ReadInt64() (ret int64) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint64(iter.readByte())
		if val > int64Max + 1 {
			iter.reportError("ReadInt64", "overflow: " + strconv.FormatUint(uint64(val), 10))
			return
		}
		return -int64(val)
	} else {
		val := iter.readUint64(c)
		if val > int64Max {
			iter.reportError("ReadInt64", "overflow: " + strconv.FormatUint(uint64(val), 10))
			return
		}
		return int64(val)
	}
}

func (iter *Iterator) ReadUint64() uint64 {
	return iter.readUint64(iter.nextToken())
}

func (iter *Iterator) readUint64(c byte) (ret uint64) {
	ind := intDigits[c]
	if ind == 0 {
		return 0 // single zero
	}
	if ind == invalidCharForNumber {
		iter.reportError("readUint64", "unexpected character: " + string([]byte{byte(ind)}))
		return
	}
	value := uint64(ind)
	for {
		for i := iter.head; i < iter.tail; i++ {
			ind = intDigits[iter.buf[i]]
			if ind == invalidCharForNumber {
				iter.head = i
				return value
			}
			if value > uint64SafeToMultiple10 {
				value2 := (value << 3) + (value << 1) + uint64(ind)
				if value2 < value {
					iter.reportError("readUint64", "overflow")
					return
				} else {
					value = value2
					continue
				}
			}
			value = (value << 3) + (value << 1) + uint64(ind)
		}
		if (!iter.loadMore()) {
			return value
		}
	}
}
<commit_msg>把floatDigits改为intDigits<commit_after>package jsoniter

import (
	"strconv"
)

var intDigits []int8

const uint32SafeToMultiply10 = uint32(0xffffffff) / 10 - 1
const uint64SafeToMultiple10 = uint64(0xffffffffffffffff) / 10 - 1
const int64Max = uint64(0x7fffffffffffffff)
const int32Max = uint32(0x7fffffff)
const int16Max = uint32(0x7fff)
const uint16Max = uint32(0xffff)
const int8Max = uint32(0x7fff)
const uint8Max = uint32(0xffff)

func init() {
	intDigits = make([]int8, 256)
	for i := 0; i < len(intDigits); i++ {
		intDigits[i] = invalidCharForNumber
	}
	for i := int8('0'); i <= int8('9'); i++ {
		intDigits[i] = i - int8('0')
	}
}

func (iter *Iterator) ReadUint() uint {
	return uint(iter.ReadUint64())
}

func (iter *Iterator) ReadInt() int {
	return int(iter.ReadInt64())
}

func (iter *Iterator) ReadInt8() (ret int8) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > int8Max + 1 {
			iter.reportError("ReadInt8", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return -int8(val)
	} else {
		val := iter.readUint32(c)
		if val > int8Max {
			iter.reportError("ReadInt8", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return int8(val)
	}
}

func (iter *Iterator) ReadUint8() (ret uint8) {
	val := iter.readUint32(iter.nextToken())
	if val > uint8Max {
		iter.reportError("ReadUint8", "overflow: " + strconv.FormatInt(int64(val), 10))
		return
	}
	return uint8(val)
}

func (iter *Iterator) ReadInt16() (ret int16) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > int16Max + 1 {
			iter.reportError("ReadInt16", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return -int16(val)
	} else {
		val := iter.readUint32(c)
		if val > int16Max {
			iter.reportError("ReadInt16", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return int16(val)
	}
}

func (iter *Iterator) ReadUint16() (ret uint16) {
	val := iter.readUint32(iter.nextToken())
	if val > uint16Max {
		iter.reportError("ReadUint16", "overflow: " + strconv.FormatInt(int64(val), 10))
		return
	}
	return uint16(val)
}

func (iter *Iterator) ReadInt32() (ret int32) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > int32Max + 1 {
			iter.reportError("ReadInt32", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return -int32(val)
	} else {
		val := iter.readUint32(c)
		if val > int32Max {
			iter.reportError("ReadInt32", "overflow: " + strconv.FormatInt(int64(val), 10))
			return
		}
		return int32(val)
	}
}

func (iter *Iterator) ReadUint32() (ret uint32) {
	return iter.readUint32(iter.nextToken())
}

func (iter *Iterator) readUint32(c byte) (ret uint32) {
	ind := intDigits[c]
	if ind == 0 {
		return 0 // single zero
	}
	if ind == invalidCharForNumber {
		iter.reportError("readUint32", "unexpected character: " + string([]byte{byte(ind)}))
		return
	}
	value := uint32(ind)
	if iter.tail - iter.head > 10 {
		i := iter.head
		ind2 := intDigits[iter.buf[i]]
		if ind2 == invalidCharForNumber {
			iter.head = i
			return value
		}
		i++
		ind3 := intDigits[iter.buf[i]]
		if ind3 == invalidCharForNumber {
			iter.head = i
			return value * 10 + uint32(ind2)
		}
		//iter.head = i + 1
		//value = value * 100 + uint32(ind2) * 10 + uint32(ind3)
		i++
		ind4 := intDigits[iter.buf[i]]
		if ind4 == invalidCharForNumber {
			iter.head = i
			return value * 100 + uint32(ind2) * 10 + uint32(ind3)
		}
		i++
		ind5 := intDigits[iter.buf[i]]
		if ind5 == invalidCharForNumber {
			iter.head = i
			return value * 1000 + uint32(ind2) * 100 + uint32(ind3) * 10 + uint32(ind4)
		}
		i++
		ind6 := intDigits[iter.buf[i]]
		if ind6 == invalidCharForNumber {
			iter.head = i
			return value * 10000 + uint32(ind2) * 1000 + uint32(ind3) * 100 + uint32(ind4) * 10 + uint32(ind5)
		}
		i++
		ind7 := intDigits[iter.buf[i]]
		if ind7 == invalidCharForNumber {
			iter.head = i
			return value * 100000 + uint32(ind2) * 10000 + uint32(ind3) * 1000 + uint32(ind4) * 100 + uint32(ind5) * 10 + uint32(ind6)
		}
		i++
		ind8 := intDigits[iter.buf[i]]
		if ind8 == invalidCharForNumber {
			iter.head = i
			return value * 1000000 + uint32(ind2) * 100000 + uint32(ind3) * 10000 + uint32(ind4) * 1000 + uint32(ind5) * 100 + uint32(ind6) * 10 + uint32(ind7)
		}
		i++
		ind9 := intDigits[iter.buf[i]]
		value = value * 10000000 + uint32(ind2) * 1000000 + uint32(ind3) * 100000 + uint32(ind4) * 10000 + uint32(ind5) * 1000 + uint32(ind6) * 100 + uint32(ind7) * 10 + uint32(ind8)
		iter.head = i
		if ind9 == invalidCharForNumber {
			return value
		}
	}
	for {
		for i := iter.head; i < iter.tail; i++ {
			ind = intDigits[iter.buf[i]]
			if ind == invalidCharForNumber {
				iter.head = i
				return value
			}
			if value > uint32SafeToMultiply10 {
				value2 := (value << 3) + (value << 1) + uint32(ind)
				if value2 < value {
					iter.reportError("readUint32", "overflow")
					return
				} else {
					value = value2
					continue
				}
			}
			value = (value << 3) + (value << 1) + uint32(ind)
		}
		if (!iter.loadMore()) {
			return value
		}
	}
}

func (iter *Iterator) ReadInt64() (ret int64) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint64(iter.readByte())
		if val > int64Max + 1 {
			iter.reportError("ReadInt64", "overflow: " + strconv.FormatUint(uint64(val), 10))
			return
		}
		return -int64(val)
	} else {
		val := iter.readUint64(c)
		if val > int64Max {
			iter.reportError("ReadInt64", "overflow: " + strconv.FormatUint(uint64(val), 10))
			return
		}
		return int64(val)
	}
}

func (iter *Iterator) ReadUint64() uint64 {
	return iter.readUint64(iter.nextToken())
}

func (iter *Iterator) readUint64(c byte) (ret uint64) {
	ind := intDigits[c]
	if ind == 0 {
		return 0 // single zero
	}
	if ind == invalidCharForNumber {
		iter.reportError("readUint64", "unexpected character: " + string([]byte{byte(ind)}))
		return
	}
	value := uint64(ind)
	for {
		for i := iter.head; i < iter.tail; i++ {
			ind = intDigits[iter.buf[i]]
			if ind == invalidCharForNumber {
				iter.head = i
				return value
			}
			if value > uint64SafeToMultiple10 {
				value2 := (value << 3) + (value << 1) + uint64(ind)
				if value2 < value {
					iter.reportError("readUint64", "overflow")
					return
				} else {
					value = value2
					continue
				}
			}
			value = (value << 3) + (value << 1) + uint64(ind)
		}
		if (!iter.loadMore()) {
			return value
		}
	}
}
<|endoftext|>


<commit_before>package jsonschema

import (
	"encoding/json"
	"fmt"
	"strings"
)

type properties map[string]json.RawMessage

func (p properties) Validate(v interface{}) []ValidationError {
	var valErrs []ValidationError
	for schemaKey, schemaValue := range p {
		dataMap, ok := v.(map[string]interface{})
		if !ok {
			continue
		}
		if dataValue, ok := dataMap[schemaKey]; ok {
			var schema Schema
			err := json.Unmarshal(schemaValue, &schema)
			if err != nil {
				break
			}
			valErrs = append(valErrs, schema.Validate(dataValue)...)
		}
	}
	return valErrs
}

type minimum struct {
	json.Number
}

func (m minimum) Validate(unnormalized interface{}) []ValidationError {
	v, err := normalizeNumber(unnormalized)
	if err != nil {
		return []ValidationError{ValidationError{err.Error()}}
	}
	var isLarger bool
	switch n := v.(type) {
	case int64:
		isLarger = m.isLargerThanInt(n)
	case float64:
		isLarger = m.isLargerThanFloat(n)
	}
	if isLarger {
		minErr := ValidationError{fmt.Sprintf("Value must be larger than %s.", m)}
		return []ValidationError{minErr}
	}
	return nil
}

func (m *minimum) UnmarshalJSON(bts []byte) error {
	var holder json.Number
	if err := json.Unmarshal(bts, &holder); err != nil {
		return err
	}
	*m = minimum{holder}
	return nil
}

func (m minimum) isLargerThanInt(n int64) bool {
	if strings.Contains(m.String(), ".") {
		flt, err := m.Float64()
		if err != nil {
			return false
		}
		if flt > float64(n) {
			return true
		}
	} else {
		intg, err := m.Int64()
		if err != nil {
			return false
		}
		if intg > n {
			return true
		}
	}
	return false
}

func (m minimum) isLargerThanFloat(n float64) bool {
	flt, err := m.Float64()
	if err != nil {
		return false
	}
	if flt > n {
		return true
	}
	return false
}
<commit_msg>Eliminate unnecessary variable.<commit_after>package jsonschema

import (
	"encoding/json"
	"fmt"
	"strings"
)

type properties map[string]json.RawMessage

func (p properties) Validate(v interface{}) []ValidationError {
	var valErrs []ValidationError
	for schemaKey, schemaValue := range p {
		dataMap, ok := v.(map[string]interface{})
		if !ok {
			continue
		}
		if dataValue, ok := dataMap[schemaKey]; ok {
			var schema Schema
			err := json.Unmarshal(schemaValue, &schema)
			if err != nil {
				break
			}
			valErrs = append(valErrs, schema.Validate(dataValue)...)
		}
	}
	return valErrs
}

type minimum struct {
	json.Number
}

func (m minimum) Validate(unnormalized interface{}) []ValidationError {
	v, err := normalizeNumber(unnormalized)
	if err != nil {
		return []ValidationError{ValidationError{err.Error()}}
	}
	var isLarger bool
	switch n := v.(type) {
	case int64:
		isLarger = m.isLargerThanInt(n)
	case float64:
		isLarger = m.isLargerThanFloat(n)
	}
	if isLarger {
		minErr := ValidationError{fmt.Sprintf("Value must be larger than %s.", m)}
		return []ValidationError{minErr}
	}
	return nil
}

func (m *minimum) UnmarshalJSON(b []byte) error {
	if err := json.Unmarshal(b, &m.Number); err != nil {
		return err
	}
	return nil
}

func (m minimum) isLargerThanInt(n int64) bool {
	if strings.Contains(m.String(), ".") {
		flt, err := m.Float64()
		if err != nil {
			return false
		}
		if flt > float64(n) {
			return true
		}
	} else {
		intg, err := m.Int64()
		if err != nil {
			return false
		}
		if intg > n {
			return true
		}
	}
	return false
}

func (m minimum) isLargerThanFloat(n float64) bool {
	flt, err := m.Float64()
	if err != nil {
		return false
	}
	if flt > n {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"time"
)

import "github.com/stathat/consistent"

// BUFFERSIZE controls the size of the [...]byte array used to read UDP data
// off the wire and into local memory.  Metrics are separated by \n
// characters.  This buffer is passed to a handler to proxy out the metrics
// to the real statsd daemons.
const BUFFERSIZE int = 1 * 1024 * 1024 // 1MiB

// packetLen is the size in bytes of data we stuff into one packet before
// sending it to statsd.  This must be lower than the MTU, IPv4 header size
// and UDP header size.
const packetLen int = 1000

// prefix is the string that will be prefixed onto self generated stats.
// Such as <prefix>.statsProcessed.  Default is "statsrelay"
var prefix string

// udpAddr is a mapping of HOST:PORT:INSTANCE to a UDPAddr object
var udpAddr = make(map[string]*net.UDPAddr)

// hashRing is our consistent hashing ring.
var hashRing *consistent.Consistent

// totalMetrics tracks the totall number of metrics processed
var totalMetrics int = 0

// totalMetricsLock is a mutex gaurding totalMetrics
var totalMetricsLock sync.Mutex

// Time we began
var epochTime int64

// Verbose/Debug output
var verbose bool

// sockBufferMaxSize() returns the maximum size that the UDP receive buffer
// in the kernel can be set to.  In bytes.
func sockBufferMaxSize() int {

	// XXX: This is Linux-only most likely
	data, err := ioutil.ReadFile("/proc/sys/net/core/rmem_max")
	if err != nil {
		log.Panicln(err)
	}

	data = bytes.TrimRight(data, "\n\r")
	i, err := strconv.Atoi(string(data))
	if err != nil {
		log.Printf("Could not parse /proc/sys/net/core/rmem_max\n")
		log.Fatalln(err)
	}

	return i
}

// getMetricName() parses the given []byte metric as a string, extracts
// the metric key name and returns it as a string.
func getMetricName(metric []byte) string {
	// statsd metrics are of the form:
	//    KEY:VALUE|TYPE|RATE
	length := bytes.IndexByte(metric, byte(':'))
	return string(metric[:length])
}

// sendPacket takes a []byte and writes that directly to a UDP socket
// that was assigned for target.
func sendPacket(buff []byte, target string) {
	conn, err := net.ListenUDP("udp", nil)
	if err != nil {
		log.Panicln(err)
	}

	conn.WriteToUDP(buff, udpAddr[target])
	conn.Close()
}

// buildPacketMap() is a helper function to initiallize a map that represents
// a UDP packet currently being built for each destination we proxy to.  As
// Go forbids taking the address of an object in a map or array so the
// bytes.Buffer object must be stored in the map as a pointer rather than
// a direct object in order to call the pointer methods on it.
func buildPacketMap() map[string]*bytes.Buffer {
	members := hashRing.Members()
	hash := make(map[string]*bytes.Buffer, len(members))

	for _, v := range members {
		hash[v] = new(bytes.Buffer)
	}

	return hash
}

// handleBuff() sorts through a full buffer of metrics and batches metrics
// to remote statsd daemons using a consistent hash.
func handleBuff(buff []byte) {
	packets := buildPacketMap()
	sep := []byte("\n")
	numMetrics := 0
	statsMetric := prefix + ".statsProcessed"

	for offset := 0; offset < len(buff); {
	loop:
		for offset < len(buff) {
			// Find our next value
			switch buff[offset] {
			case '\n':
				offset++
			case '\r':
				offset++
			case 0:
				offset++
			default:
				break loop
			}
		}

		size := bytes.IndexByte(buff[offset:], '\n')
		if size == -1 {
			// last metric in buffer
			size = len(buff) - offset
		}
		target, err := hashRing.Get(getMetricName(buff[offset : offset+size]))
		if err != nil {
			log.Panicln(err)
		}

		// check built packet size and send if metric doesn't fit
		if packets[target].Len()+size > packetLen {
			sendPacket(packets[target].Bytes(), target)
			packets[target].Reset()
		}
		// add to packet
		packets[target].Write(buff[offset : offset+size])
		packets[target].Write(sep)

		numMetrics++
		offset = offset + size + 1
	}

	if numMetrics == 0 {
		// if we haven't handled any metrics, then don't update counters/stats
		// or send packets
		return
	}

	// Update interal counter
	totalMetricsLock.Lock()
	totalMetrics = totalMetrics + numMetrics
	totalMetricsLock.Unlock()

	// Handle reporting our own stats
	stats := fmt.Sprintf("%s:%d|c\n", statsMetric, numMetrics)
	target, err := hashRing.Get(statsMetric)
	if err != nil {
		log.Panicln(err)
	}
	if packets[target].Len()+len(stats) > packetLen {
		sendPacket(packets[target].Bytes(), target)
		packets[target].Reset()
	}
	packets[target].Write([]byte(stats))

	// Empty out any remaining data
	for _, target := range hashRing.Members() {
		if packets[target].Len() > 0 {
			sendPacket(packets[target].Bytes(), target)
		}
	}

	if verbose {
		log.Printf("Procssed %d metrics. Running total: %d. Metrics/sec: %d\n",
			numMetrics, totalMetrics,
			int64(totalMetrics)/(time.Now().Unix()-epochTime))
	}
}

// readUDP() a goroutine that just reads data off of a UDP socket and fills
// buffers.  Once a buffer is full, it passes it to handleBuff().
func readUDP(ip string, port int, c chan []byte) {
	var buff *[BUFFERSIZE]byte
	var offset int
	var timeout bool
	var addr = net.UDPAddr{
		Port: port,
		IP:   net.ParseIP(ip),
	}

	log.Printf("Listening on %s:%d\n", ip, port)
	sock, err := net.ListenUDP("udp", &addr)
	if err != nil {
		log.Printf("Error opening UDP socket.\n")
		log.Fatalln(err)
	}
	defer sock.Close()

	log.Printf("Setting socket read buffer size to: %d\n", sockBufferMaxSize())
	err = sock.SetReadBuffer(sockBufferMaxSize())
	if err != nil {
		log.Printf("Unable to set read buffer size on socket.  Non-fatal.")
		log.Println(err)
	}
	err = sock.SetDeadline(time.Now().Add(time.Second))
	if err != nil {
		log.Printf("Unable to set timeout on socket.\n")
		log.Fatalln(err)
	}

	if verbose {
		log.Printf("Rock and Roll!\n")
	}
	for {
		if buff == nil {
			buff = new([BUFFERSIZE]byte)
			offset = 0
			timeout = false
		}

		i, err := sock.Read(buff[offset:])
		if err == nil {
			buff[offset+i] = '\n'
			offset = offset + i + 1
		} else if err.(net.Error).Timeout() {
			timeout = true
			err = sock.SetDeadline(time.Now().Add(time.Second))
			if err != nil {
				log.Panicln(err)
			}
		} else {
			log.Printf("Read Error: %s\n", err)
			continue
		}

		if offset > BUFFERSIZE-4096 || timeout {
			// Approching make buff size
			// we use a 4KiB margin
			c <- buff[:offset]
			buff = nil
		}
	}
}

// runServer() runs and manages this daemon, deals with OS signals, and handles
// communication channels.
func runServer(host string, port int) {
	var c chan []byte = make(chan []byte, 256)
	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	var sig chan os.Signal = make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, os.Kill)

	// read incming UDP packets
	go readUDP(host, port, c)

	for {
		select {
		case buff := <-c:
			//fmt.Printf("Handling %d length buffer...\n", len(buff))
			go handleBuff(buff)
		case <-sig:
			log.Printf("Signal received.  Shutting down...\n")
			log.Printf("Received %d metrics.\n", totalMetrics)
			return
		}
	}
}

func main() {
	var bindAddress string
	var port int

	flag.IntVar(&port, "port", 9125, "Port to listen on")
	flag.IntVar(&port, "p", 9125, "Port to listen on")

	flag.StringVar(&bindAddress, "bind", "0.0.0.0", "IP Address to listen on")
	flag.StringVar(&bindAddress, "b", "0.0.0.0", "IP Address to listen on")

	flag.StringVar(&prefix, "prefix", "statsrelay", "The prefix to use with self generated stats")

	flag.BoolVar(&verbose, "verbose", false, "Verbose output")
	flag.BoolVar(&verbose, "v", false, "Verbose output")

	flag.Parse()

	if len(flag.Args()) == 0 {
		log.Fatalf("One or most host specifications are needed to locate statsd daemons.\n")
	}

	hashRing = consistent.New()
	hashRing.NumberOfReplicas = 1

	for _, v := range flag.Args() {
		var addr *net.UDPAddr
		var err error
		host := strings.Split(v, ":")

		switch len(host) {
		case 1:
			log.Printf("Invalid statsd location: %s\n", v)
			log.Fatalf("Must be of the form HOST:PORT or HOST:PORT:INSTANCE\n")
		case 2:
			addr, err = net.ResolveUDPAddr("udp", v)
			if err != nil {
				log.Printf("Error parsing HOST:PORT \"%s\"\n", v)
				log.Fatalf("%s\n", err.Error())
			}
		case 3:
			addr, err = net.ResolveUDPAddr("udp", host[0]+":"+host[1])
			if err != nil {
				log.Printf("Error parsing HOST:PORT:INSTANCE \"%s\"\n", v)
				log.Fatalf("%s\n", err.Error())
			}
		default:
			log.Fatalf("Unrecongnized host specification: %s\n", v)
		}

		if addr != nil {
			udpAddr[v] = addr
			hashRing.Add(v)
		}
	}

	epochTime = time.Now().Unix()
	runServer(bindAddress, port)

	log.Printf("Normal shutdown.\n")
}
<commit_msg>statshat has relocated their Go libraries<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"time"
)

import "stathat.com/c/consistent"

// BUFFERSIZE controls the size of the [...]byte array used to read UDP data
// off the wire and into local memory.  Metrics are separated by \n
// characters.  This buffer is passed to a handler to proxy out the metrics
// to the real statsd daemons.
const BUFFERSIZE int = 1 * 1024 * 1024 // 1MiB

// packetLen is the size in bytes of data we stuff into one packet before
// sending it to statsd.  This must be lower than the MTU, IPv4 header size
// and UDP header size.
const packetLen int = 1000

// prefix is the string that will be prefixed onto self generated stats.
// Such as <prefix>.statsProcessed.  Default is "statsrelay"
var prefix string

// udpAddr is a mapping of HOST:PORT:INSTANCE to a UDPAddr object
var udpAddr = make(map[string]*net.UDPAddr)

// hashRing is our consistent hashing ring.
var hashRing *consistent.Consistent

// totalMetrics tracks the totall number of metrics processed
var totalMetrics int = 0

// totalMetricsLock is a mutex gaurding totalMetrics
var totalMetricsLock sync.Mutex

// Time we began
var epochTime int64

// Verbose/Debug output
var verbose bool

// sockBufferMaxSize() returns the maximum size that the UDP receive buffer
// in the kernel can be set to.  In bytes.
func sockBufferMaxSize() int {

	// XXX: This is Linux-only most likely
	data, err := ioutil.ReadFile("/proc/sys/net/core/rmem_max")
	if err != nil {
		log.Panicln(err)
	}

	data = bytes.TrimRight(data, "\n\r")
	i, err := strconv.Atoi(string(data))
	if err != nil {
		log.Printf("Could not parse /proc/sys/net/core/rmem_max\n")
		log.Fatalln(err)
	}

	return i
}

// getMetricName() parses the given []byte metric as a string, extracts
// the metric key name and returns it as a string.
func getMetricName(metric []byte) string {
	// statsd metrics are of the form:
	//    KEY:VALUE|TYPE|RATE
	length := bytes.IndexByte(metric, byte(':'))
	return string(metric[:length])
}

// sendPacket takes a []byte and writes that directly to a UDP socket
// that was assigned for target.
func sendPacket(buff []byte, target string) {
	conn, err := net.ListenUDP("udp", nil)
	if err != nil {
		log.Panicln(err)
	}

	conn.WriteToUDP(buff, udpAddr[target])
	conn.Close()
}

// buildPacketMap() is a helper function to initiallize a map that represents
// a UDP packet currently being built for each destination we proxy to.  As
// Go forbids taking the address of an object in a map or array so the
// bytes.Buffer object must be stored in the map as a pointer rather than
// a direct object in order to call the pointer methods on it.
func buildPacketMap() map[string]*bytes.Buffer {
	members := hashRing.Members()
	hash := make(map[string]*bytes.Buffer, len(members))

	for _, v := range members {
		hash[v] = new(bytes.Buffer)
	}

	return hash
}

// handleBuff() sorts through a full buffer of metrics and batches metrics
// to remote statsd daemons using a consistent hash.
func handleBuff(buff []byte) {
	packets := buildPacketMap()
	sep := []byte("\n")
	numMetrics := 0
	statsMetric := prefix + ".statsProcessed"

	for offset := 0; offset < len(buff); {
	loop:
		for offset < len(buff) {
			// Find our next value
			switch buff[offset] {
			case '\n':
				offset++
			case '\r':
				offset++
			case 0:
				offset++
			default:
				break loop
			}
		}

		size := bytes.IndexByte(buff[offset:], '\n')
		if size == -1 {
			// last metric in buffer
			size = len(buff) - offset
		}
		target, err := hashRing.Get(getMetricName(buff[offset : offset+size]))
		if err != nil {
			log.Panicln(err)
		}

		// check built packet size and send if metric doesn't fit
		if packets[target].Len()+size > packetLen {
			sendPacket(packets[target].Bytes(), target)
			packets[target].Reset()
		}
		// add to packet
		packets[target].Write(buff[offset : offset+size])
		packets[target].Write(sep)

		numMetrics++
		offset = offset + size + 1
	}

	if numMetrics == 0 {
		// if we haven't handled any metrics, then don't update counters/stats
		// or send packets
		return
	}

	// Update interal counter
	totalMetricsLock.Lock()
	totalMetrics = totalMetrics + numMetrics
	totalMetricsLock.Unlock()

	// Handle reporting our own stats
	stats := fmt.Sprintf("%s:%d|c\n", statsMetric, numMetrics)
	target, err := hashRing.Get(statsMetric)
	if err != nil {
		log.Panicln(err)
	}
	if packets[target].Len()+len(stats) > packetLen {
		sendPacket(packets[target].Bytes(), target)
		packets[target].Reset()
	}
	packets[target].Write([]byte(stats))

	// Empty out any remaining data
	for _, target := range hashRing.Members() {
		if packets[target].Len() > 0 {
			sendPacket(packets[target].Bytes(), target)
		}
	}

	if verbose {
		log.Printf("Procssed %d metrics. Running total: %d. Metrics/sec: %d\n",
			numMetrics, totalMetrics,
			int64(totalMetrics)/(time.Now().Unix()-epochTime))
	}
}

// readUDP() a goroutine that just reads data off of a UDP socket and fills
// buffers.  Once a buffer is full, it passes it to handleBuff().
func readUDP(ip string, port int, c chan []byte) {
	var buff *[BUFFERSIZE]byte
	var offset int
	var timeout bool
	var addr = net.UDPAddr{
		Port: port,
		IP:   net.ParseIP(ip),
	}

	log.Printf("Listening on %s:%d\n", ip, port)
	sock, err := net.ListenUDP("udp", &addr)
	if err != nil {
		log.Printf("Error opening UDP socket.\n")
		log.Fatalln(err)
	}
	defer sock.Close()

	log.Printf("Setting socket read buffer size to: %d\n", sockBufferMaxSize())
	err = sock.SetReadBuffer(sockBufferMaxSize())
	if err != nil {
		log.Printf("Unable to set read buffer size on socket.  Non-fatal.")
		log.Println(err)
	}
	err = sock.SetDeadline(time.Now().Add(time.Second))
	if err != nil {
		log.Printf("Unable to set timeout on socket.\n")
		log.Fatalln(err)
	}

	if verbose {
		log.Printf("Rock and Roll!\n")
	}
	for {
		if buff == nil {
			buff = new([BUFFERSIZE]byte)
			offset = 0
			timeout = false
		}

		i, err := sock.Read(buff[offset:])
		if err == nil {
			buff[offset+i] = '\n'
			offset = offset + i + 1
		} else if err.(net.Error).Timeout() {
			timeout = true
			err = sock.SetDeadline(time.Now().Add(time.Second))
			if err != nil {
				log.Panicln(err)
			}
		} else {
			log.Printf("Read Error: %s\n", err)
			continue
		}

		if offset > BUFFERSIZE-4096 || timeout {
			// Approching make buff size
			// we use a 4KiB margin
			c <- buff[:offset]
			buff = nil
		}
	}
}

// runServer() runs and manages this daemon, deals with OS signals, and handles
// communication channels.
func runServer(host string, port int) {
	var c chan []byte = make(chan []byte, 256)
	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	var sig chan os.Signal = make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, os.Kill)

	// read incming UDP packets
	go readUDP(host, port, c)

	for {
		select {
		case buff := <-c:
			//fmt.Printf("Handling %d length buffer...\n", len(buff))
			go handleBuff(buff)
		case <-sig:
			log.Printf("Signal received.  Shutting down...\n")
			log.Printf("Received %d metrics.\n", totalMetrics)
			return
		}
	}
}

func main() {
	var bindAddress string
	var port int

	flag.IntVar(&port, "port", 9125, "Port to listen on")
	flag.IntVar(&port, "p", 9125, "Port to listen on")

	flag.StringVar(&bindAddress, "bind", "0.0.0.0", "IP Address to listen on")
	flag.StringVar(&bindAddress, "b", "0.0.0.0", "IP Address to listen on")

	flag.StringVar(&prefix, "prefix", "statsrelay", "The prefix to use with self generated stats")

	flag.BoolVar(&verbose, "verbose", false, "Verbose output")
	flag.BoolVar(&verbose, "v", false, "Verbose output")

	flag.Parse()

	if len(flag.Args()) == 0 {
		log.Fatalf("One or most host specifications are needed to locate statsd daemons.\n")
	}

	hashRing = consistent.New()
	hashRing.NumberOfReplicas = 1

	for _, v := range flag.Args() {
		var addr *net.UDPAddr
		var err error
		host := strings.Split(v, ":")

		switch len(host) {
		case 1:
			log.Printf("Invalid statsd location: %s\n", v)
			log.Fatalf("Must be of the form HOST:PORT or HOST:PORT:INSTANCE\n")
		case 2:
			addr, err = net.ResolveUDPAddr("udp", v)
			if err != nil {
				log.Printf("Error parsing HOST:PORT \"%s\"\n", v)
				log.Fatalf("%s\n", err.Error())
			}
		case 3:
			addr, err = net.ResolveUDPAddr("udp", host[0]+":"+host[1])
			if err != nil {
				log.Printf("Error parsing HOST:PORT:INSTANCE \"%s\"\n", v)
				log.Fatalf("%s\n", err.Error())
			}
		default:
			log.Fatalf("Unrecongnized host specification: %s\n", v)
		}

		if addr != nil {
			udpAddr[v] = addr
			hashRing.Add(v)
		}
	}

	epochTime = time.Now().Unix()
	runServer(bindAddress, port)

	log.Printf("Normal shutdown.\n")
}
<|endoftext|>


<commit_before>package drax

import (
	"bytes"
	"fmt"
	"strings"
	"testing"

	"github.com/cpuguy83/drax/api"
	libkvstore "github.com/docker/libkv/store"
)

type testRaft struct {
	leader bool
	s      *store
}

func (r *testRaft) IsLeader() bool { return r.leader }

func (r *testRaft) LeaderCh() <-chan interface{} {
	ch := make(chan interface{})
	if r.leader {
		close(ch)
	}
	return ch
}

func (r *testRaft) GetLeader() string {
	return "test"
}

func (r *testRaft) ShutdownCh() <-chan struct{} {
	return make(chan struct{})
}

// TODO: decouple raft handling from the store itself
func (r *testRaft) Apply(b []byte) error {
	buf := bytes.NewBuffer(b)
	var req api.Request
	if err := api.Decode(&req, buf); err != nil {
		return err
	}

	switch req.Action {
	case api.Put:
		r.s.data.KV[req.Key] = &libkvstore.KVPair{
			Key:   req.Key,
			Value: req.Value,
		}
	case api.Delete:
		delete(r.s.data.KV, req.Key)
	case api.DeleteTree:
		for k := range r.s.data.KV {
			if !strings.HasPrefix(k, req.Key) {
				continue
			}
			delete(r.s.data.KV, k)
			delete(r.s.data.TTLs, k)
		}

	default:
		return fmt.Errorf("unknown action")
	}
	return nil
}

func TestStoreGet(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	if _, err := s.Get("hello"); err == nil {
		t.Fatal("expected error getting non-existent key")
	}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}

	kv, err := s.Get("hello")
	if err != nil {
		t.Fatal(err)
	}
	if kv.Key != "hello" {
		t.Fatalf("exepcted key `hello`, got %s", kv.Key)
	}
	if string(kv.Value) != "world" {
		t.Fatal("expected value `world`, got %s", string(kv.Value))
	}
}

func TestStorePut(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}
	kv, err := s.Get("hello")
	if err != nil {
		t.Fatal(err)
	}
	if kv.Key != "hello" {
		t.Fatalf("exepcted key `hello`, got %s", kv.Key)
	}
	if string(kv.Value) != "world" {
		t.Fatal("expected value `world`, got %s", string(kv.Value))
	}
}

func TestStoreDelete(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}

	if _, err := s.Get("hello"); err != nil {
		t.Fatal(err)
	}

	if err := s.Delete("hello"); err != nil {
		t.Fatal(err)
	}

	if _, err := s.Get("hello"); err == nil {
		t.Fatal("expected error looking up deleted key")
	}
}

func TestStoreExists(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	exists, err := s.Exists("hello")
	if err != nil {
		t.Fatal(err)
	}

	if exists {
		t.Fatal("expected key to not exist")
	}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}

	exists, err = s.Exists("hello")
	if err != nil {
		t.Fatal(err)
	}

	if !exists {
		t.Fatal("expected key to exist")
	}
}

func TestStoreList(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	if _, err := s.List("hello"); err == nil {
		t.Fatal("expected error looking up non-existent key")
	}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}

	if err := s.Put("hello/world", []byte("bam"), nil); err != nil {
		t.Fatal(err)
	}

	if err := s.Put("hello/batman", []byte("pow"), nil); err != nil {
		t.Fatal(err)
	}

	ls, err := s.List("hello")
	if err != nil {
		t.Fatal(err)
	}

	if len(ls) != 3 {
		t.Fatalf("expected 3 keys, got %d", len(ls))
	}
}

func TestStoreDeleteTree(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}

	if err := s.Put("hello/world", []byte("bam"), nil); err != nil {
		t.Fatal(err)
	}

	if err := s.Put("hello/batman", []byte("pow"), nil); err != nil {
		t.Fatal(err)
	}

	if err := s.DeleteTree("hello"); err != nil {
		t.Fatal(err)
	}

	if _, err := s.List("hello"); err == nil {
		t.Fatal("expected error on listing non-existig tree")
	}
}
<commit_msg>fix vet issues<commit_after>package drax

import (
	"bytes"
	"fmt"
	"strings"
	"testing"

	"github.com/cpuguy83/drax/api"
	libkvstore "github.com/docker/libkv/store"
)

type testRaft struct {
	leader bool
	s      *store
}

func (r *testRaft) IsLeader() bool { return r.leader }

func (r *testRaft) LeaderCh() <-chan interface{} {
	ch := make(chan interface{})
	if r.leader {
		close(ch)
	}
	return ch
}

func (r *testRaft) GetLeader() string {
	return "test"
}

func (r *testRaft) ShutdownCh() <-chan struct{} {
	return make(chan struct{})
}

// TODO: decouple raft handling from the store itself
func (r *testRaft) Apply(b []byte) error {
	buf := bytes.NewBuffer(b)
	var req api.Request
	if err := api.Decode(&req, buf); err != nil {
		return err
	}

	switch req.Action {
	case api.Put:
		r.s.data.KV[req.Key] = &libkvstore.KVPair{
			Key:   req.Key,
			Value: req.Value,
		}
	case api.Delete:
		delete(r.s.data.KV, req.Key)
	case api.DeleteTree:
		for k := range r.s.data.KV {
			if !strings.HasPrefix(k, req.Key) {
				continue
			}
			delete(r.s.data.KV, k)
			delete(r.s.data.TTLs, k)
		}

	default:
		return fmt.Errorf("unknown action")
	}
	return nil
}

func TestStoreGet(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	if _, err := s.Get("hello"); err == nil {
		t.Fatal("expected error getting non-existent key")
	}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}

	kv, err := s.Get("hello")
	if err != nil {
		t.Fatal(err)
	}
	if kv.Key != "hello" {
		t.Fatalf("exepcted key `hello`, got %s", kv.Key)
	}
	if string(kv.Value) != "world" {
		t.Fatalf("expected value `world`, got %s", string(kv.Value))
	}
}

func TestStorePut(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}
	kv, err := s.Get("hello")
	if err != nil {
		t.Fatal(err)
	}
	if kv.Key != "hello" {
		t.Fatalf("exepcted key `hello`, got %s", kv.Key)
	}
	if string(kv.Value) != "world" {
		t.Fatalf("expected value `world`, got %s", string(kv.Value))
	}
}

func TestStoreDelete(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}

	if _, err := s.Get("hello"); err != nil {
		t.Fatal(err)
	}

	if err := s.Delete("hello"); err != nil {
		t.Fatal(err)
	}

	if _, err := s.Get("hello"); err == nil {
		t.Fatal("expected error looking up deleted key")
	}
}

func TestStoreExists(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	exists, err := s.Exists("hello")
	if err != nil {
		t.Fatal(err)
	}

	if exists {
		t.Fatal("expected key to not exist")
	}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}

	exists, err = s.Exists("hello")
	if err != nil {
		t.Fatal(err)
	}

	if !exists {
		t.Fatal("expected key to exist")
	}
}

func TestStoreList(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	if _, err := s.List("hello"); err == nil {
		t.Fatal("expected error looking up non-existent key")
	}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}

	if err := s.Put("hello/world", []byte("bam"), nil); err != nil {
		t.Fatal(err)
	}

	if err := s.Put("hello/batman", []byte("pow"), nil); err != nil {
		t.Fatal(err)
	}

	ls, err := s.List("hello")
	if err != nil {
		t.Fatal(err)
	}

	if len(ls) != 3 {
		t.Fatalf("expected 3 keys, got %d", len(ls))
	}
}

func TestStoreDeleteTree(t *testing.T) {
	s := newStore()
	s.r = &testRaft{true, s}

	if err := s.Put("hello", []byte("world"), nil); err != nil {
		t.Fatal(err)
	}

	if err := s.Put("hello/world", []byte("bam"), nil); err != nil {
		t.Fatal(err)
	}

	if err := s.Put("hello/batman", []byte("pow"), nil); err != nil {
		t.Fatal(err)
	}

	if err := s.DeleteTree("hello"); err != nil {
		t.Fatal(err)
	}

	if _, err := s.List("hello"); err == nil {
		t.Fatal("expected error on listing non-existig tree")
	}
}
<|endoftext|>