<commit_before>package util

import (
	"github.com/ghts/ghts/lib"
	"github.com/ghts/ghts/lib/daily_price_data"
	"github.com/ghts/ghts/lib/krx_time"
	xt "github.com/ghts/ghts/xing/base"
	xing "github.com/ghts/ghts/xing/go"

	"database/sql"
	"time"
)

func F당일_일일_가격정보_수집(db *sql.DB) (에러 error) {
	if krx.F한국증시_동시호가_시간임() || krx.F한국증시_정규_거래_시간임() {
		lib.F문자열_출력("장 중에는 정확한 데이터를 수집할 수 없습니다.")
		return
	}

	daily_price_data.F일일_가격정보_테이블_생성(db)

	당일 := lib.F일자2정수(xing.F당일())
	현재가_맵, 에러 := xing.TrT8407_현물_멀티_현재가_조회_전종목()
	lib.F확인(에러)

	for 종목코드, 값 := range 현재가_맵 {
		s := new(daily_price_data.S일일_가격정보)
		s.M종목코드 = 종목코드
		s.M일자 = 당일
		s.M시가 = float64(값.M시가)
		s.M고가 = float64(값.M고가)
		s.M저가 = float64(값.M저가)
		s.M종가 = float64(값.M현재가)
		s.M거래량 = float64(값.M누적_거래량)

		종목별_일일_가격정보_모음, 에러 := daily_price_data.New종목별_일일_가격정보_모음([]*daily_price_data.S일일_가격정보{s})
		lib.F확인(에러)

		lib.F확인(종목별_일일_가격정보_모음.DB저장(db))
	}

	lib.F문자열_출력("당일 가격정보 수집 완료.")

	return nil
}

func F일개월_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	return f고정_기간_일일_가격정보_수집(db, 종목코드_모음, 31*lib.P1일)
}

func F일년_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	return f고정_기간_일일_가격정보_수집(db, 종목코드_모음, lib.P1년)
}

func f고정_기간_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string, 기간 time.Duration) (에러 error) {
	daily_price_data.F일일_가격정보_테이블_생성(db)

	시작일 := lib.F금일().Add(-1 * 기간)

	for i, 종목코드 := range 종목코드_모음 {
		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, i)
	}

	return nil
}

func F일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	var 시작일, 마지막_저장일 time.Time
	var 종목별_일일_가격정보_모음 *daily_price_data.S종목별_일일_가격정보_모음

	daily_price_data.F일일_가격정보_테이블_생성(db)

	for i, 종목코드 := range 종목코드_모음 {
		종목별_일일_가격정보_모음, 에러 = daily_price_data.New종목별_일일_가격정보_모음_DB읽기(db, 종목코드)
		lib.F확인(에러)

		// 시작일 설정
		시작일 = lib.F지금().AddDate(-30, 0, 0)
		if 에러 == nil && len(종목별_일일_가격정보_모음.M저장소) > 0 {
			// lib.S종목별_일일_가격정보_모음 는 일자 순서로 정렬되어 있음.
			마지막_저장일 = 종목별_일일_가격정보_모음.M저장소[len(종목별_일일_가격정보_모음.M저장소)-1].G일자2()
			시작일 = 마지막_저장일.AddDate(0, 0, 1)
		}

		if 시작일.After(xing.F당일()) {
			//lib.F문자열_출력("%v [%v] : 최신 데이터 업데이트.", i, 종목코드)
			continue
		} else if 시작일.After(lib.F금일().AddDate(0, 0, -14)) {
			// 데이터 수량이 1개이나 100개이나 소요 시간은 비슷함.
			시작일 = lib.F금일().AddDate(0, 0, -14)
		}

		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, i)
	}

	return nil
}

func f일일_가격정보_수집_도우미(db *sql.DB, 종목코드 string, 시작일 time.Time, i int) {
	var 종료일 time.Time

	// 종료일 설정
	if lib.F지금().After(xing.F당일().Add(15*lib.P1시간 + lib.P30분)) {
		종료일 = xing.F당일()
	} else {
		종료일 = xing.F전일()
	}

	// 시작일 오류 확인
	if 시작일 = lib.F2일자(시작일); 시작일.After(종료일) {
		return
	} else if 시작일.Equal(종료일) { // 시작일과 종료일이 같으면 수천 개의 데이터를 불러오는 현상이 있음.
		시작일 = 시작일.AddDate(0, 0, -1)
	}

	// 데이터 수집
	값_모음, 에러 := xing.TrT8413_현물_차트_일주월(종목코드, 시작일, 종료일, xt.P일주월_일)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	} else if len(값_모음) == 0 {
		lib.F체크포인트(i, 종목코드, "추가 저장할 데이터가 없음.")
		return // 추가 저장할 데이터가 없음.
	}

	일일_가격정보_슬라이스 := make([]*daily_price_data.S일일_가격정보, len(값_모음))

	for i, 일일_데이터 := range 값_모음 {
		일일_가격정보_슬라이스[i] = daily_price_data.New일일_가격정보(
			일일_데이터.M종목코드,
			일일_데이터.M일자,
			일일_데이터.M시가,
			일일_데이터.M고가,
			일일_데이터.M저가,
			일일_데이터.M종가,
			일일_데이터.M거래량)
	}

	lib.F문자열_출력("%v %v %v~%v %v개", i+1, 종목코드, 시작일.Format(lib.P일자_형식), 종료일.Format(lib.P일자_형식), len(값_모음))

	종목별_일일_가격정보_모음, 에러 := daily_price_data.New종목별_일일_가격정보_모음(일일_가격정보_슬라이스)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	}

	lib.F확인(종목별_일일_가격정보_모음.DB저장(db))
}
<commit_msg>출력 문자열 수정.<commit_after>package util

import (
	"github.com/ghts/ghts/lib"
	"github.com/ghts/ghts/lib/daily_price_data"
	"github.com/ghts/ghts/lib/krx_time"
	xt "github.com/ghts/ghts/xing/base"
	xing "github.com/ghts/ghts/xing/go"

	"database/sql"
	"time"
)

func F당일_일일_가격정보_수집(db *sql.DB) (에러 error) {
	if krx.F한국증시_동시호가_시간임() || krx.F한국증시_정규_거래_시간임() {
		lib.F문자열_출력("장 중에는 정확한 데이터를 수집할 수 없습니다.")
		return
	}

	daily_price_data.F일일_가격정보_테이블_생성(db)

	당일 := lib.F일자2정수(xing.F당일())
	현재가_맵, 에러 := xing.TrT8407_현물_멀티_현재가_조회_전종목()
	lib.F확인(에러)

	for 종목코드, 값 := range 현재가_맵 {
		s := new(daily_price_data.S일일_가격정보)
		s.M종목코드 = 종목코드
		s.M일자 = 당일
		s.M시가 = float64(값.M시가)
		s.M고가 = float64(값.M고가)
		s.M저가 = float64(값.M저가)
		s.M종가 = float64(값.M현재가)
		s.M거래량 = float64(값.M누적_거래량)

		종목별_일일_가격정보_모음, 에러 := daily_price_data.New종목별_일일_가격정보_모음([]*daily_price_data.S일일_가격정보{s})
		lib.F확인(에러)

		lib.F확인(종목별_일일_가격정보_모음.DB저장(db))
	}

	lib.F문자열_출력("당일 가격정보 수집 완료.")

	return nil
}

func F일개월_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	return f고정_기간_일일_가격정보_수집(db, 종목코드_모음, 31*lib.P1일)
}

func F일년_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	return f고정_기간_일일_가격정보_수집(db, 종목코드_모음, lib.P1년)
}

func f고정_기간_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string, 기간 time.Duration) (에러 error) {
	daily_price_data.F일일_가격정보_테이블_생성(db)

	시작일 := lib.F금일().Add(-1 * 기간)

	for i, 종목코드 := range 종목코드_모음 {
		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, i)
	}

	return nil
}

func F일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	var 시작일, 마지막_저장일 time.Time
	var 종목별_일일_가격정보_모음 *daily_price_data.S종목별_일일_가격정보_모음

	daily_price_data.F일일_가격정보_테이블_생성(db)

	for i, 종목코드 := range 종목코드_모음 {
		종목별_일일_가격정보_모음, 에러 = daily_price_data.New종목별_일일_가격정보_모음_DB읽기(db, 종목코드)
		lib.F확인(에러)

		// 시작일 설정
		시작일 = lib.F지금().AddDate(-30, 0, 0)
		if 에러 == nil && len(종목별_일일_가격정보_모음.M저장소) > 0 {
			// lib.S종목별_일일_가격정보_모음 는 일자 순서로 정렬되어 있음.
			마지막_저장일 = 종목별_일일_가격정보_모음.M저장소[len(종목별_일일_가격정보_모음.M저장소)-1].G일자2()
			시작일 = 마지막_저장일.AddDate(0, 0, 1)
		}

		if 시작일.After(xing.F당일()) {
			//lib.F문자열_출력("%v [%v] : 최신 데이터 업데이트.", i, 종목코드)
			continue
		} else if 시작일.After(lib.F금일().AddDate(0, 0, -14)) {
			// 데이터 수량이 1개이나 100개이나 소요 시간은 비슷함.
			시작일 = lib.F금일().AddDate(0, 0, -14)
		}

		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, i)
	}

	return nil
}

func f일일_가격정보_수집_도우미(db *sql.DB, 종목코드 string, 시작일 time.Time, i int) {
	var 종료일 time.Time

	// 종료일 설정
	if lib.F지금().After(xing.F당일().Add(15*lib.P1시간 + lib.P30분)) {
		종료일 = xing.F당일()
	} else {
		종료일 = xing.F전일()
	}

	// 시작일 오류 확인
	if 시작일 = lib.F2일자(시작일); 시작일.After(종료일) {
		return
	} else if 시작일.Equal(종료일) { // 시작일과 종료일이 같으면 수천 개의 데이터를 불러오는 현상이 있음.
		시작일 = 시작일.AddDate(0, 0, -1)
	}

	// 데이터 수집
	값_모음, 에러 := xing.TrT8413_현물_차트_일주월(종목코드, 시작일, 종료일, xt.P일주월_일)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	} else if len(값_모음) == 0 {
		lib.F체크포인트(i, 종목코드, "추가 저장할 데이터가 없음.")
		return // 추가 저장할 데이터가 없음.
	}

	일일_가격정보_슬라이스 := make([]*daily_price_data.S일일_가격정보, len(값_모음))

	for i, 일일_데이터 := range 값_모음 {
		일일_가격정보_슬라이스[i] = daily_price_data.New일일_가격정보(
			일일_데이터.M종목코드,
			일일_데이터.M일자,
			일일_데이터.M시가,
			일일_데이터.M고가,
			일일_데이터.M저가,
			일일_데이터.M종가,
			일일_데이터.M거래량)
	}

	lib.F문자열_출력("%v %v %v~%v %v개", i+1, xing.F종목_식별_문자열(종목코드), 시작일.Format(lib.P일자_형식), 종료일.Format(lib.P일자_형식), len(값_모음))

	종목별_일일_가격정보_모음, 에러 := daily_price_data.New종목별_일일_가격정보_모음(일일_가격정보_슬라이스)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	}

	lib.F확인(종목별_일일_가격정보_모음.DB저장(db))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"

	"golang.org/x/tools/go/ast/astutil"
)

const pkgPath = "github.com/joeshaw/gengen/generic"
const genericPkg = "generic"

var genericTypes = []string{"T", "U", "V"}

func generate(filename string, typenames ...string) ([]byte, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments)
	if err != nil {
		return nil, err
	}

	f = replace(func(node ast.Node) ast.Node {
		se, ok := node.(*ast.SelectorExpr)
		if !ok {
			return node
		}

		x, ok := se.X.(*ast.Ident)
		if !ok || x.Name != genericPkg {
			return node
		}

		for i, t := range genericTypes {
			if se.Sel.Name == t {
				return &ast.Ident{NamePos: 0, Name: typenames[i]}
				//				return ast.NewIdent(typenames[i])
			}
		}

		return node
	}, f).(*ast.File)

	if !astutil.UsesImport(f, pkgPath) {
		astutil.DeleteImport(fset, f, pkgPath)
	}

	var buf bytes.Buffer
	err = format.Node(&buf, fset, f)
	return buf.Bytes(), err
}

func main() {
	var outfile = flag.String("o", "", "output file")
	flag.Parse()

	if flag.NArg() < 2 {
		fmt.Fprintf(os.Stderr, "usage: %s [-o <output.go>] <file.go> <replacement types...>\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "example: %s -o lists.go list_gen.go int string\n", os.Args[0])
		os.Exit(1)
	}

	buf, err := generate(flag.Arg(0), flag.Args()[1:]...)
	if err != nil {
		die(err)
	}

	if *outfile == "" {
		_, err = io.Copy(os.Stdout, bytes.NewBuffer(buf))
	} else {
		err = ioutil.WriteFile(*outfile, buf, 0644)
	}

	if err != nil {
		die(err)
	}
}

func die(err error) {
	fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
	os.Exit(1)
}
<commit_msg>Clean up a commented line of code<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"

	"golang.org/x/tools/go/ast/astutil"
)

const pkgPath = "github.com/joeshaw/gengen/generic"
const genericPkg = "generic"

var genericTypes = []string{"T", "U", "V"}

func generate(filename string, typenames ...string) ([]byte, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments)
	if err != nil {
		return nil, err
	}

	f = replace(func(node ast.Node) ast.Node {
		se, ok := node.(*ast.SelectorExpr)
		if !ok {
			return node
		}

		x, ok := se.X.(*ast.Ident)
		if !ok || x.Name != genericPkg {
			return node
		}

		for i, t := range genericTypes {
			if se.Sel.Name == t {
				return &ast.Ident{NamePos: 0, Name: typenames[i]}
			}
		}

		return node
	}, f).(*ast.File)

	if !astutil.UsesImport(f, pkgPath) {
		astutil.DeleteImport(fset, f, pkgPath)
	}

	var buf bytes.Buffer
	err = format.Node(&buf, fset, f)
	return buf.Bytes(), err
}

func main() {
	var outfile = flag.String("o", "", "output file")
	flag.Parse()

	if flag.NArg() < 2 {
		fmt.Fprintf(os.Stderr, "usage: %s [-o <output.go>] <file.go> <replacement types...>\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "example: %s -o lists.go list_gen.go int string\n", os.Args[0])
		os.Exit(1)
	}

	buf, err := generate(flag.Arg(0), flag.Args()[1:]...)
	if err != nil {
		die(err)
	}

	if *outfile == "" {
		_, err = io.Copy(os.Stdout, bytes.NewBuffer(buf))
	} else {
		err = ioutil.WriteFile(*outfile, buf, 0644)
	}

	if err != nil {
		die(err)
	}
}

func die(err error) {
	fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
	os.Exit(1)
}
<|endoftext|>


<commit_before><commit_msg>Corrected the import, commented the methods<commit_after><|endoftext|>


<commit_before><commit_msg>fix cffi_slice size computation<commit_after><|endoftext|>


<commit_before>package connect

import (
	"bufio"
	"bytes"
	"github.com/subutai-io/Subutai/agent/lib/gpg"
	"github.com/subutai-io/Subutai/agent/log"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

type Key struct {
	Id  string `json:"-"`
	Key string `json:"key"`
}

func (k *Key) String() string {
	return k.Key
}

//add public key to local keyring
//call Get() before calling Store()
func (k *Key) Store() string {
	err := k.Write("epub.key")
	log.Check(log.WarnLevel, "Adding Public Key(PK) to local keyring", err)

	status := gpg.ImportPk("epub.key")
	os.Remove("epub.key")
	k.Id = ExtractKeyId(status)
	log.Debug("Found KeyID: " + k.Id)

	return status
}

func (k *Key) ExtractKeyEmail() string {
	command := exec.Command("gpg")
	stdin, err := command.StdinPipe()
	stdin.Write([]byte(k.String()))
	stdin.Close()

	out, err := command.Output()
	log.Check(log.WarnLevel, "Extracting Email from Key", err)

	scanner := bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "uid") {
			line := strings.Fields(scanner.Text())
			if len(line) > 1 {
				email := line[len(line)-1]
				return email
			}
		}
	}
	return ""
}

//write pubkey to temp file
func (k *Key) Write(file string) error {
	err := ioutil.WriteFile(file, []byte(k.String()), 0644)
	if log.Check(log.WarnLevel, "Writing Pubkey to temp file", err) {
		return err
	}
	return nil
}

func (k *Key) Remove(file string) error {
	err := os.Remove(file)
	if log.Check(log.WarnLevel, "Removing file"+file, err) {
		return err
	}
	return nil
}

func ExtractKeyId(s string) string {
	arr := strings.Split(s, " ")
	for i, word := range arr {
		if word == "key" {
			return arr[i+1][0 : len(arr[i+1])-1]
		}
	}
	return ""
}
<commit_msg>Replaced mechanism of identifying MH Public key.<commit_after>package connect

import (
	"bufio"
	"bytes"
	"github.com/subutai-io/Subutai/agent/config"
	"github.com/subutai-io/Subutai/agent/lib/gpg"
	"github.com/subutai-io/Subutai/agent/log"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

type Key struct {
	Id  string `json:"-"`
	Key string `json:"key"`
}

func (k *Key) String() string {
	return k.Key
}

//add public key to local keyring
//call Get() before calling Store()
func (k *Key) Store() string {
	err := k.Write("epub.key")
	log.Check(log.WarnLevel, "Adding Public Key(PK) to local keyring", err)

	status := gpg.ImportPk("epub.key")
	os.Remove("epub.key")
	k.Id = ExtractKeyId(status)
	log.Debug("Found KeyID: " + k.Id)
	config.Management.GpgUser = k.Id
	return status
}

func (k *Key) ExtractKeyEmail() string {
	command := exec.Command("gpg")
	stdin, err := command.StdinPipe()
	stdin.Write([]byte(k.String()))
	stdin.Close()

	out, err := command.Output()
	log.Check(log.WarnLevel, "Extracting Email from Key", err)

	scanner := bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "uid") {
			line := strings.Fields(scanner.Text())
			if len(line) > 1 {
				email := line[len(line)-1]
				return email
			}
		}
	}
	return ""
}

//write pubkey to temp file
func (k *Key) Write(file string) error {
	err := ioutil.WriteFile(file, []byte(k.String()), 0644)
	if log.Check(log.WarnLevel, "Writing Pubkey to temp file", err) {
		return err
	}
	return nil
}

func (k *Key) Remove(file string) error {
	err := os.Remove(file)
	if log.Check(log.WarnLevel, "Removing file"+file, err) {
		return err
	}
	return nil
}

func ExtractKeyId(s string) string {
	arr := strings.Split(s, " ")
	for i, word := range arr {
		if word == "key" {
			return arr[i+1][0 : len(arr[i+1])-1]
		}
	}
	return ""
}
<|endoftext|>


<commit_before>package numbers

import (
	"errors"
	"reflect"
	"testing"
)

// TestFactorsList unit test FactorsList function.
func TestFactorsList(t *testing.T) {
	testCases := []struct {
		input    int
		expected []int
		err      error
	}{
		{2, []int{2}, nil},
		{15, []int{3, 5}, nil},
		{26, []int{2, 13}, nil},
		{37, []int{37}, nil},
		{42, []int{2, 3, 7}, nil},
		{-1, []int(nil), errors.New("no prime factors for numbers below two. received '-1'")},
	}
	for _, test := range testCases {
		observed, err := FactorsList(test.input)
		if err != nil {
			if test.err.Error() != err.Error() {
				t.Error(err)
			}
		}
		if !reflect.DeepEqual(observed, test.expected) {
			t.Errorf("for input '%d', expected '%v', got '%v'",
				test.input, test.expected, observed)
		}
	}
}

// BenchmarkFactorList benchmark FactorsList function.
func BenchmarkFactorList(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_, err := FactorsList(1000)
		if err != nil {
			b.Error(err)
		}
	}
}

// TestIsPrime unit test IsPrime function.
func TestIsPrime(t *testing.T) {
	testCases := []struct {
		input    int
		expected bool
	}{
		{-5, false},
		{0, false},
		{2, true},
		{6, false},
		{13, true},
	}
	for _, test := range testCases {
		observed := IsPrime(test.input)
		if observed != test.expected {
			t.Errorf("for input '%d', expected '%t', got '%t'",
				test.input, test.expected, observed)
		}
	}
}

// BenchmarkIsPrime benchmark IsPrime function.
func BenchmarkIsPrime(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_ = IsPrime(1728459)
	}
}

// TestFibonacciGen unit test FibonacciGen function.
func TestFibonacciGen(t *testing.T) {
	testCases := []struct {
		input, expected int
	}{
		{1, 0},
		{2, 1},
		{3, 1},
		{4, 2},
		{5, 3},
		{8, 13},
		{10, 34},
		{12, 89},
	}
	for _, test := range testCases {
		var observed int
		c := FibonacciGen()
		for n := 0; n < test.input; n++ {
			observed = <-c
		}
		if observed != test.expected {
			t.Errorf("for input '%d', expected '%d', got '%d'", test.input, test.expected, observed)
		}
	}
}

// BenchmarkFibonacciGen benchmark FibonacciGen function.
func BenchmarkFibonacciGen(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		c := FibonacciGen()
		for n := 0; n < 10000; n++ {
			_ = <-c
		}
	}
}

// TestNthFibonacci unit test NthFibonacci function.
func TestNthFibonacci(t *testing.T) {
	var testCases = []struct {
		input, expected int
	}{
		{1, 1},
		{5, 5},
		{7, 13},
		{10, 55},
		{19, 4181},
	}
	for _, test := range testCases {
		observed := NthFibonacci(test.input)
		if observed != test.expected {
			t.Errorf("for input '%d', expected '%d', got '%d'",
				test.input, test.expected, observed)
		}
	}
}

// BenchmarkNthFibonacci benchmark NthFibonacci function.
func BenchmarkNthFibonacci(b *testing.B) {
	for i := 0; i < b.N; i++ {
		_ = NthFibonacci(10000)
	}
}

// TestEvenNumbers unit test EvenNumbers function.
func TestEvenNumbers(t *testing.T) {
	testCases := []struct {
		inputA, inputB int
		expected       []int
		err            error
	}{
		{1, 1, []int(nil), nil},
		{1, 3, []int{2}, nil},
		{4, 8, []int{4, 6, 8}, nil},
		{-4, 8, []int{-4, -2, 0, 2, 4, 6, 8}, nil},
		{-1, -2, []int(nil), errors.New("a must be greater than b. received a '-1' and b '-2'")},
	}
	for _, test := range testCases {
		observed, err := EvenNumbers(test.inputA, test.inputB)
		if err != nil {
			if err.Error() != test.err.Error() {
				t.Error(err)
			}
		}
		if !reflect.DeepEqual(observed, test.expected) {
			t.Errorf("for a '%d' and b '%d', expected '%v', got '%v'",
				test.inputA, test.inputB, test.expected, observed)
		}
	}
}

// BenchmarkEvenNumbers benchmark EvenNumbers function.
func BenchmarkEvenNumbers(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_, err := EvenNumbers(1, 100000)
		if err != nil {
			b.Error(err)
		}
	}
}

// TestOddNumbers unit test OddNumbers function.
func TestOddNumbers(t *testing.T) {
	testCases := []struct {
		inputA, inputB int
		expected       []int
		err            error
	}{
		{1, 1, []int{1}, nil},
		{1, 3, []int{1, 3}, nil},
		{4, 8, []int{5, 7}, nil},
		{-4, 8, []int{-3, -1, 1, 3, 5, 7}, nil},
		{-1, -2, []int(nil), errors.New("a must be greater than b. received a '-1' and b '-2'")},
	}
	for _, test := range testCases {
		observed, err := OddNumbers(test.inputA, test.inputB)
		if err != nil {
			if err.Error() != test.err.Error() {
				t.Error(err)
			}
		}
		if !reflect.DeepEqual(observed, test.expected) {
			t.Errorf("for a '%d' and b '%d', expected '%v', got '%v'",
				test.inputA, test.inputB, test.expected, observed)
		}
	}
}

// BenchmarkOddNumbers benchmark OddNumbers function.
func BenchmarkOddNumbers(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_, err := OddNumbers(1, 100000)
		if err != nil {
			b.Error(err)
		}
	}
}

// TestNthEven unit test NthEven function.
func TestNthEven(t *testing.T) {
	testCases := []struct {
		input, expected int
		err             error
	}{
		{-1, 0, errors.New("input must be a positive number. received '-1'")},
		{-2, 0, errors.New("input must be a positive number. received '-2'")},
		{1, 0, nil},
		{2, 2, nil},
		{10, 18, nil},
	}
	for _, test := range testCases {
		observed, err := NthEven(test.input)
		if err != nil {
			if err.Error() != test.err.Error() {
				t.Errorf("for input '%d', expected '%s', got '%s'",
					test.input, test.err.Error(), err.Error())
			}
		}
		if !reflect.DeepEqual(observed, test.expected) {
			t.Errorf("for input '%d', expected '%v', got '%v'",
				test.input, test.expected, observed)
		}
	}
}

// BenchmarkNthEven benchmark NthEven function.
func BenchmarkNthEven(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_, err := NthEven(100000)
		if err != nil {
			b.Error(err)
		}
	}
}

// TestNthOdd unit test NthOdd function.
func TestNthOdd(t *testing.T) {
	testCases := []struct {
		input, expected int
		err             error
	}{
		{-1, 0, errors.New("input must be a positive number. received '-1'")},
		{-2, 0, errors.New("input must be a positive number. received '-2'")},
		{1, 1, nil},
		{2, 3, nil},
		{10, 19, nil},
	}
	for _, test := range testCases {
		observed, err := NthOdd(test.input)
		if err != nil {
			if err.Error() != test.err.Error() {
				t.Errorf("for input '%d', expected '%s', got '%s'",
					test.input, test.err.Error(), err.Error())
			}
		}
		if !reflect.DeepEqual(observed, test.expected) {
			t.Errorf("for input '%d', expected '%v', got '%v'",
				test.input, test.expected, observed)
		}
	}
}

// BenchmarkNthOdd benchmark NthOdd function.
func BenchmarkNthOdd(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_, err := NthOdd(100000)
		if err != nil {
			b.Error(err)
		}
	}
}
<commit_msg>Add unit test for SliceByParity function<commit_after>package numbers

import (
	"errors"
	"reflect"
	"testing"
)

// TestFactorsList unit test FactorsList function.
func TestFactorsList(t *testing.T) {
	testCases := []struct {
		input    int
		expected []int
		err      error
	}{
		{2, []int{2}, nil},
		{15, []int{3, 5}, nil},
		{26, []int{2, 13}, nil},
		{37, []int{37}, nil},
		{42, []int{2, 3, 7}, nil},
		{-1, []int(nil), errors.New("no prime factors for numbers below two. received '-1'")},
	}
	for _, test := range testCases {
		observed, err := FactorsList(test.input)
		if err != nil {
			if test.err.Error() != err.Error() {
				t.Error(err)
			}
		}
		if !reflect.DeepEqual(observed, test.expected) {
			t.Errorf("for input '%d', expected '%v', got '%v'",
				test.input, test.expected, observed)
		}
	}
}

// BenchmarkFactorList benchmark FactorsList function.
func BenchmarkFactorList(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_, err := FactorsList(1000)
		if err != nil {
			b.Error(err)
		}
	}
}

// TestIsPrime unit test IsPrime function.
func TestIsPrime(t *testing.T) {
	testCases := []struct {
		input    int
		expected bool
	}{
		{-5, false},
		{0, false},
		{2, true},
		{6, false},
		{13, true},
	}
	for _, test := range testCases {
		observed := IsPrime(test.input)
		if observed != test.expected {
			t.Errorf("for input '%d', expected '%t', got '%t'",
				test.input, test.expected, observed)
		}
	}
}

// BenchmarkIsPrime benchmark IsPrime function.
func BenchmarkIsPrime(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_ = IsPrime(1728459)
	}
}

// TestFibonacciGen unit test FibonacciGen function.
func TestFibonacciGen(t *testing.T) {
	testCases := []struct {
		input, expected int
	}{
		{1, 0},
		{2, 1},
		{3, 1},
		{4, 2},
		{5, 3},
		{8, 13},
		{10, 34},
		{12, 89},
	}
	for _, test := range testCases {
		var observed int
		c := FibonacciGen()
		for n := 0; n < test.input; n++ {
			observed = <-c
		}
		if observed != test.expected {
			t.Errorf("for input '%d', expected '%d', got '%d'", test.input, test.expected, observed)
		}
	}
}

// BenchmarkFibonacciGen benchmark FibonacciGen function.
func BenchmarkFibonacciGen(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		c := FibonacciGen()
		for n := 0; n < 10000; n++ {
			_ = <-c
		}
	}
}

// TestNthFibonacci unit test NthFibonacci function.
func TestNthFibonacci(t *testing.T) {
	var testCases = []struct {
		input, expected int
	}{
		{1, 1},
		{5, 5},
		{7, 13},
		{10, 55},
		{19, 4181},
	}
	for _, test := range testCases {
		observed := NthFibonacci(test.input)
		if observed != test.expected {
			t.Errorf("for input '%d', expected '%d', got '%d'",
				test.input, test.expected, observed)
		}
	}
}

// BenchmarkNthFibonacci benchmark NthFibonacci function.
func BenchmarkNthFibonacci(b *testing.B) {
	for i := 0; i < b.N; i++ {
		_ = NthFibonacci(10000)
	}
}

// TestEvenNumbers unit test EvenNumbers function.
func TestEvenNumbers(t *testing.T) {
	testCases := []struct {
		inputA, inputB int
		expected       []int
		err            error
	}{
		{1, 1, []int(nil), nil},
		{1, 3, []int{2}, nil},
		{4, 8, []int{4, 6, 8}, nil},
		{-4, 8, []int{-4, -2, 0, 2, 4, 6, 8}, nil},
		{-1, -2, []int(nil), errors.New("a must be greater than b. received a '-1' and b '-2'")},
	}
	for _, test := range testCases {
		observed, err := EvenNumbers(test.inputA, test.inputB)
		if err != nil {
			if err.Error() != test.err.Error() {
				t.Error(err)
			}
		}
		if !reflect.DeepEqual(observed, test.expected) {
			t.Errorf("for a '%d' and b '%d', expected '%v', got '%v'",
				test.inputA, test.inputB, test.expected, observed)
		}
	}
}

// BenchmarkEvenNumbers benchmark EvenNumbers function.
func BenchmarkEvenNumbers(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_, err := EvenNumbers(1, 100000)
		if err != nil {
			b.Error(err)
		}
	}
}

// TestOddNumbers unit test OddNumbers function.
func TestOddNumbers(t *testing.T) {
	testCases := []struct {
		inputA, inputB int
		expected       []int
		err            error
	}{
		{1, 1, []int{1}, nil},
		{1, 3, []int{1, 3}, nil},
		{4, 8, []int{5, 7}, nil},
		{-4, 8, []int{-3, -1, 1, 3, 5, 7}, nil},
		{-1, -2, []int(nil), errors.New("a must be greater than b. received a '-1' and b '-2'")},
	}
	for _, test := range testCases {
		observed, err := OddNumbers(test.inputA, test.inputB)
		if err != nil {
			if err.Error() != test.err.Error() {
				t.Error(err)
			}
		}
		if !reflect.DeepEqual(observed, test.expected) {
			t.Errorf("for a '%d' and b '%d', expected '%v', got '%v'",
				test.inputA, test.inputB, test.expected, observed)
		}
	}
}

// BenchmarkOddNumbers benchmark OddNumbers function.
func BenchmarkOddNumbers(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_, err := OddNumbers(1, 100000)
		if err != nil {
			b.Error(err)
		}
	}
}

// TestNthEven unit test NthEven function.
func TestNthEven(t *testing.T) {
	testCases := []struct {
		input, expected int
		err             error
	}{
		{-1, 0, errors.New("input must be a positive number. received '-1'")},
		{-2, 0, errors.New("input must be a positive number. received '-2'")},
		{1, 0, nil},
		{2, 2, nil},
		{10, 18, nil},
	}
	for _, test := range testCases {
		observed, err := NthEven(test.input)
		if err != nil {
			if err.Error() != test.err.Error() {
				t.Errorf("for input '%d', expected '%s', got '%s'",
					test.input, test.err.Error(), err.Error())
			}
		}
		if !reflect.DeepEqual(observed, test.expected) {
			t.Errorf("for input '%d', expected '%v', got '%v'",
				test.input, test.expected, observed)
		}
	}
}

// BenchmarkNthEven benchmark NthEven function.
func BenchmarkNthEven(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_, err := NthEven(100000)
		if err != nil {
			b.Error(err)
		}
	}
}

// TestNthOdd unit test NthOdd function.
func TestNthOdd(t *testing.T) {
	testCases := []struct {
		input, expected int
		err             error
	}{
		{-1, 0, errors.New("input must be a positive number. received '-1'")},
		{-2, 0, errors.New("input must be a positive number. received '-2'")},
		{1, 1, nil},
		{2, 3, nil},
		{10, 19, nil},
	}
	for _, test := range testCases {
		observed, err := NthOdd(test.input)
		if err != nil {
			if err.Error() != test.err.Error() {
				t.Errorf("for input '%d', expected '%s', got '%s'",
					test.input, test.err.Error(), err.Error())
			}
		}
		if !reflect.DeepEqual(observed, test.expected) {
			t.Errorf("for input '%d', expected '%v', got '%v'",
				test.input, test.expected, observed)
		}
	}
}

// BenchmarkNthOdd benchmark NthOdd function.
func BenchmarkNthOdd(b *testing.B) {
	for i := 0; i <= b.N; i++ {
		_, err := NthOdd(100000)
		if err != nil {
			b.Error(err)
		}
	}
}

// TestSliceByParity unit test SliceByParity  function.
func TestSliceByParity(t *testing.T) {
	testCases := []struct {
		inputA, inputB            int
		expectedEven, expectedOdd []int
	}{
		{1, 1, []int(nil), []int{1}},
		{1, 3, []int{2}, []int{1, 3}},
		{4, 8, []int{4, 6, 8}, []int{5, 7}},
		{-4, 8, []int{-4, -2, 0, 2, 4, 6, 8}, []int{-3, -1, 1, 3, 5, 7}},
	}
	for _, test := range testCases {
		observedEven, observedOdd, err := SliceByParity(test.inputA, test.inputB)
		if err != nil {
			t.Error(err)
		}
		if !reflect.DeepEqual(observedEven, test.expectedEven) {
			t.Errorf("for a '%d' and b '%d', expected even '%v', got '%v'",
				test.inputA, test.inputB, test.expectedEven, observedEven)
		}
		if !reflect.DeepEqual(observedOdd, test.expectedOdd) {
			t.Errorf("for a '%d' and b '%d', expected odd '%v', got '%v'",
				test.inputA, test.inputB, test.expectedOdd, observedOdd)
		}
	}
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type ServerOptions struct {
	cpuprofile *string
	v          VolumeServerOptions
}

var (
	serverOptions    ServerOptions
	masterOptions    MasterOptions
	filerOptions     FilerOptions
	s3Options        S3Options
	msgBrokerOptions MessageBrokerOptions
)

func init() {
	cmdServer.Run = runServer // break init cycle
}

var cmdServer = &Command{
	UsageLine: "server -dir=/tmp -volume.max=5 -ip=server_name",
	Short:     "start a master server, a volume server, and optionally a filer and a S3 gateway",
	Long: `start both a volume server to provide storage spaces
  and a master server to provide volume=>location mapping service and sequence number of file ids

  This is provided as a convenient way to start both volume server and master server.
  The servers acts exactly the same as starting them separately.
  So other volume servers can connect to this master server also.

  Optionally, a filer server can be started.
  Also optionally, a S3 gateway can be started.

  `,
}

var (
	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
	serverBindIp              = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
	serverRack                = cmdServer.Flag.String("rack", "", "current volume server's rack name")
	serverWhiteListOption     = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	serverDisableHttp         = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
	volumeDataFolders         = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")

	// pulseSeconds              = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	isStartingFiler     = cmdServer.Flag.Bool("filer", false, "whether to start filer")
	isStartingS3        = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
	isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker")

	serverWhiteList []string

	False = false
)

func init() {
	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")

	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "000", "Default replication type if not specified.")
	masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")

	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
	filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")

	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 1024, "limit file size to avoid out of memory")
	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
	serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
	serverOptions.v.pprof = &False

	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")

	msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port")

}

func runServer(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)
	util.LoadConfiguration("master", false)

	if *serverOptions.cpuprofile != "" {
		f, err := os.Create(*serverOptions.cpuprofile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *isStartingS3 {
		*isStartingFiler = true
	}
	if *isStartingMsgBroker {
		*isStartingFiler = true
	}

	_, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers)
	peers := strings.Join(peerList, ",")
	masterOptions.peers = &peers

	masterOptions.ip = serverIp
	masterOptions.ipBind = serverBindIp
	filerOptions.masters = &peers
	filerOptions.ip = serverIp
	filerOptions.bindIp = serverBindIp
	serverOptions.v.ip = serverIp
	serverOptions.v.bindIp = serverBindIp
	serverOptions.v.masters = &peers
	serverOptions.v.idleConnectionTimeout = serverTimeout
	serverOptions.v.dataCenter = serverDataCenter
	serverOptions.v.rack = serverRack
	msgBrokerOptions.ip = serverIp

	// serverOptions.v.pulseSeconds = pulseSeconds
	// masterOptions.pulseSeconds = pulseSeconds

	masterOptions.whiteList = serverWhiteListOption

	filerOptions.dataCenter = serverDataCenter
	filerOptions.disableHttp = serverDisableHttp
	masterOptions.disableHttp = serverDisableHttp

	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
	s3Options.filer = &filerAddress
	msgBrokerOptions.filer = &filerAddress

	if *filerOptions.defaultReplicaPlacement == "" {
		*filerOptions.defaultReplicaPlacement = *masterOptions.defaultReplication
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	folders := strings.Split(*volumeDataFolders, ",")

	if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
	}

	if *masterOptions.metaFolder == "" {
		*masterOptions.metaFolder = folders[0]
	}
	if err := util.TestFolderWritable(util.ResolvePath(*masterOptions.metaFolder)); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
	}
	filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder

	if *serverWhiteListOption != "" {
		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
	}

	if *isStartingFiler {
		go func() {
			time.Sleep(1 * time.Second)

			filerOptions.startFiler()

		}()
	}

	if *isStartingS3 {
		go func() {
			time.Sleep(2 * time.Second)

			s3Options.startS3Server()

		}()
	}

	if *isStartingMsgBroker {
		go func() {
			time.Sleep(2 * time.Second)
			msgBrokerOptions.startQueueServer()
		}()
	}

	// start volume server
	{
		go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent)

	}

	startMaster(masterOptions, serverWhiteList)

	return true
}
<commit_msg>Allow option to enable volume pprof on server<commit_after>package command

import (
	"fmt"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type ServerOptions struct {
	cpuprofile *string
	v          VolumeServerOptions
}

var (
	serverOptions    ServerOptions
	masterOptions    MasterOptions
	filerOptions     FilerOptions
	s3Options        S3Options
	msgBrokerOptions MessageBrokerOptions
)

func init() {
	cmdServer.Run = runServer // break init cycle
}

var cmdServer = &Command{
	UsageLine: "server -dir=/tmp -volume.max=5 -ip=server_name",
	Short:     "start a master server, a volume server, and optionally a filer and a S3 gateway",
	Long: `start both a volume server to provide storage spaces
  and a master server to provide volume=>location mapping service and sequence number of file ids

  This is provided as a convenient way to start both volume server and master server.
  The servers acts exactly the same as starting them separately.
  So other volume servers can connect to this master server also.

  Optionally, a filer server can be started.
  Also optionally, a S3 gateway can be started.

  `,
}

var (
	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
	serverBindIp              = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
	serverRack                = cmdServer.Flag.String("rack", "", "current volume server's rack name")
	serverWhiteListOption     = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	serverDisableHttp         = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
	volumeDataFolders         = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")

	// pulseSeconds              = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	isStartingFiler     = cmdServer.Flag.Bool("filer", false, "whether to start filer")
	isStartingS3        = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
	isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker")

	serverWhiteList []string

	False = false
)

func init() {
	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")

	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "000", "Default replication type if not specified.")
	masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")

	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
	filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")

	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 1024, "limit file size to avoid out of memory")
	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
	serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
	serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")

	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")

	msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port")

}

func runServer(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)
	util.LoadConfiguration("master", false)

	if *serverOptions.cpuprofile != "" {
		f, err := os.Create(*serverOptions.cpuprofile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *isStartingS3 {
		*isStartingFiler = true
	}
	if *isStartingMsgBroker {
		*isStartingFiler = true
	}

	_, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers)
	peers := strings.Join(peerList, ",")
	masterOptions.peers = &peers

	masterOptions.ip = serverIp
	masterOptions.ipBind = serverBindIp
	filerOptions.masters = &peers
	filerOptions.ip = serverIp
	filerOptions.bindIp = serverBindIp
	serverOptions.v.ip = serverIp
	serverOptions.v.bindIp = serverBindIp
	serverOptions.v.masters = &peers
	serverOptions.v.idleConnectionTimeout = serverTimeout
	serverOptions.v.dataCenter = serverDataCenter
	serverOptions.v.rack = serverRack
	msgBrokerOptions.ip = serverIp

	// serverOptions.v.pulseSeconds = pulseSeconds
	// masterOptions.pulseSeconds = pulseSeconds

	masterOptions.whiteList = serverWhiteListOption

	filerOptions.dataCenter = serverDataCenter
	filerOptions.disableHttp = serverDisableHttp
	masterOptions.disableHttp = serverDisableHttp

	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
	s3Options.filer = &filerAddress
	msgBrokerOptions.filer = &filerAddress

	if *filerOptions.defaultReplicaPlacement == "" {
		*filerOptions.defaultReplicaPlacement = *masterOptions.defaultReplication
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	folders := strings.Split(*volumeDataFolders, ",")

	if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
	}

	if *masterOptions.metaFolder == "" {
		*masterOptions.metaFolder = folders[0]
	}
	if err := util.TestFolderWritable(util.ResolvePath(*masterOptions.metaFolder)); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
	}
	filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder

	if *serverWhiteListOption != "" {
		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
	}

	if *isStartingFiler {
		go func() {
			time.Sleep(1 * time.Second)

			filerOptions.startFiler()

		}()
	}

	if *isStartingS3 {
		go func() {
			time.Sleep(2 * time.Second)

			s3Options.startS3Server()

		}()
	}

	if *isStartingMsgBroker {
		go func() {
			time.Sleep(2 * time.Second)
			msgBrokerOptions.startQueueServer()
		}()
	}

	// start volume server
	{
		go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent)

	}

	startMaster(masterOptions, serverWhiteList)

	return true
}
<|endoftext|>


<commit_before>// TODO: Add documentation
package s3gof3r

import (
	"bytes"
	"fmt"
	"github.com/rlmcpherson/s3/s3util"
	"io"
	"log"
	"net/http"
	"net/url"
	//"os"
	"sync"
	"syscall"
	"time"
)

const (
	_        = iota
	KB int64 = 1 << (10 * iota)
	MB
	GB
	TB
	PB
	EB
)

const (
	buffer_size = 20 * MB
)

type getter struct {
	url    url.URL
	client *http.Client
	conf   *s3util.Config
	bufsz  int64
	err    error
	wg     sync.WaitGroup

	cur_chunk_id   int
	cur_chunk      *chunk
	content_length int64
	chunk_total    int
	get_ch         chan *chunk
	read_ch        chan *chunk

	bp bufferpool

	q_wait map[int]*chunk

	concurrency int64
	nTry        int
	closed      bool
}

type chunk struct {
	id     int
	header http.Header
	start  int64
	size   int64
	b      *bytes.Buffer
	len    int64
}

type bufferpool struct {
	get  chan *bytes.Buffer
	give chan *bytes.Buffer
}

func s3Get(raw_url string, c *s3util.Config) (io.ReadCloser, http.Header, error) {

	p_url, err := url.Parse(raw_url)
	if err != nil {
		return nil, nil, err
	}

	return newGetter(*p_url, c)
}

func newGetter(p_url url.URL, c *s3util.Config) (io.ReadCloser, http.Header, error) {

	// initialize getter
	g := new(getter)
	g.conf = c
	g.url = p_url
	g.bufsz = buffer_size
	g.bp.get, g.bp.give = makeRecycler()
	g.get_ch = make(chan *chunk)
	g.read_ch = make(chan *chunk)
	g.nTry = 5
	g.q_wait = make(map[int]*chunk)

	// get content length
	r, err := http.NewRequest("HEAD", p_url.String(), nil)
	if err != nil {
		return nil, nil, err
	}
	r.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
	r.Header.Set("User-Agent", "s3Gof3r")

	g.conf.Sign(r, *g.conf.Keys)
	g.client = g.conf.Client

	resp, err := g.client.Do(r)
	if err != nil {
		return nil, nil, err
	}
	if resp.Status != "200 OK" {
		return nil, nil, fmt.Errorf("Expected HTTP Status 200, received %q", resp.Status)
	}
	g.content_length = resp.ContentLength

	g.concurrency = min(int64(g.conf.Concurrency), (g.content_length/buffer_size)+1)
	log.Println("Concurrency:", g.concurrency)

	for i := int64(0); i < g.concurrency; i++ {
		go g.worker()
	}
	go g.init_chunks()

	log.Println("End of initialize")
	return g, resp.Header, nil
}

func (g *getter) init_chunks() {
	for i := int64(0); i < g.content_length; {
		size := min(g.bufsz, g.content_length-i)
		c := &chunk{
			id: g.chunk_total,
			header: http.Header{
				"Range": {fmt.Sprintf("bytes=%d-%d",
					i, i+size-1)},
				"User-Agent": {"S3Gof3r"},
				"Date":       {time.Now().UTC().Format(http.TimeFormat)}},

			start: i,
			size:  size,
			b:     nil,
			len:   0}

		i += size
		g.chunk_total++
		g.wg.Add(1)

		// put on get chan
		log.Println("Sending chunk ", c.id, " Offset: ", c.start, " Size:", c.size)
		g.get_ch <- c
	}

}

func (g *getter) worker() {
	for c := range g.get_ch {
		g.retryGetChunk(c)
	}

}

func (g *getter) retryGetChunk(c *chunk) {

	defer g.wg.Done()
	var err error
	// get buffer to write
	c.b = <-g.bp.get
	for i := 0; i < g.nTry; i++ {
		err = g.getChunk(c)
		if err == nil {
			return
		}
		log.Println(err)
	}
	g.err = err

}

func (g *getter) getChunk(c *chunk) error {
	// ensure buffer is empty
	c.b.Reset()

	r, err := http.NewRequest("GET", g.url.String(), nil)
	if err != nil {
		return err
	}
	r.Header = c.header

	g.conf.Sign(r, *g.conf.Keys)

	resp, err := g.client.Do(r)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.Status != "206 Partial Content" {
		//resp.Write(os.Stderr)
		return fmt.Errorf("Expected HTTP Status 206, received %q",
			resp.Status)
	}
	//resp.Header.Write(os.Stderr)
	log.Println("buffer size: ", c.b.Len())

	//n, err := io.Copy(c.b, resp.Body) //TODO: Md5 checking
	n, err := c.b.ReadFrom(resp.Body)
	//n, err := io.CopyN(c.b, resp.Body, c.size-1)
	log.Println("Body length:", c.b.Len())
	//c.b.Write(b)
	if err != nil {
		return err
	}
	if n != c.size {
		log.Println("Size mismatch")
		return fmt.Errorf("Chunk %d: Expected %d bytes, received %d",
			c.id, c.size, n)
	}

	g.read_ch <- c
	return nil
}

func (g *getter) Read(p []byte) (int, error) {
	var err error
	if g.closed {
		return 0, syscall.EINVAL
	}
	if g.err != nil {
		return 0, g.err
	}
	if g.cur_chunk == nil {
		g.cur_chunk, err = g.get_cur_chunk()
		if err != nil {
			return 0, err
		}
	}
	n, err := g.cur_chunk.b.Read(p)

	// Empty buffer, move on to next
	if err == io.EOF {
		log.Printf("Completed chunk %d of %d\n", g.cur_chunk.id, g.chunk_total-1)
		// Do not send EOF for each chunk.
		if g.cur_chunk.id == g.chunk_total-1 && g.cur_chunk.b.Len() == 0 {
			log.Println("Last Chunk:", g.cur_chunk)
			return 0, io.EOF
		}
		g.bp.give <- g.cur_chunk.b
		g.cur_chunk = nil
		g.cur_chunk_id++
		return n - 1, nil
	}

	return n, err
}

func (g *getter) get_cur_chunk() (*chunk, error) {
	var cur_chunk *chunk
	var err error
	for {
		// first check q_wait
		if cur_chunk, ok := g.q_wait[g.cur_chunk_id]; ok {
			delete(g.q_wait, g.cur_chunk_id)
			//log.Println("return cur_chunk:", cur_chunk)
			return cur_chunk, err
		}
		// if not present, read from channel
		cur_chunk = <-g.read_ch
		g.q_wait[cur_chunk.id] = cur_chunk

	}
	return cur_chunk, err
}

func (g *getter) Close() error {
	if g.closed {
		return syscall.EINVAL
	}
	if g.err != nil {
		return g.err
	}
	g.wg.Wait()
	//close(g.read_ch) //TODO: close these
	//close(g.get_ch)
	//close(g.bp.give)
	//close(g.bp.get)

	g.closed = true
	log.Println("makes:", Makes)
	log.Println("max q len:", Q_max)

	return nil
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Don't need to close channels.<commit_after>// TODO: Add documentation
package s3gof3r

import (
	"bytes"
	"fmt"
	"github.com/rlmcpherson/s3/s3util"
	"io"
	"log"
	"net/http"
	"net/url"
	//"os"
	"sync"
	"syscall"
	"time"
)

const (
	_        = iota
	KB int64 = 1 << (10 * iota)
	MB
	GB
	TB
	PB
	EB
)

const (
	buffer_size = 20 * MB
)

type getter struct {
	url    url.URL
	client *http.Client
	conf   *s3util.Config
	bufsz  int64
	err    error
	wg     sync.WaitGroup

	cur_chunk_id   int
	cur_chunk      *chunk
	content_length int64
	chunk_total    int
	get_ch         chan *chunk
	read_ch        chan *chunk

	bp bufferpool

	q_wait map[int]*chunk

	concurrency int64
	nTry        int
	closed      bool
}

type chunk struct {
	id     int
	header http.Header
	start  int64
	size   int64
	b      *bytes.Buffer
	len    int64
}

type bufferpool struct {
	get  chan *bytes.Buffer
	give chan *bytes.Buffer
}

func s3Get(raw_url string, c *s3util.Config) (io.ReadCloser, http.Header, error) {

	p_url, err := url.Parse(raw_url)
	if err != nil {
		return nil, nil, err
	}

	return newGetter(*p_url, c)
}

func newGetter(p_url url.URL, c *s3util.Config) (io.ReadCloser, http.Header, error) {

	// initialize getter
	g := new(getter)
	g.conf = c
	g.url = p_url
	g.bufsz = buffer_size
	g.bp.get, g.bp.give = makeRecycler()
	g.get_ch = make(chan *chunk)
	g.read_ch = make(chan *chunk)
	g.nTry = 5
	g.q_wait = make(map[int]*chunk)

	// get content length
	r, err := http.NewRequest("HEAD", p_url.String(), nil)
	if err != nil {
		return nil, nil, err
	}
	r.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
	r.Header.Set("User-Agent", "s3Gof3r")

	g.conf.Sign(r, *g.conf.Keys)
	g.client = g.conf.Client

	resp, err := g.client.Do(r)
	if err != nil {
		return nil, nil, err
	}
	if resp.Status != "200 OK" {
		return nil, nil, fmt.Errorf("Expected HTTP Status 200, received %q", resp.Status)
	}
	g.content_length = resp.ContentLength

	g.concurrency = min(int64(g.conf.Concurrency), (g.content_length/buffer_size)+1)
	log.Println("Concurrency:", g.concurrency)

	for i := int64(0); i < g.concurrency; i++ {
		go g.worker()
	}
	go g.init_chunks()

	log.Println("End of initialize")
	return g, resp.Header, nil
}

func (g *getter) init_chunks() {
	for i := int64(0); i < g.content_length; {
		size := min(g.bufsz, g.content_length-i)
		c := &chunk{
			id: g.chunk_total,
			header: http.Header{
				"Range": {fmt.Sprintf("bytes=%d-%d",
					i, i+size-1)},
				"User-Agent": {"S3Gof3r"},
				"Date":       {time.Now().UTC().Format(http.TimeFormat)}},

			start: i,
			size:  size,
			b:     nil,
			len:   0}

		i += size
		g.chunk_total++
		g.wg.Add(1)

		// put on get chan
		log.Println("Sending chunk ", c.id, " Offset: ", c.start, " Size:", c.size)
		g.get_ch <- c
	}

}

func (g *getter) worker() {
	for c := range g.get_ch {
		g.retryGetChunk(c)
	}

}

func (g *getter) retryGetChunk(c *chunk) {

	defer g.wg.Done()
	var err error
	// get buffer to write
	c.b = <-g.bp.get
	for i := 0; i < g.nTry; i++ {
		err = g.getChunk(c)
		if err == nil {
			return
		}
		log.Println(err)
	}
	g.err = err

}

func (g *getter) getChunk(c *chunk) error {
	// ensure buffer is empty
	c.b.Reset()

	r, err := http.NewRequest("GET", g.url.String(), nil)
	if err != nil {
		return err
	}
	r.Header = c.header

	g.conf.Sign(r, *g.conf.Keys)

	resp, err := g.client.Do(r)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.Status != "206 Partial Content" {
		//resp.Write(os.Stderr)
		return fmt.Errorf("Expected HTTP Status 206, received %q",
			resp.Status)
	}
	//resp.Header.Write(os.Stderr)
	log.Println("buffer size: ", c.b.Len())

	//n, err := io.Copy(c.b, resp.Body) //TODO: Md5 checking
	n, err := c.b.ReadFrom(resp.Body)
	//n, err := io.CopyN(c.b, resp.Body, c.size-1)
	log.Println("Body length:", c.b.Len())
	//c.b.Write(b)
	if err != nil {
		return err
	}
	if n != c.size {
		log.Println("Size mismatch")
		return fmt.Errorf("Chunk %d: Expected %d bytes, received %d",
			c.id, c.size, n)
	}

	g.read_ch <- c
	return nil
}

func (g *getter) Read(p []byte) (int, error) {
	var err error
	if g.closed {
		return 0, syscall.EINVAL
	}
	if g.err != nil {
		return 0, g.err
	}
	if g.cur_chunk == nil {
		g.cur_chunk, err = g.get_cur_chunk()
		if err != nil {
			return 0, err
		}
	}
	n, err := g.cur_chunk.b.Read(p)

	// Empty buffer, move on to next
	if err == io.EOF {
		log.Printf("Completed chunk %d of %d\n", g.cur_chunk.id, g.chunk_total-1)
		// Do not send EOF for each chunk.
		if g.cur_chunk.id == g.chunk_total-1 && g.cur_chunk.b.Len() == 0 {
			log.Println("Last Chunk:", g.cur_chunk)
			return 0, io.EOF
		}
		g.bp.give <- g.cur_chunk.b
		g.cur_chunk = nil
		g.cur_chunk_id++
		return n - 1, nil
	}

	return n, err
}

func (g *getter) get_cur_chunk() (*chunk, error) {
	var cur_chunk *chunk
	var err error
	for {
		// first check q_wait
		if cur_chunk, ok := g.q_wait[g.cur_chunk_id]; ok {
			delete(g.q_wait, g.cur_chunk_id)
			//log.Println("return cur_chunk:", cur_chunk)
			return cur_chunk, err
		}
		// if not present, read from channel
		cur_chunk = <-g.read_ch
		g.q_wait[cur_chunk.id] = cur_chunk

	}
	return cur_chunk, err
}

func (g *getter) Close() error {
	if g.closed {
		return syscall.EINVAL
	}
	if g.err != nil {
		return g.err
	}
	g.wg.Wait()
	g.closed = true
	log.Println("makes:", Makes)
	log.Println("max q len:", Q_max)

	return nil
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
)

type DataStore struct {
	dir string
}

func NewDataStore(dir string) *DataStore {
	return &DataStore{dir: dir}
}

func (s *DataStore) CreateFile(id string, size int64, contentType string) error {
	file, err := os.OpenFile(s.filePath(id), os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()

	entry := logEntry{Meta: &metaEntry{Size: size, ContentType: contentType}}
	return s.appendFileLog(id, entry)
}

func (s *DataStore) WriteFileChunk(id string, start int64, end int64, src io.Reader) error {
	file, err := os.OpenFile(s.filePath(id), os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()

	if n, err := file.Seek(start, os.SEEK_SET); err != nil {
		return err
	} else if n != start {
		return errors.New("WriteFileChunk: seek failure")
	}

	size := end - start + 1
	n, err := io.CopyN(file, src, size)
	if n > 0 {
		entry := logEntry{Chunk: &chunkEntry{Start: start, End: n - 1}}
		if err := s.appendFileLog(id, entry); err != nil {
			return err
		}
	}

	if err != nil {
		return err
	} else if n != size {
		return errors.New("WriteFileChunk: partial copy")
	}
	return nil
}

func (s *DataStore) GetFileMeta(id string) (*fileMeta, error) {
	// @TODO stream the file / limit log file size?
	data, err := ioutil.ReadFile(s.logPath(id))
	if err != nil {
		return nil, err
	}
	lines := strings.Split(string(data), "\n")
	// last line is always empty, lets skip it
	lines = lines[:len(lines)-1]

	meta := &fileMeta{
		Chunks: make(chunkSet, 0, len(lines)),
	}

	for _, line := range lines {
		entry := logEntry{}
		if err := json.Unmarshal([]byte(line), &entry); err != nil {
			return nil, err
		}

		if entry.Chunk != nil {
			meta.Chunks.Add(chunk{Start: entry.Chunk.Start, End: entry.Chunk.End})
		}

		if entry.Meta != nil {
			meta.ContentType = entry.Meta.ContentType
			meta.Size = entry.Meta.Size
		}
	}

	return meta, nil
}

func (s *DataStore) ReadFile(id string) (io.ReadCloser, error) {
	file, err := os.Open(s.filePath(id))
	if err != nil {
		return nil, err
	}

	return file, nil
}

func (s *DataStore) appendFileLog(id string, entry interface{}) error {
	logPath := s.logPath(id)
	logFile, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return err
	}
	defer logFile.Close()

	data, err := json.Marshal(entry)
	if err != nil {
		return err
	}

	if _, err := logFile.WriteString(string(data) + "\n"); err != nil {
		return err
	}
	return nil
}

func (s *DataStore) filePath(id string) string {
	return path.Join(s.dir, id)
}

func (s *DataStore) logPath(id string) string {
	return s.filePath(id) + ".log"
}

type fileMeta struct {
	ContentType string
	Size        int64
	Chunks      chunkSet
}

type logEntry struct {
	Chunk *chunkEntry `json:",omitempty"`
	Meta  *metaEntry  `json:",omitempty"`
}

type chunkEntry struct {
	Start, End int64
}
type metaEntry struct {
	Size        int64
	ContentType string
}
<commit_msg>Fix: Upload broken<commit_after>package main

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
)

type DataStore struct {
	dir string
}

func NewDataStore(dir string) *DataStore {
	return &DataStore{dir: dir}
}

func (s *DataStore) CreateFile(id string, size int64, contentType string) error {
	file, err := os.OpenFile(s.filePath(id), os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()

	entry := logEntry{Meta: &metaEntry{Size: size, ContentType: contentType}}
	return s.appendFileLog(id, entry)
}

func (s *DataStore) WriteFileChunk(id string, start int64, end int64, src io.Reader) error {
	file, err := os.OpenFile(s.filePath(id), os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()

	if n, err := file.Seek(start, os.SEEK_SET); err != nil {
		return err
	} else if n != start {
		return errors.New("WriteFileChunk: seek failure")
	}

	size := end - start + 1
	n, err := io.CopyN(file, src, size)
	if n > 0 {
		entry := logEntry{Chunk: &chunkEntry{Start: start, End: start+n-1}}
		if err := s.appendFileLog(id, entry); err != nil {
			return err
		}
	}

	if err != nil {
		return err
	} else if n != size {
		return errors.New("WriteFileChunk: partial copy")
	}
	return nil
}

func (s *DataStore) GetFileMeta(id string) (*fileMeta, error) {
	// @TODO stream the file / limit log file size?
	data, err := ioutil.ReadFile(s.logPath(id))
	if err != nil {
		return nil, err
	}
	lines := strings.Split(string(data), "\n")
	// last line is always empty, lets skip it
	lines = lines[:len(lines)-1]

	meta := &fileMeta{
		Chunks: make(chunkSet, 0, len(lines)),
	}

	for _, line := range lines {
		entry := logEntry{}
		if err := json.Unmarshal([]byte(line), &entry); err != nil {
			return nil, err
		}

		if entry.Chunk != nil {
			meta.Chunks.Add(chunk{Start: entry.Chunk.Start, End: entry.Chunk.End})
		}

		if entry.Meta != nil {
			meta.ContentType = entry.Meta.ContentType
			meta.Size = entry.Meta.Size
		}
	}

	return meta, nil
}

func (s *DataStore) ReadFile(id string) (io.ReadCloser, error) {
	file, err := os.Open(s.filePath(id))
	if err != nil {
		return nil, err
	}

	return file, nil
}

func (s *DataStore) appendFileLog(id string, entry interface{}) error {
	logPath := s.logPath(id)
	logFile, err := os.OpenFile(logPath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return err
	}
	defer logFile.Close()

	data, err := json.Marshal(entry)
	if err != nil {
		return err
	}

	if _, err := logFile.WriteString(string(data) + "\n"); err != nil {
		return err
	}
	return nil
}

func (s *DataStore) filePath(id string) string {
	return path.Join(s.dir, id)
}

func (s *DataStore) logPath(id string) string {
	return s.filePath(id) + ".log"
}

type fileMeta struct {
	ContentType string
	Size        int64
	Chunks      chunkSet
}

type logEntry struct {
	Chunk *chunkEntry `json:",omitempty"`
	Meta  *metaEntry  `json:",omitempty"`
}

type chunkEntry struct {
	Start, End int64
}
type metaEntry struct {
	Size        int64
	ContentType string
}
<|endoftext|>


<commit_before>package version

import (
	"fmt"

	protoversion "go.pedge.io/proto/version"
)

const (
	// MajorVersion is the current major version for pachyderm.
	MajorVersion = 1
	// MinorVersion is the current minor version for pachyderm.
	MinorVersion = 2
	// MicroVersion is the patch number for pachyderm.
	MicroVersion = 0
)

var (
	// AdditionalVersion is the string provided at release time
	// The value is passed to the linker at build time
	// DO NOT set the value of this variable here
	AdditionalVersion string
	// Version is the current version for pachyderm.
	Version = &protoversion.Version{
		Major:      MajorVersion,
		Minor:      MinorVersion,
		Micro:      MicroVersion,
		Additional: AdditionalVersion,
	}
)

// PrettyPrintVersion returns a version string optionally tagged with metadata.
// For example: "1.2.3", or "1.2.3-rc1" if version.Additional is "rc1".
func PrettyPrintVersion(version *protoversion.Version) string {
	result := fmt.Sprintf("%d.%d.%d", version.Major, version.Minor, version.Micro)
	if version.Additional != "" {
		result += fmt.Sprintf("-%s", version.Additional)
	}
	return result
}
<commit_msg>Bump to 1.2.1<commit_after>package version

import (
	"fmt"

	protoversion "go.pedge.io/proto/version"
)

const (
	// MajorVersion is the current major version for pachyderm.
	MajorVersion = 1
	// MinorVersion is the current minor version for pachyderm.
	MinorVersion = 2
	// MicroVersion is the patch number for pachyderm.
	MicroVersion = 1
)

var (
	// AdditionalVersion is the string provided at release time
	// The value is passed to the linker at build time
	// DO NOT set the value of this variable here
	AdditionalVersion string
	// Version is the current version for pachyderm.
	Version = &protoversion.Version{
		Major:      MajorVersion,
		Minor:      MinorVersion,
		Micro:      MicroVersion,
		Additional: AdditionalVersion,
	}
)

// PrettyPrintVersion returns a version string optionally tagged with metadata.
// For example: "1.2.3", or "1.2.3-rc1" if version.Additional is "rc1".
func PrettyPrintVersion(version *protoversion.Version) string {
	result := fmt.Sprintf("%d.%d.%d", version.Major, version.Minor, version.Micro)
	if version.Additional != "" {
		result += fmt.Sprintf("-%s", version.Additional)
	}
	return result
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package isvcs

import (
	"github.com/zenoss/glog"

	"fmt"
	"net/http"
	"time"
)

var dockerRegistry *IService

const registryPort = 5000

func init() {
	var err error

	defaultHealthCheck := healthCheckDefinition{
		healthCheck: registryHealthCheck,
		Interval:    DEFAULT_HEALTHCHECK_INTERVAL,
		Timeout:     DEFAULT_HEALTHCHECK_TIMEOUT,
	}
	healthChecks := map[string]healthCheckDefinition{
		DEFAULT_HEALTHCHECK_NAME: defaultHealthCheck,
	}

	dockerPortBinding := portBinding{
		HostIp:         "0.0.0.0",
		HostIpOverride: "", // docker registry should always be open
		HostPort:       registryPort,
	}
	command := `SETTINGS_FLAVOR=serviced exec /opt/registry/registry /opt/registry/registry-config.yml`
	dockerRegistry, err = NewIService(
		IServiceDefinition{
			Name:         "docker-registry",
			Repo:         IMAGE_REPO,
			Tag:          IMAGE_TAG,
			Command:      func() string { return command },
			PortBindings: []portBinding{dockerPortBinding},
			Volumes:      map[string]string{"registry": "/tmp/registry"},
			HealthChecks: healthChecks,
		},
	)
	if err != nil {
		glog.Fatalf("Error initializing docker-registry container: %s", err)
	}
}

func registryHealthCheck(halt <-chan struct{}) error {
	url := fmt.Sprintf("http://localhost:%d/", registryPort)
	for {
		if _, err := http.Get(url); err == nil {
			break
		} else {
			glog.V(1).Infof("Still trying to connect to docker registry at %s: %v", url, err)
		}

		select {
		case <-halt:
			glog.V(1).Infof("Quit healthcheck for docker registry at %s", url)
			return nil
		default:
			time.Sleep(time.Second)
		}
	}
	glog.V(1).Infof("docker registry running, browser at %s", url)
	return nil
}
<commit_msg>repointed the docker registry to the correct volume path<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package isvcs

import (
	"github.com/zenoss/glog"

	"fmt"
	"net/http"
	"time"
)

var dockerRegistry *IService

const registryPort = 5000

func init() {
	var err error

	defaultHealthCheck := healthCheckDefinition{
		healthCheck: registryHealthCheck,
		Interval:    DEFAULT_HEALTHCHECK_INTERVAL,
		Timeout:     DEFAULT_HEALTHCHECK_TIMEOUT,
	}
	healthChecks := map[string]healthCheckDefinition{
		DEFAULT_HEALTHCHECK_NAME: defaultHealthCheck,
	}

	dockerPortBinding := portBinding{
		HostIp:         "0.0.0.0",
		HostIpOverride: "", // docker registry should always be open
		HostPort:       registryPort,
	}
	command := `SETTINGS_FLAVOR=serviced exec /opt/registry/registry /opt/registry/registry-config.yml`
	dockerRegistry, err = NewIService(
		IServiceDefinition{
			Name:         "docker-registry",
			Repo:         IMAGE_REPO,
			Tag:          IMAGE_TAG,
			Command:      func() string { return command },
			PortBindings: []portBinding{dockerPortBinding},
			Volumes:      map[string]string{"registry": "/tmp/registry-dev"},
			HealthChecks: healthChecks,
		},
	)
	if err != nil {
		glog.Fatalf("Error initializing docker-registry container: %s", err)
	}
}

func registryHealthCheck(halt <-chan struct{}) error {
	url := fmt.Sprintf("http://localhost:%d/", registryPort)
	for {
		if _, err := http.Get(url); err == nil {
			break
		} else {
			glog.V(1).Infof("Still trying to connect to docker registry at %s: %v", url, err)
		}

		select {
		case <-halt:
			glog.V(1).Infof("Quit healthcheck for docker registry at %s", url)
			return nil
		default:
			time.Sleep(time.Second)
		}
	}
	glog.V(1).Infof("docker registry running, browser at %s", url)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Windows environment variables.

package windows

import (
	"unicode/utf16"
	"unsafe"
)

func Getenv(key string) (value string, found bool) {
	keyp, err := UTF16PtrFromString(key)
	if err != nil {
		return "", false
	}
	n := uint32(100)
	for {
		b := make([]uint16, n)
		n, err = GetEnvironmentVariable(keyp, &b[0], uint32(len(b)))
		if n == 0 && err == ERROR_ENVVAR_NOT_FOUND {
			return "", false
		}
		if n <= uint32(len(b)) {
			return string(utf16.Decode(b[:n])), true
		}
	}
}

func Setenv(key, value string) error {
	v, err := UTF16PtrFromString(value)
	if err != nil {
		return err
	}
	keyp, err := UTF16PtrFromString(key)
	if err != nil {
		return err
	}
	e := SetEnvironmentVariable(keyp, v)
	if e != nil {
		return e
	}
	return nil
}

func Clearenv() {
	for _, s := range Environ() {
		// Environment variables can begin with =
		// so start looking for the separator = at j=1.
		// http://blogs.msdn.com/b/oldnewthing/archive/2010/05/06/10008132.aspx
		for j := 1; j < len(s); j++ {
			if s[j] == '=' {
				Setenv(s[0:j], "")
				break
			}
		}
	}
}

func Environ() []string {
	s, e := GetEnvironmentStrings()
	if e != nil {
		return nil
	}
	defer FreeEnvironmentStrings(s)
	r := make([]string, 0, 50) // Empty with room to grow.
	for from, i, p := 0, 0, (*[1 << 24]uint16)(unsafe.Pointer(s)); true; i++ {
		if p[i] == 0 {
			// empty string marks the end
			if i <= from {
				break
			}
			r = append(r, string(utf16.Decode(p[from:i])))
			from = i + 1
		}
	}
	return r
}
<commit_msg>windows: implement the environment functions by wrapping syscall<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Windows environment variables.

package windows

import "syscall"

func Getenv(key string) (value string, found bool) {
	return syscall.Getenv(key)
}

func Setenv(key, value string) error {
	return syscall.Setenv(key, value)
}

func Clearenv() {
	syscall.Clearenv()
}

func Environ() []string {
	return syscall.Environ()
}
<|endoftext|>


<commit_before>package algoliasearch

import "net/http"
import "net/url"
import "io/ioutil"
import "bytes"
import "encoding/json"
import "strconv"
import "errors"
import "time"
import "reflect"
import "strings"
import "fmt"

// Fix certificates
import _ "crypto/sha512"

const (
	version = "1.3.0"
)

const (
	search = 1 << iota
	write
	read
)

type Transport struct {
	httpClient        *http.Client
	appID             string
	apiKey            string
	headers           map[string]string
	hosts             []string
	hostsProvided     bool
	connectTimeout    time.Duration
	buildReadTimeout  time.Duration
	searchReadTimeout time.Duration
}

func NewTransport(appID, apiKey string) *Transport {
	transport := new(Transport)
	transport.appID = appID
	transport.apiKey = apiKey
	tr := &http.Transport{DisableKeepAlives: false, MaxIdleConnsPerHost: 2}
	transport.httpClient = &http.Client{Transport: tr}
	transport.headers = make(map[string]string)
	//transport.hosts = [3]string{"https://" + appID + suffix[perm[0]], "https://" + appID + suffix[perm[1]], "https://" + appID + suffix[perm[2]], }
	transport.hosts = make([]string, 3)
	transport.hosts[0] = appID + "-1.algolianet.com"
	transport.hosts[1] = appID + "-2.algolianet.com"
	transport.hosts[2] = appID + "-3.algolianet.com"
	transport.hostsProvided = false
	transport.connectTimeout = 1 * time.Second
	transport.buildReadTimeout = 30 * time.Second
	transport.searchReadTimeout = 5 * time.Second
	return transport
}

func NewTransportWithHosts(appID, apiKey string, hosts []string) *Transport {
	transport := new(Transport)
	transport.appID = appID
	transport.apiKey = apiKey
	tr := &http.Transport{DisableKeepAlives: false, MaxIdleConnsPerHost: 2}
	transport.httpClient = &http.Client{Transport: tr}
	transport.headers = make(map[string]string)
	transport.hosts = hosts
	transport.hostsProvided = true
	transport.connectTimeout = 2 * time.Second
	transport.buildReadTimeout = 30 * time.Second
	transport.searchReadTimeout = 5 * time.Second
	return transport
}

func (t *Transport) setTimeout(connectTimeout time.Duration, readTimeout time.Duration) {
	t.connectTimeout = connectTimeout
	t.buildReadTimeout = readTimeout
}

func (t *Transport) setSearchTimeout(searchTimeout time.Duration) {
	t.searchReadTimeout = searchTimeout
}

func (t *Transport) urlEncode(value string) string {
	return url.QueryEscape(value)
}

func (t *Transport) setExtraHeader(key string, value string) {
	t.headers[key] = value
}

func (t *Transport) EncodeParams(params interface{}) string {
	v := url.Values{}
	if params != nil {
		for key, value := range params.(map[string]interface{}) {
			if reflect.TypeOf(value).Name() == "string" {
				v.Add(key, value.(string))
			} else if reflect.TypeOf(value).Name() == "float64" {
				v.Add(key, strconv.FormatFloat(value.(float64), 'f', -1, 64))
			} else if reflect.TypeOf(value).Name() == "int" {
				v.Add(key, strconv.Itoa(value.(int)))
			} else {
				jsonValue, _ := json.Marshal(value)
				v.Add(key, string(jsonValue[:]))
			}
		}
	}
	return v.Encode()
}

func (t *Transport) request(method, path string, body interface{}, typeCall int) (interface{}, error) {
	var host string
	errorMsg := ""
	t.httpClient.Transport.(*http.Transport).TLSHandshakeTimeout = t.connectTimeout
	if typeCall == write {
		host = t.appID + ".algolia.net"
		t.httpClient.Transport.(*http.Transport).ResponseHeaderTimeout = t.searchReadTimeout
	} else {
		host = t.appID + "-dsn.algolia.net"
		if typeCall == read {
			t.httpClient.Transport.(*http.Transport).ResponseHeaderTimeout = t.buildReadTimeout
		} else {
			t.httpClient.Transport.(*http.Transport).ResponseHeaderTimeout = t.searchReadTimeout
		}
	}
	if !t.hostsProvided {
		req, err := t.buildRequest(method, host, path, body)
		if err != nil {
			return nil, err
		}
		req = t.addHeaders(req)
		resp, err := t.httpClient.Do(req)
		if err != nil {
			if len(errorMsg) > 0 {
				errorMsg = fmt.Sprintf("%s, %s:%s", errorMsg, host, err)
			} else {
				errorMsg = fmt.Sprintf("%s:%s", host, err)
			}
		} else if (resp.StatusCode/100) == 2 || (resp.StatusCode/100) == 4 { // Bad request, not found, forbidden
			return t.handleResponse(resp)
		}
	}
	for it := range t.hosts {
		req, err := t.buildRequest(method, t.hosts[it], path, body)
		if err != nil {
			return nil, err
		}
		req = t.addHeaders(req)
		resp, err := t.httpClient.Do(req)
		if err != nil {
			if len(errorMsg) > 0 {
				errorMsg = fmt.Sprintf("%s, %s:%s", errorMsg, t.hosts[it], err)
			} else {
				errorMsg = fmt.Sprintf("%s:%s", t.hosts[it], err)
			}
			continue
		}
		if (resp.StatusCode/100) == 2 || (resp.StatusCode/100) == 4 { // Bad request, not found, forbidden
			return t.handleResponse(resp)
		}
	}
	return nil, errors.New(fmt.Sprintf("Cannot reach any host. (%s)", errorMsg))
}

func (t *Transport) buildRequest(method, host, path string, body interface{}) (*http.Request, error) {
	var req *http.Request
	var err error
	if body != nil {
		bodyBytes, err := json.Marshal(body)
		if err != nil {
			return nil, errors.New("Invalid JSON in the query")
		}
		reader := bytes.NewReader(bodyBytes)
		req, err = http.NewRequest(method, "https://"+host+path, reader)
		req.Header.Add("Content-Length", strconv.Itoa(len(string(bodyBytes))))
		req.Header.Add("Content-Type", "application/json; charset=utf-8")
	} else {
		req, err = http.NewRequest(method, "https://"+host+path, nil)
	}
	if strings.Contains(path, "/*/") {
		req.URL = &url.URL{
			Scheme: "https",
			Host:   host,
			Opaque: "//" + host + path, //Remove url encoding
		}
	}
	t.httpClient.Transport.(*http.Transport).TLSHandshakeTimeout += 2 * time.Second
	t.httpClient.Transport.(*http.Transport).ResponseHeaderTimeout = 10 * time.Second
	return req, err
}

func (t *Transport) addHeaders(req *http.Request) *http.Request {
	req.Header.Add("X-Algolia-API-Key", t.apiKey)
	req.Header.Add("X-Algolia-Application-Id", t.appID)
	req.Header.Add("Connection", "keep-alive")
	req.Header.Add("User-Agent", "Algolia for go "+version)
	for key := range t.headers {
		req.Header.Add(key, t.headers[key])
	}
	return req
}

func (t *Transport) handleResponse(resp *http.Response) (interface{}, error) {
	res, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	var jsonResp interface{}
	err = json.Unmarshal(res, &jsonResp)
	if err != nil {
		return nil, errors.New("Invalid JSON in the response")
	}
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		return jsonResp, nil
	} else {
		return nil, errors.New(string(res))
	}
}
<commit_msg>fix transport, is shared resource. should close response bodies.<commit_after>package algoliasearch

import "net/http"
import "net/url"
import "io/ioutil"
import "bytes"
import "encoding/json"
import "strconv"
import "errors"
import "time"
import "reflect"
import "strings"
import "fmt"

// Fix certificates
import _ "crypto/sha512"

const (
	version = "1.3.0"
)

const (
	search = 1 << iota
	write
	read
)

type Transport struct {
	httpClient    *http.Client
	appID         string
	apiKey        string
	headers       map[string]string
	hosts         []string
	hostsProvided bool
}

func NewTransport(appID, apiKey string) *Transport {
	transport := new(Transport)
	transport.appID = appID
	transport.apiKey = apiKey
	tr := &http.Transport{
		DisableKeepAlives:     false,
		MaxIdleConnsPerHost:   2,
		TLSHandshakeTimeout:   time.Second,
		ResponseHeaderTimeout: time.Second * 10}

	transport.httpClient = &http.Client{Transport: tr}
	transport.headers = make(map[string]string)
	transport.hosts = make([]string, 3)
	transport.hosts[0] = appID + "-1.algolianet.com"
	transport.hosts[1] = appID + "-2.algolianet.com"
	transport.hosts[2] = appID + "-3.algolianet.com"
	transport.hostsProvided = false
	return transport
}

func NewTransportWithHosts(appID, apiKey string, hosts []string) *Transport {
	transport := new(Transport)
	transport.appID = appID
	transport.apiKey = apiKey
	tr := &http.Transport{
		DisableKeepAlives:     false,
		MaxIdleConnsPerHost:   2,
		TLSHandshakeTimeout:   time.Second,
		ResponseHeaderTimeout: time.Second * 10}

	transport.httpClient = &http.Client{Transport: tr}
	transport.headers = make(map[string]string)
	transport.hosts = hosts
	transport.hostsProvided = true
	return transport
}

func (t *Transport) setTimeout(connectTimeout time.Duration, readTimeout time.Duration) {
	t.httpClient.Transport.(*http.Transport).TLSHandshakeTimeout = connectTimeout
	t.httpClient.Transport.(*http.Transport).ResponseHeaderTimeout = readTimeout
}

func (t *Transport) urlEncode(value string) string {
	return url.QueryEscape(value)
}

func (t *Transport) setExtraHeader(key string, value string) {
	t.headers[key] = value
}

func (t *Transport) EncodeParams(params interface{}) string {
	v := url.Values{}
	if params != nil {
		for key, value := range params.(map[string]interface{}) {
			if reflect.TypeOf(value).Name() == "string" {
				v.Add(key, value.(string))
			} else if reflect.TypeOf(value).Name() == "float64" {
				v.Add(key, strconv.FormatFloat(value.(float64), 'f', -1, 64))
			} else if reflect.TypeOf(value).Name() == "int" {
				v.Add(key, strconv.Itoa(value.(int)))
			} else {
				jsonValue, _ := json.Marshal(value)
				v.Add(key, string(jsonValue[:]))
			}
		}
	}
	return v.Encode()
}

func (t *Transport) request(method, path string, body interface{}, typeCall int) (interface{}, error) {
	var host string
	errorMsg := ""
	if typeCall == write {
		host = t.appID + ".algolia.net"
	} else {
		host = t.appID + "-dsn.algolia.net"
	}

	if !t.hostsProvided {
		req, err := t.buildRequest(method, host, path, body)
		if err != nil {
			return nil, err
		}
		req = t.addHeaders(req)
		resp, err := t.httpClient.Do(req)
		if err != nil {
			if len(errorMsg) > 0 {
				errorMsg = fmt.Sprintf("%s, %s:%s", errorMsg, host, err)
			} else {
				errorMsg = fmt.Sprintf("%s:%s", host, err)
			}
		} else if (resp.StatusCode/100) == 2 || (resp.StatusCode/100) == 4 { // Bad request, not found, forbidden
			return t.handleResponse(resp)
		} else {
			resp.Body.Close()
		}
	}

	for it := range t.hosts {
		req, err := t.buildRequest(method, t.hosts[it], path, body)
		if err != nil {
			return nil, err
		}
		req = t.addHeaders(req)
		resp, err := t.httpClient.Do(req)
		if err != nil {
			if len(errorMsg) > 0 {
				errorMsg = fmt.Sprintf("%s, %s:%s", errorMsg, t.hosts[it], err)
			} else {
				errorMsg = fmt.Sprintf("%s:%s", t.hosts[it], err)
			}
			continue
		}
		if (resp.StatusCode/100) == 2 || (resp.StatusCode/100) == 4 { // Bad request, not found, forbidden
			return t.handleResponse(resp)
		} else {
			resp.Body.Close()
		}
	}
	return nil, errors.New(fmt.Sprintf("Cannot reach any host. (%s)", errorMsg))
}

func (t *Transport) buildRequest(method, host, path string, body interface{}) (*http.Request, error) {
	var req *http.Request
	var err error
	if body != nil {
		bodyBytes, err := json.Marshal(body)
		if err != nil {
			return nil, errors.New("Invalid JSON in the query")
		}
		reader := bytes.NewReader(bodyBytes)
		req, err = http.NewRequest(method, "https://"+host+path, reader)
		req.Header.Add("Content-Length", strconv.Itoa(len(string(bodyBytes))))
		req.Header.Add("Content-Type", "application/json; charset=utf-8")
	} else {
		req, err = http.NewRequest(method, "https://"+host+path, nil)
	}
	if strings.Contains(path, "/*/") {
		req.URL = &url.URL{
			Scheme: "https",
			Host:   host,
			Opaque: "//" + host + path, //Remove url encoding
		}
	}
	t.httpClient.Transport.(*http.Transport).TLSHandshakeTimeout += 2 * time.Second
	t.httpClient.Transport.(*http.Transport).ResponseHeaderTimeout = 10 * time.Second
	return req, err
}

func (t *Transport) addHeaders(req *http.Request) *http.Request {
	req.Header.Add("X-Algolia-API-Key", t.apiKey)
	req.Header.Add("X-Algolia-Application-Id", t.appID)
	req.Header.Add("Connection", "keep-alive")
	req.Header.Add("User-Agent", "Algolia for go "+version)
	for key := range t.headers {
		req.Header.Add(key, t.headers[key])
	}
	return req
}

func (t *Transport) handleResponse(resp *http.Response) (interface{}, error) {
	res, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, err
	}
	var jsonResp interface{}
	err = json.Unmarshal(res, &jsonResp)
	if err != nil {
		return nil, errors.New("Invalid JSON in the response")
	}
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		return jsonResp, nil
	} else {
		return nil, errors.New(string(res))
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package elastigo

import (
	"encoding/json"
	"fmt"
	"strings"
)

// The cluster health API allows to get a very simple status on the health of the cluster.
// see http://www.elasticsearch.org/guide/reference/api/admin-cluster-health.html
// TODO: implement wait_for_status, timeout, wait_for_relocating_shards, wait_for_nodes
// TODO: implement level (Can be one of cluster, indices or shards. Controls the details level of the health
// information returned. Defaults to cluster.)
func (c *Conn) Health(indices ...string) (ClusterHealthResponse, error) {
	var url string
	var retval ClusterHealthResponse
	if len(indices) > 0 {
		url = fmt.Sprintf("/_cluster/health/%s", strings.Join(indices, ","))
	} else {
		url = "/_cluster/health"
	}
	body, err := c.DoCommand("GET", url, nil, nil)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	//fmt.Println(body)
	return retval, err
}

func (c *Conn) WaitForStatus(status string, timeout int, indices ...string) (ClusterHealthResponse, error) {
	var url string
	var retval ClusterHealthResponse
	if len(indices) > 0 {
		url = fmt.Sprintf("/_cluster/health/%s", strings.Join(indices, ","))
	} else {
		url = "/_cluster/health"
	}

	body, err := c.DoCommand("GET", url, map[string]interface{}{
		"wait_for_status": status,
		"timout":          timeout,
	}, nil)

	if err != nil {
		return retval, err
	}

	if err == nil {
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}

type ClusterStateFilter struct {
	FilterNodes        bool
	FilterRoutingTable bool
	FilterMetadata     bool
	FilterBlocks       bool
	FilterIndices      []string
}

func (f ClusterStateFilter) Parameterize() []string {
	var parts []string

	if f.FilterNodes {
		parts = append(parts, "filter_nodes=true")
	}

	if f.FilterRoutingTable {
		parts = append(parts, "filter_routing_table=true")
	}

	if f.FilterMetadata {
		parts = append(parts, "filter_metadata=true")
	}

	if f.FilterBlocks {
		parts = append(parts, "filter_blocks=true")
	}

	if f.FilterIndices != nil && len(f.FilterIndices) > 0 {
		parts = append(parts, strings.Join([]string{"filter_indices=", strings.Join(f.FilterIndices, ",")}, ""))
	}

	return parts
}

func (c *Conn) ClusterState(filter ClusterStateFilter) (ClusterStateResponse, error) {
	var parameters []string
	var url string
	var retval ClusterStateResponse

	parameters = filter.Parameterize()

	url = fmt.Sprintf("/_cluster/state?%s", strings.Join(parameters, "&"))

	body, err := c.DoCommand("GET", url, nil, nil)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err

}
<commit_msg>Fixed out of date documentation link<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package elastigo

import (
	"encoding/json"
	"fmt"
	"strings"
)

// The cluster health API allows to get a very simple status on the health of the cluster.
// see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/cluster-health.html
// TODO: implement wait_for_status, timeout, wait_for_relocating_shards, wait_for_nodes
// TODO: implement level (Can be one of cluster, indices or shards. Controls the details level of the health
// information returned. Defaults to cluster.)
func (c *Conn) Health(indices ...string) (ClusterHealthResponse, error) {
	var url string
	var retval ClusterHealthResponse
	if len(indices) > 0 {
		url = fmt.Sprintf("/_cluster/health/%s", strings.Join(indices, ","))
	} else {
		url = "/_cluster/health"
	}
	body, err := c.DoCommand("GET", url, nil, nil)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}

func (c *Conn) WaitForStatus(status string, timeout int, indices ...string) (ClusterHealthResponse, error) {
	var url string
	var retval ClusterHealthResponse
	if len(indices) > 0 {
		url = fmt.Sprintf("/_cluster/health/%s", strings.Join(indices, ","))
	} else {
		url = "/_cluster/health"
	}

	body, err := c.DoCommand("GET", url, map[string]interface{}{
		"wait_for_status": status,
		"timout":          timeout,
	}, nil)

	if err != nil {
		return retval, err
	}

	if err == nil {
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}

type ClusterStateFilter struct {
	FilterNodes        bool
	FilterRoutingTable bool
	FilterMetadata     bool
	FilterBlocks       bool
	FilterIndices      []string
}

func (f ClusterStateFilter) Parameterize() []string {
	var parts []string

	if f.FilterNodes {
		parts = append(parts, "filter_nodes=true")
	}

	if f.FilterRoutingTable {
		parts = append(parts, "filter_routing_table=true")
	}

	if f.FilterMetadata {
		parts = append(parts, "filter_metadata=true")
	}

	if f.FilterBlocks {
		parts = append(parts, "filter_blocks=true")
	}

	if f.FilterIndices != nil && len(f.FilterIndices) > 0 {
		parts = append(parts, strings.Join([]string{"filter_indices=", strings.Join(f.FilterIndices, ",")}, ""))
	}

	return parts
}

func (c *Conn) ClusterState(filter ClusterStateFilter) (ClusterStateResponse, error) {
	var parameters []string
	var url string
	var retval ClusterStateResponse

	parameters = filter.Parameterize()

	url = fmt.Sprintf("/_cluster/state?%s", strings.Join(parameters, "&"))

	body, err := c.DoCommand("GET", url, nil, nil)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"math"
	"os"
)

var (
	// ByteUnits is a k=>v map of units for conversion
	ByteUnits = map[int]string{
		0: "B",
		1: "kB",
		2: "MB",
		3: "GB",
		4: "TB",
	}
)

// ConvertBytes converts bytes to the highest possible unit
func ConvertBytes(in uint64) (float64, string, error) {
	out := float64(in)
	idx := 0
	maxIdx := len(ByteUnits)
	for idx < maxIdx && out > 1 {
		tmp := float64(out) / 1024
		if tmp < 1 {
			break
		}
		out = tmp
		idx++
	}
	return out, ByteUnits[idx], nil
}

// ConvertBytesTo converts bytes to the specified unit
func ConvertBytesTo(in uint64, maxUnit string) (float64, string, error) {
	out := float64(in)
	idx := 0
	maxIdx := len(ByteUnits)
	for idx < maxIdx && maxUnit != ByteUnits[idx] {
		out = float64(out) / 1024
		idx++
	}
	return out, ByteUnits[idx], nil
}

// ConvertNetmask converts CIDR (netmask) to Netmask
func ConvertNetmask(in uint8) (string, error) {
	if in > 32 {
		return "", fmt.Errorf("Invalid Netmask given.")
	}
	octets := map[uint8]uint8{
		1: 0,
		2: 0,
		3: 0,
		4: 0,
	}
	var idx uint8 = 1
	for in > 0 && idx < 5 {
		if (in / 8) > 0 {
			in = in - 8
			octets[idx] = 255
		} else {
			mod := in % 8
			octets[idx] = 255 - uint8(math.Pow(2, float64(8-mod))) + 1
			in = 0
		}
		idx++
	}
	return fmt.Sprintf("%d.%d.%d.%d", octets[1], octets[2], octets[3],
		octets[4]), nil
}

func GetHostEtc() string {
	host_etc := os.Getenv("HOST_ETC")
	if host_etc == "" {
		host_etc = "/etc"
	}
	return host_etc
}

func GetHostSys() string {
	host_sys := os.Getenv("HOST_SYS")
	if host_sys == "" {
		host_sys = "/sys"
	}
	return host_sys
}

func GetHostProc() string {
	host_proc := os.Getenv("HOST_PROC")
	if host_proc == "" {
		host_proc = "/proc"
	}
	return host_proc
}
<commit_msg>Check if Unit has been given in ConvertBytesTo()<commit_after>package common

import (
	"fmt"
	"math"
	"os"
)

var (
	// ByteUnits is a k=>v map of units for conversion
	ByteUnits = map[int]string{
		0: "B",
		1: "kB",
		2: "MB",
		3: "GB",
		4: "TB",
	}
)

// ConvertBytes converts bytes to the highest possible unit
func ConvertBytes(in uint64) (float64, string, error) {
	out := float64(in)
	idx := 0
	maxIdx := len(ByteUnits)
	for idx < maxIdx && out > 1 {
		tmp := float64(out) / 1024
		if tmp < 1 {
			break
		}
		out = tmp
		idx++
	}
	return out, ByteUnits[idx], nil
}

// ConvertBytesTo converts bytes to the specified unit
func ConvertBytesTo(in uint64, maxUnit string) (float64, string, error) {
	if maxUnit == "" {
		return 0, "", fmt.Errorf("Given maximum unit is invalid.")
	}
	out := float64(in)
	idx := 0
	maxIdx := len(ByteUnits)
	for idx < maxIdx && maxUnit != ByteUnits[idx] {
		out = float64(out) / 1024
		idx++
	}
	return out, ByteUnits[idx], nil
}

// ConvertNetmask converts CIDR (netmask) to Netmask
func ConvertNetmask(in uint8) (string, error) {
	if in > 32 {
		return "", fmt.Errorf("Invalid Netmask given.")
	}
	octets := map[uint8]uint8{
		1: 0,
		2: 0,
		3: 0,
		4: 0,
	}
	var idx uint8 = 1
	for in > 0 && idx < 5 {
		if (in / 8) > 0 {
			in = in - 8
			octets[idx] = 255
		} else {
			mod := in % 8
			octets[idx] = 255 - uint8(math.Pow(2, float64(8-mod))) + 1
			in = 0
		}
		idx++
	}
	return fmt.Sprintf("%d.%d.%d.%d", octets[1], octets[2], octets[3],
		octets[4]), nil
}

func GetHostEtc() string {
	host_etc := os.Getenv("HOST_ETC")
	if host_etc == "" {
		host_etc = "/etc"
	}
	return host_etc
}

func GetHostSys() string {
	host_sys := os.Getenv("HOST_SYS")
	if host_sys == "" {
		host_sys = "/sys"
	}
	return host_sys
}

func GetHostProc() string {
	host_proc := os.Getenv("HOST_PROC")
	if host_proc == "" {
		host_proc = "/proc"
	}
	return host_proc
}
<|endoftext|>