<commit_before>package pure

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"reflect"
	"strconv"
)

type state int

const tagName = "pure"

type unmarshaler struct {
	Scanner  *scanner
	errors   []*pureError
	tagID    string
	tagValue string
	tagTok   Token
	tagTyp   string
}

// Shamelessly stolen from the Golang JSON decode source. Forgive
func (u *unmarshaler) indirect(v reflect.Value) reflect.Value {
	if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
		v = v.Addr()
	}

	for {
		if v.Kind() == reflect.Interface && !v.IsNil() {
			e := v.Elem()
			if e.Kind() == reflect.Ptr && !e.IsNil() {
				v = e
				continue
			}
		}

		if v.Kind() != reflect.Ptr {
			break
		}

		if v.Elem().Kind() != reflect.Ptr && v.CanSet() {
			break
		}

		if v.IsNil() {
			v.Set(reflect.New(v.Type().Elem()))
		}

		if v.Type().NumMethod() > 0 {
			// TODO
		}

		v = v.Elem()
	}
	return v
}

func (u *unmarshaler) newError(msg string) *pureError {
	s := fmt.Sprintf("Error unmarhsaling Pure property: %s\r\n[%d:%d]-%s", msg, u.Scanner.line, u.Scanner.col, string(u.Scanner.buf.Bytes()))
	err := &pureError{}
	err.error = fmt.Errorf(s)
	return err
}

func (u *unmarshaler) Scan() (tok Token, lit string) {
	return u.Scanner.Scan()
}

func (u *unmarshaler) ScanSkipWhitespace() (tok Token, lit string) {
	for tok, lit = u.Scanner.Scan(); tok == WHITESPACE; {
		tok, lit = u.Scanner.Scan()
	}
	return
}

func (u *unmarshaler) field(v reflect.Value) *pureError {
	var field reflect.Value

	switch v.Kind() {
	case reflect.Ptr:
		iv := u.indirect(v.Elem())
		for i := 0; i < iv.NumField(); i++ {
			tag := iv.Type().Field(i).Tag.Get(tagName)
			if tag != "" && tag != "-" && tag == u.tagID {
				field = iv.Field(i)
				break
			}
		}
	case reflect.Struct:
		iv := u.indirect(v)
		tv := reflect.TypeOf(v.Interface())

		for i := 0; i < iv.NumField(); i++ {
			tag := tv.Field(i).Tag.Get(tagName)
			if tag != "" && tag != "-" && tag == u.tagID {
				field = iv.Field(i)
				break
			}

		}
	}

	if !field.IsValid() {
		field = u.indirect(v)
	}

	switch {
	case field.Kind() == reflect.Int && u.tagTyp == "int":
		_i, err := strconv.Atoi(u.tagValue)
		if err != nil {
			return u.newError(fmt.Sprintf("bad number value '%s'", u.tagValue))
		}
		field.SetInt(int64(_i))
		return nil
	case field.Kind() == reflect.String && (u.tagTyp == "string" || u.tagTyp == "quantity" || u.tagTyp == "path"):
		field.SetString(u.tagValue)
		return nil
	case field.Kind() == reflect.Float64 && u.tagTyp == "double":
		f, err := strconv.ParseFloat(u.tagValue, 64)
		if err != nil {
			return u.newError(fmt.Sprintf("bad floating point value '%s'", u.tagValue))
		}
		field.SetFloat(f)
		return nil
	case field.Kind() == reflect.Bool && u.tagTyp == "bool":
		b, err := strconv.ParseBool(u.tagValue)
		if err != nil {
			return u.newError(fmt.Sprintf("bad bool value '%s'", u.tagValue))
		}
		field.SetBool(b)
		return nil
	case field.Kind() == reflect.Ptr && u.tagTyp == "group":
		return u.group(field.Interface())
	default:
		fi := field.Interface()
		switch fi.(type) {
		case *Quantity:
			if u.tagTyp != "quantity" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Quantity' & '%s", u.tagTyp)))
				return nil
			}
			fi = NewQuantity(u.tagValue)
			field.Set(reflect.ValueOf(fi))
			return nil
		case Quantity:
			if u.tagTyp != "quantity" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Quantity' & '%s", u.tagTyp)))
				return nil
			}
			fi = NewQuantity(u.tagValue)
			field.Set(u.indirect(reflect.ValueOf(fi)))
			return nil
		case *Env:
			if u.tagTyp != "env" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Env' & '%s'", u.tagTyp)))
				return nil
			}
			fi = NewEnv(u.tagValue)
			field.Set(reflect.ValueOf(fi))
			return nil
		case Env:
			if u.tagTyp != "env" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Env' & '%s'", u.tagTyp)))
				return nil
			}
			fi = NewEnv(u.tagValue)
			field.Set(u.indirect(reflect.ValueOf(fi)))
			return nil
		case *Path:
			if u.tagTyp != "path" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Path' & '%s'", u.tagTyp)))
				return nil
			}
			fi = NewPath(u.tagValue)
			field.Set(reflect.ValueOf(fi))
			return nil
		case Path:
			if u.tagTyp != "path" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Path' & '%s'", u.tagTyp)))
				return nil
			}
			fi = NewPath(u.tagValue)
			field.Set(u.indirect(reflect.ValueOf(fi)))
			return nil
		}

	}
	return nil
}

// Peek copies the unmarshalers buffer (to not advance the buffer we're reading form)
// and returns the next n bytes
func (u *unmarshaler) Peek(n int) []byte {
	return bytes.NewBuffer(u.Scanner.buf.Bytes()).Next(n)
}

// This is not pretty, but it works ¯\_(ツ)_/¯
func (u *unmarshaler) PeekLiteral() string {
	buf := bytes.NewBuffer(u.Scanner.buf.Bytes())
	for {
		b, _ := buf.ReadByte()

		if IsAlpha(b) {
			buf.WriteByte(b)
			for {
				b, _ := buf.ReadByte()
				if IsWhitespace(b) {
					break
				}
				buf.WriteByte(b)
			}
			break
		}
	}
	return buf.String()
}

func (u *unmarshaler) group(v interface{}) *pureError {
	iv := u.indirect(reflect.ValueOf(v))
	tv := reflect.TypeOf(v)
	for i := 0; i < iv.NumField(); i++ {
		tag := tv.Elem().Field(i).Tag.Get(tagName)

		if tag == u.tagID {
			f := iv.Field(i)
			for {
				tok, lit := u.Scan()
				if tok == EOF {
					return nil
				}

				if lit == "\r" {
					if b := u.Peek(2); b[0] == '\n' && (IsWhitespace(b[len(b)] - 1)) {
						continue
					}
					return nil
				}

				if lit == " " || lit == "\n" || lit == "\t" {
					continue
				}
				if tok == DOT || lit == "." {
					tok, lit = u.ScanSkipWhitespace()
				}

				if tok == GROUP {
					struc := u.GetStruct(u.tagID, v)
					field := u.GetField(lit, struc)
					u.tagID = u.PeekLiteral()
					err := u.group(field.Interface())
					if err != nil {
						fmt.Println(err.Error())
					}
				}

				u.tagID = lit

				tok, lit = u.ScanSkipWhitespace()

				if tok == EQUALS {
					u.tagTok, u.tagValue = u.ScanSkipWhitespace()
				}

				switch u.tagTok {
				case STRING, IDENTIFIER:
					u.tagTyp = "string"
				case INT:
					u.tagTyp = "int"
				case DOUBLE:
					u.tagTyp = "double"
				case BOOL:
					u.tagTyp = "bool"
				case QUANTITY:
					u.tagTyp = "quantity"
				case ENV:
					u.tagTyp = "env"
				case PATH:
					u.tagTyp = "path"
				}

				err := u.field(f)
				if err != nil {
					fmt.Println(err.Error())
				}
			}
		}
	}
	return nil
}

func (u *unmarshaler) GetStruct(name string, v interface{}) reflect.Value {
	iv := u.indirect(reflect.ValueOf(v))
	for i := 0; i < iv.NumField(); i++ {
		tag := reflect.TypeOf(v).Elem().Field(i).Tag.Get(tagName)
		if tag == name {
			return iv.Field(i)
		}
	}
	return reflect.Zero(nil)
}

func (u *unmarshaler) GetField(name string, v reflect.Value) reflect.Value {
	if v.Kind() == reflect.Ptr {
		iv := u.indirect(v.Elem())

		for i := 0; i < iv.NumField(); i++ {
			tag := iv.Type().Field(i).Tag.Get(tagName)

			if tag == name {
				return iv.Field(i)
			}
		}
	}

	if v.Kind() == reflect.Struct {
		iv := u.indirect(v)
		tv := reflect.TypeOf(v.Interface())
		for i := 0; i < iv.NumField(); i++ {
			tag := tv.Field(i).Tag.Get(tagName)
			if tag == "" {
				tag = iv.Type().Field(i).Tag.Get(tagName)
			}
			if tag == name || tag == u.tagID {
				if iv.Kind() == reflect.Struct || iv.Kind() == reflect.Ptr {
					return u.GetField(u.tagID, reflect.ValueOf(iv.Field(i)))
				}
				return iv.Field(i)
			}
		}
	}

	return v
}

// Gotta pretty this up it's really ugly
// Makes me wanna vomit
func (u *unmarshaler) unmarshal(v interface{}) {
	pv := u.indirect(reflect.ValueOf(v))
	for {
		tok, lit := u.ScanSkipWhitespace()
		u.tagID = lit
		u.tagTok = tok

		if tok == EOF {
			return
		}

		switch tok {
		case IDENTIFIER:
			if tok, _ := u.ScanSkipWhitespace(); tok == EQUALS {
				u.tagTok, u.tagValue = u.ScanSkipWhitespace()
				switch u.tagTok {
				case INT:
					u.tagTyp = "int"
				case DOUBLE:
					u.tagTyp = "double"
				case STRING, IDENTIFIER:
					u.tagTyp = "string"
				case BOOL:
					u.tagTyp = "bool"
				case QUANTITY:
					u.tagTyp = "quantity"
				case ENV:
					u.tagTyp = "env"
				case PATH:
					u.tagTyp = "path"
				}
			} else if tok == REF {
				var field reflect.Value
				temp := lit
				tok, lit = u.ScanSkipWhitespace()
				if b := u.Peek(1); b[0] == '.' {
					group := lit
					tok, lit = u.ScanSkipWhitespace()
					tok, lit = u.ScanSkipWhitespace()
					u.tagID = lit
					struc := u.GetStruct(group, v)
					u.tagID = temp
					field = u.GetField(lit, struc)
				} else {
					// Assume it's a regular property and not a group property
					tok, lit = u.ScanSkipWhitespace()
					field = u.GetField(u.tagID, u.indirect(reflect.ValueOf(v)))
				}
				switch field.Kind() {
				case reflect.Int:
					u.tagTyp = "int"
					u.tagValue = strconv.Itoa(int(field.Int()))
				case reflect.Float64:
					u.tagTyp = "double"
					u.tagValue = strconv.FormatFloat(field.Float(), 'f', 16, 64)
				case reflect.String:
					u.tagTyp = "string"
					u.tagValue = field.String()
				case reflect.Bool:
					u.tagTyp = "bool"
					u.tagValue = strconv.FormatBool(field.Bool())
				default:
					fi := field.Interface()

					switch fi.(type) {
					case *Quantity:
						u.tagTyp = "quantity"
						u.tagValue = fi.(*Quantity).value
					case Quantity:
						u.tagTyp = "quantity"
						u.tagValue = fi.(Quantity).value
					case *Env:
						u.tagTyp = "env"
						u.tagValue = fi.(*Env).value
					case Env:
						u.tagTyp = "env"
						u.tagValue = fi.(Env).value
					case *Path:
						u.tagTyp = "path"
						u.tagValue = fi.(*Path).value
					case Path:
						u.tagTyp = "Path"
						u.tagValue = fi.(Path).value
					}
				}
			}

			err := u.field(pv)
			if err != nil {
				u.errors = append(u.errors, err)
			}

		case GROUP:
			err := u.group(v)
			if err != nil {
				u.errors = append(u.errors, err)
			}
		case INCLUDE:
			b, err := ioutil.ReadFile(lit)
			if err != nil {
				fmt.Println(u.newError(err.Error()).Error())
			}
			Unmarshal(b, v)
		}
	}
}

func Unmarshal(b []byte, v interface{}) *pureError {
	u := &unmarshaler{
		Scanner: newScanner(b),
	}
	u.unmarshal(v)

	// Should improve error reporting
	// Maybe as soon as they're discovered?
	if len(u.errors) > 0 {
		return u.errors[0]
	}
	return nil
}
<commit_msg>added some comments<commit_after>package pure

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"reflect"
	"strconv"
)

type state int

const tagName = "pure"

type unmarshaler struct {
	Scanner  *scanner
	errors   []*pureError
	tagID    string
	tagValue string
	tagTok   Token
	tagTyp   string
}

// Shamelessly stolen from the Golang JSON decode source. Forgive
func (u *unmarshaler) indirect(v reflect.Value) reflect.Value {
	if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
		v = v.Addr()
	}

	for {
		if v.Kind() == reflect.Interface && !v.IsNil() {
			e := v.Elem()
			if e.Kind() == reflect.Ptr && !e.IsNil() {
				v = e
				continue
			}
		}

		if v.Kind() != reflect.Ptr {
			break
		}

		if v.Elem().Kind() != reflect.Ptr && v.CanSet() {
			break
		}

		if v.IsNil() {
			v.Set(reflect.New(v.Type().Elem()))
		}

		if v.Type().NumMethod() > 0 {
			// TODO
		}

		v = v.Elem()
	}
	return v
}

func (u *unmarshaler) newError(msg string) *pureError {
	s := fmt.Sprintf("Error unmarhsaling Pure property: %s\r\n[%d:%d]-%s", msg, u.Scanner.line, u.Scanner.col, string(u.Scanner.buf.Bytes()))
	err := &pureError{}
	err.error = fmt.Errorf(s)
	return err
}

func (u *unmarshaler) Scan() (tok Token, lit string) {
	return u.Scanner.Scan()
}

func (u *unmarshaler) ScanSkipWhitespace() (tok Token, lit string) {
	for tok, lit = u.Scanner.Scan(); tok == WHITESPACE; {
		tok, lit = u.Scanner.Scan()
	}
	return
}

func (u *unmarshaler) field(v reflect.Value) *pureError {
	var field reflect.Value

	switch v.Kind() {
	case reflect.Ptr:
		iv := u.indirect(v.Elem())
		for i := 0; i < iv.NumField(); i++ {
			tag := iv.Type().Field(i).Tag.Get(tagName)
			if tag != "" && tag != "-" && tag == u.tagID {
				field = iv.Field(i)
				break
			}
		}
	case reflect.Struct:
		iv := u.indirect(v)
		tv := reflect.TypeOf(v.Interface())

		for i := 0; i < iv.NumField(); i++ {
			tag := tv.Field(i).Tag.Get(tagName)
			if tag != "" && tag != "-" && tag == u.tagID {
				field = iv.Field(i)
				break
			}

		}
	}

	if !field.IsValid() {
		field = u.indirect(v)
	}
	// There has to be a better way for this
	switch {
	case field.Kind() == reflect.Int && u.tagTyp == "int":
		_i, err := strconv.Atoi(u.tagValue)
		if err != nil {
			return u.newError(fmt.Sprintf("bad number value '%s'", u.tagValue))
		}
		field.SetInt(int64(_i))
		return nil
	case field.Kind() == reflect.String && u.tagTyp == "string":
		field.SetString(u.tagValue)
		return nil
	case field.Kind() == reflect.Float64 && u.tagTyp == "double":
		f, err := strconv.ParseFloat(u.tagValue, 64)
		if err != nil {
			return u.newError(fmt.Sprintf("bad floating point value '%s'", u.tagValue))
		}
		field.SetFloat(f)
		return nil
	case field.Kind() == reflect.Bool && u.tagTyp == "bool":
		b, err := strconv.ParseBool(u.tagValue)
		if err != nil {
			return u.newError(fmt.Sprintf("bad bool value '%s'", u.tagValue))
		}
		field.SetBool(b)
		return nil
	case field.Kind() == reflect.Ptr && u.tagTyp == "group":
		return u.group(field.Interface())
	default:
		fi := field.Interface()
		switch fi.(type) {
		case *Quantity:
			if u.tagTyp != "quantity" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Quantity' & '%s", u.tagTyp)))
				return nil
			}
			fi = NewQuantity(u.tagValue)
			field.Set(reflect.ValueOf(fi))
			return nil
		case Quantity:
			if u.tagTyp != "quantity" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Quantity' & '%s", u.tagTyp)))
				return nil
			}
			fi = NewQuantity(u.tagValue)
			field.Set(u.indirect(reflect.ValueOf(fi)))
			return nil
		case *Env:
			if u.tagTyp != "env" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Env' & '%s'", u.tagTyp)))
				return nil
			}
			fi = NewEnv(u.tagValue)
			field.Set(reflect.ValueOf(fi))
			return nil
		case Env:
			if u.tagTyp != "env" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Env' & '%s'", u.tagTyp)))
				return nil
			}
			fi = NewEnv(u.tagValue)
			field.Set(u.indirect(reflect.ValueOf(fi)))
			return nil
		case *Path:
			if u.tagTyp != "path" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Path' & '%s'", u.tagTyp)))
				return nil
			}
			fi = NewPath(u.tagValue)
			field.Set(reflect.ValueOf(fi))
			return nil
		case Path:
			if u.tagTyp != "path" {
				fmt.Println(u.newError(fmt.Sprintf("mismatched types 'Path' & '%s'", u.tagTyp)))
				return nil
			}
			fi = NewPath(u.tagValue)
			field.Set(u.indirect(reflect.ValueOf(fi)))
			return nil
		}

	}
	return nil
}

// Peek copies the unmarshalers buffer (to not advance the buffer we're reading form)
// and returns the next n bytes
func (u *unmarshaler) Peek(n int) []byte {
	return bytes.NewBuffer(u.Scanner.buf.Bytes()).Next(n)
}

// This is not pretty, but it works ¯\_(ツ)_/¯
func (u *unmarshaler) PeekLiteral() string {
	buf := bytes.NewBuffer(u.Scanner.buf.Bytes())
	for {
		b, _ := buf.ReadByte()

		if IsAlpha(b) {
			buf.WriteByte(b)
			for {
				b, _ := buf.ReadByte()
				if IsWhitespace(b) {
					break
				}
				buf.WriteByte(b)
			}
			break
		}
	}
	return buf.String()
}

func (u *unmarshaler) group(v interface{}) *pureError {
	iv := u.indirect(reflect.ValueOf(v))
	tv := reflect.TypeOf(v)
	for i := 0; i < iv.NumField(); i++ {
		tag := tv.Elem().Field(i).Tag.Get(tagName)

		if tag == u.tagID {
			f := iv.Field(i)
			for {
				tok, lit := u.Scan()
				if tok == EOF {
					return nil
				}

				if lit == "\r" {
					if b := u.Peek(2); b[0] == '\n' && (IsWhitespace(b[len(b)] - 1)) {
						continue
					}
					return nil
				}

				if lit == " " || lit == "\n" || lit == "\t" {
					continue
				}
				if tok == DOT || lit == "." {
					tok, lit = u.ScanSkipWhitespace()
				}

				if tok == GROUP {
					struc := u.GetStruct(u.tagID, v)
					field := u.GetField(lit, struc)
					u.tagID = u.PeekLiteral()
					err := u.group(field.Interface())
					if err != nil {
						fmt.Println(err.Error())
					}
				}

				u.tagID = lit

				tok, lit = u.ScanSkipWhitespace()

				if tok == EQUALS {
					u.tagTok, u.tagValue = u.ScanSkipWhitespace()
				}

				switch u.tagTok {
				case STRING, IDENTIFIER:
					u.tagTyp = "string"
				case INT:
					u.tagTyp = "int"
				case DOUBLE:
					u.tagTyp = "double"
				case BOOL:
					u.tagTyp = "bool"
				case QUANTITY:
					u.tagTyp = "quantity"
				case ENV:
					u.tagTyp = "env"
				case PATH:
					u.tagTyp = "path"
				}

				err := u.field(f)
				if err != nil {
					fmt.Println(err.Error())
				}
			}
		}
	}
	return nil
}

func (u *unmarshaler) GetStruct(name string, v interface{}) reflect.Value {
	iv := u.indirect(reflect.ValueOf(v))
	for i := 0; i < iv.NumField(); i++ {
		tag := reflect.TypeOf(v).Elem().Field(i).Tag.Get(tagName)
		if tag == name {
			return iv.Field(i)
		}
	}
	return reflect.Zero(nil)
}

func (u *unmarshaler) GetField(name string, v reflect.Value) reflect.Value {
	if v.Kind() == reflect.Ptr {
		iv := u.indirect(v.Elem())

		for i := 0; i < iv.NumField(); i++ {
			tag := iv.Type().Field(i).Tag.Get(tagName)

			if tag == name {
				return iv.Field(i)
			}
		}
	}

	if v.Kind() == reflect.Struct {
		iv := u.indirect(v)
		tv := reflect.TypeOf(v.Interface())
		for i := 0; i < iv.NumField(); i++ {
			tag := tv.Field(i).Tag.Get(tagName)
			if tag == "" {
				tag = iv.Type().Field(i).Tag.Get(tagName)
			}
			if tag == name || tag == u.tagID {
				if iv.Kind() == reflect.Struct || iv.Kind() == reflect.Ptr {
					return u.GetField(u.tagID, reflect.ValueOf(iv.Field(i)))
				}
				return iv.Field(i)
			}
		}
	}

	return v
}

// Gotta pretty this up it's really ugly
// Makes me wanna vomit
func (u *unmarshaler) unmarshal(v interface{}) {
	pv := u.indirect(reflect.ValueOf(v))
	for {
		tok, lit := u.ScanSkipWhitespace()
		u.tagID = lit
		u.tagTok = tok

		if tok == EOF {
			return
		}

		switch tok {
		case IDENTIFIER:
			// Check if the next token is a an '='
			if tok, _ := u.ScanSkipWhitespace(); tok == EQUALS {
				// Consume the '=' and get the token and value for the property
				u.tagTok, u.tagValue = u.ScanSkipWhitespace()
				// type check
				switch u.tagTok {
				case INT:
					u.tagTyp = "int"
				case DOUBLE:
					u.tagTyp = "double"
				case STRING, IDENTIFIER:
					u.tagTyp = "string"
				case BOOL:
					u.tagTyp = "bool"
				case QUANTITY:
					u.tagTyp = "quantity"
				case ENV:
					u.tagTyp = "env"
				case PATH:
					u.tagTyp = "path"
				}
				// Else if it's a reference
			} else if tok == REF {
				var field reflect.Value
				// Store the token id in temp
				// and get the next token
				temp := lit
				tok, lit = u.ScanSkipWhitespace()

				// If the peeked token is a '.' then we're going into a group
				// so lit MUST be a group id (ex. 'someGroupId')
				if b := u.Peek(1); b[0] == '.' {
					group := lit
					//Consime the '.'
					tok, lit = u.ScanSkipWhitespace()
					// Get the property id
					tok, lit = u.ScanSkipWhitespace()
					u.tagID = lit
					// Get the struct with the correct tag id from 'v'
					struc := u.GetStruct(group, v)
					// reset the tag id from the temp value
					u.tagID = temp
					// get the field inside the struct we just got, with the tag id
					field = u.GetField(lit, struc)
				} else {
					// If there's no '.'
					// assume it's a regular property and not a group property
					tok, lit = u.ScanSkipWhitespace()
					field = u.GetField(u.tagID, u.indirect(reflect.ValueOf(v)))
				}
				// type check
				// this can probably be made prettier
				switch field.Kind() {
				case reflect.Int:
					u.tagTyp = "int"
					u.tagValue = strconv.Itoa(int(field.Int()))
				case reflect.Float64:
					u.tagTyp = "double"
					u.tagValue = strconv.FormatFloat(field.Float(), 'f', 16, 64)
				case reflect.String:
					u.tagTyp = "string"
					u.tagValue = field.String()
				case reflect.Bool:
					u.tagTyp = "bool"
					u.tagValue = strconv.FormatBool(field.Bool())
				default:
					fi := field.Interface()

					switch fi.(type) {
					case *Quantity:
						u.tagTyp = "quantity"
						u.tagValue = fi.(*Quantity).value
					case Quantity:
						u.tagTyp = "quantity"
						u.tagValue = fi.(Quantity).value
					case *Env:
						u.tagTyp = "env"
						u.tagValue = fi.(*Env).value
					case Env:
						u.tagTyp = "env"
						u.tagValue = fi.(Env).value
					case *Path:
						u.tagTyp = "path"
						u.tagValue = fi.(*Path).value
					case Path:
						u.tagTyp = "Path"
						u.tagValue = fi.(Path).value
					}
				}
			}

			// assign the value to the field
			err := u.field(pv)
			if err != nil {
				u.errors = append(u.errors, err)
			}

		case GROUP:
			err := u.group(v)
			if err != nil {
				u.errors = append(u.errors, err)
			}
		case INCLUDE:
			// if we're including a file all we do is unmarshal that BEFORE we do anything else
			b, err := ioutil.ReadFile(lit)
			if err != nil {
				fmt.Println(u.newError(err.Error()).Error())
			}
			Unmarshal(b, v)
		}
	}
}

func Unmarshal(b []byte, v interface{}) *pureError {
	u := &unmarshaler{
		Scanner: newScanner(b),
	}
	u.unmarshal(v)

	// Should improve error reporting
	// Maybe as soon as they're discovered?
	if len(u.errors) > 0 {
		return u.errors[0]
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"golang.org/x/crypto/ssh"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/go-itchio"
	"github.com/itchio/wharf"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/sync"
	"github.com/itchio/wharf/tlc"
)

func push(buildPath string, spec string) {
	must(doPush(buildPath, spec))
}

const (
	defaultPort = 22
)

func doPush(buildPath string, spec string) error {
	sourceContainerChan := make(chan *tlc.Container)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs)

	address := *pushArgs.address

	if !strings.Contains(address, ":") {
		address = fmt.Sprintf("%s:%d", address, defaultPort)
	}

	comm.Logf("Connecting to %s", address)
	conn, err := wharf.Connect(address, *pushArgs.identity, "butler", version)
	if err != nil {
		return err
	}
	defer conn.Close()
	comm.Logf("Connected to %s", conn.Conn.RemoteAddr())

	go ssh.DiscardRequests(conn.Reqs)

	req := &wharf.AuthenticationRequest{}
	res := &wharf.AuthenticationResponse{}

	err = conn.SendRequest("authenticate", req, res)
	if err != nil {
		return fmt.Errorf("Authentication error; %s", err.Error())
	}

	// TODO: if buildPath is an archive, warn and unpack it

	client := itchio.ClientWithKey(res.Key)

	target, channel, err := parseSpec(spec)
	if err != nil {
		return err
	}

	newBuildRes, err := client.CreateBuild(target, channel)
	if err != nil {
		return err
	}

	buildID := newBuildRes.ID
	parentID := newBuildRes.ParentID

	var targetSignature []sync.BlockHash
	var targetContainer *tlc.Container

	if parentID == 0 {
		comm.Logf("This is the first build for channel %s", channel)
		targetSignature = make([]sync.BlockHash, 0)
		targetContainer = &tlc.Container{}
	} else {
		comm.Logf("Latest build for channel %s is %d, downloading its signature", channel, parentID)
		buildFiles, err := client.ListBuildFiles(parentID)
		if err != nil {
			return err
		}

		var signatureFileID int64 = 0
		for _, f := range buildFiles.Files {
			if f.Type == itchio.BuildFileType_SIGNATURE {
				signatureFileID = f.ID
				break
			}
		}

		if signatureFileID == 0 {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		signatureReader, err := client.DownloadBuildFile(parentID, signatureFileID)

		targetContainer, targetSignature, err = pwr.ReadSignature(signatureReader)
		if err != nil {
			return err
		}
	}

	newPatchRes, err := client.CreateBuildFile(buildID, itchio.BuildFileType_PATCH)
	if err != nil {
		return err
	}

	uploadPatchReq, patchWriter, err := newMultipartUpload(newPatchRes.UploadURL, fmt.Sprintf("%d-%d.pwr", parentID, buildID))
	if err != nil {
		return err
	}

	newSignatureRes, err := client.CreateBuildFile(buildID, itchio.BuildFileType_SIGNATURE)
	if err != nil {
		return err
	}

	uploadSignatureReq, signatureWriter, err := newMultipartUpload(newSignatureRes.UploadURL, fmt.Sprintf("%d-%d.pwr", parentID, buildID))
	if err != nil {
		return err
	}

	done := make(chan bool)
	errs := make(chan error)

	go doReq(uploadPatchReq, done, errs)
	go doReq(uploadSignatureReq, done, errs)

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	go func() {
		defer patchWriter.Close()
		defer signatureWriter.Close()

		// we started walking the source container in the beginning,
		// we actually need it now.
		// note that we could actually start diffing before all the file
		// creation & upload setup is done

		var sourceContainer *tlc.Container

		select {
		case err := <-walkErrs:
			errs <- err
			return
		case sourceContainer = <-sourceContainerChan:
			break
		}

		dctx := &pwr.DiffContext{
			Compression: &pwr.CompressionSettings{
				Algorithm: pwr.CompressionAlgorithm_BROTLI,
				Quality:   1,
			},

			SourceContainer: sourceContainer,
			SourcePath:      buildPath,

			TargetContainer: targetContainer,
			TargetSignature: targetSignature,

			Consumer: comm.NewStateConsumer(),
		}

		comm.Logf("Sending patch and signature computed on-the-fly...")
		comm.StartProgress()
		err := dctx.WritePatch(patchCounter, signatureCounter)
		comm.EndProgress()

		if err != nil {
			errs <- err
			return
		}

		{
			prettyPatchSize := humanize.Bytes(uint64(patchCounter.Count()))
			percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
			relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
			prettyFreshSize := humanize.Bytes(uint64(dctx.FreshBytes))

			comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
			comm.Statf("%s patch (%.2f%% of the full size)", prettyPatchSize, relToNew)
		}

		_, err = client.FinalizeBuildFile(buildID, newPatchRes.ID, patchCounter.Count())
		if err != nil {
			errs <- err
			return
		}

		_, err = client.FinalizeBuildFile(buildID, newSignatureRes.ID, signatureCounter.Count())
		if err != nil {
			errs <- err
			return
		}

		done <- true
	}()

	// wait for all goroutines to complete
	for c := 0; c < 3; c++ {
		select {
		case err := <-errs:
			return err
		case <-done:
		}
	}
	comm.Logf("Everything sent off!")

	return nil
}

func doWalk(path string, result chan *tlc.Container, errs chan error) {
	// TODO: allow custom filter function
	container, err := tlc.Walk(path, nil)
	if err != nil {
		errs <- err
	}

	result <- container
}

func parseSpec(spec string) (string, string, error) {
	tokens := strings.Split(spec, ":")

	if len(tokens) == 1 {
		return "", "", fmt.Errorf("invalid spec: %s, missing channel (examples: %s:windows-32-beta, %s:linux-64)", spec, spec, spec)
	} else if len(tokens) != 2 {
		return "", "", fmt.Errorf("invalid spec: %s, expected something of the form user/page:channel", spec)
	}

	return tokens[0], tokens[1], nil
}

func parseAddress(address string) string {
	if strings.Contains(address, ":") {
		return address
	} else {
		return fmt.Sprintf("%s:%d", address, defaultPort)
	}
}
<commit_msg>Close ssh connection explicitly<commit_after>package main

import (
	"fmt"
	"strings"

	"golang.org/x/crypto/ssh"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/go-itchio"
	"github.com/itchio/wharf"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/sync"
	"github.com/itchio/wharf/tlc"
)

func push(buildPath string, spec string) {
	must(doPush(buildPath, spec))
}

const (
	defaultPort = 22
)

func doPush(buildPath string, spec string) error {
	sourceContainerChan := make(chan *tlc.Container)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs)

	address := *pushArgs.address

	if !strings.Contains(address, ":") {
		address = fmt.Sprintf("%s:%d", address, defaultPort)
	}

	comm.Logf("Connecting to %s", address)
	conn, err := wharf.Connect(address, *pushArgs.identity, "butler", version)
	if err != nil {
		return err
	}
	comm.Logf("Connected to %s", conn.Conn.RemoteAddr())

	go ssh.DiscardRequests(conn.Reqs)

	req := &wharf.AuthenticationRequest{}
	res := &wharf.AuthenticationResponse{}

	err = conn.SendRequest("authenticate", req, res)
	if err != nil {
		return fmt.Errorf("Authentication error; %s", err.Error())
	}

	err = conn.Close()
	if err != nil {
		return err
	}

	// TODO: if buildPath is an archive, warn and unpack it

	client := itchio.ClientWithKey(res.Key)

	target, channel, err := parseSpec(spec)
	if err != nil {
		return err
	}

	newBuildRes, err := client.CreateBuild(target, channel)
	if err != nil {
		return err
	}

	buildID := newBuildRes.ID
	parentID := newBuildRes.ParentID

	var targetSignature []sync.BlockHash
	var targetContainer *tlc.Container

	if parentID == 0 {
		comm.Logf("This is the first build for channel %s", channel)
		targetSignature = make([]sync.BlockHash, 0)
		targetContainer = &tlc.Container{}
	} else {
		comm.Logf("Latest build for channel %s is %d, downloading its signature", channel, parentID)
		buildFiles, err := client.ListBuildFiles(parentID)
		if err != nil {
			return err
		}

		var signatureFileID int64 = 0
		for _, f := range buildFiles.Files {
			if f.Type == itchio.BuildFileType_SIGNATURE {
				signatureFileID = f.ID
				break
			}
		}

		if signatureFileID == 0 {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		signatureReader, err := client.DownloadBuildFile(parentID, signatureFileID)

		targetContainer, targetSignature, err = pwr.ReadSignature(signatureReader)
		if err != nil {
			return err
		}
	}

	newPatchRes, err := client.CreateBuildFile(buildID, itchio.BuildFileType_PATCH)
	if err != nil {
		return err
	}

	uploadPatchReq, patchWriter, err := newMultipartUpload(newPatchRes.UploadURL, fmt.Sprintf("%d-%d.pwr", parentID, buildID))
	if err != nil {
		return err
	}

	newSignatureRes, err := client.CreateBuildFile(buildID, itchio.BuildFileType_SIGNATURE)
	if err != nil {
		return err
	}

	uploadSignatureReq, signatureWriter, err := newMultipartUpload(newSignatureRes.UploadURL, fmt.Sprintf("%d-%d.pwr", parentID, buildID))
	if err != nil {
		return err
	}

	done := make(chan bool)
	errs := make(chan error)

	go doReq(uploadPatchReq, done, errs)
	go doReq(uploadSignatureReq, done, errs)

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	go func() {
		defer patchWriter.Close()
		defer signatureWriter.Close()

		// we started walking the source container in the beginning,
		// we actually need it now.
		// note that we could actually start diffing before all the file
		// creation & upload setup is done

		var sourceContainer *tlc.Container

		select {
		case err := <-walkErrs:
			errs <- err
			return
		case sourceContainer = <-sourceContainerChan:
			break
		}

		dctx := &pwr.DiffContext{
			Compression: &pwr.CompressionSettings{
				Algorithm: pwr.CompressionAlgorithm_BROTLI,
				Quality:   1,
			},

			SourceContainer: sourceContainer,
			SourcePath:      buildPath,

			TargetContainer: targetContainer,
			TargetSignature: targetSignature,

			Consumer: comm.NewStateConsumer(),
		}

		comm.Logf("Sending patch and signature computed on-the-fly...")
		comm.StartProgress()
		err := dctx.WritePatch(patchCounter, signatureCounter)
		comm.EndProgress()

		if err != nil {
			errs <- err
			return
		}

		{
			prettyPatchSize := humanize.Bytes(uint64(patchCounter.Count()))
			percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
			relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
			prettyFreshSize := humanize.Bytes(uint64(dctx.FreshBytes))

			comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
			comm.Statf("%s patch (%.2f%% of the full size)", prettyPatchSize, relToNew)
		}

		_, err = client.FinalizeBuildFile(buildID, newPatchRes.ID, patchCounter.Count())
		if err != nil {
			errs <- err
			return
		}

		_, err = client.FinalizeBuildFile(buildID, newSignatureRes.ID, signatureCounter.Count())
		if err != nil {
			errs <- err
			return
		}

		done <- true
	}()

	// wait for all goroutines to complete
	for c := 0; c < 3; c++ {
		select {
		case err := <-errs:
			return err
		case <-done:
		}
	}
	comm.Logf("Everything sent off!")

	return nil
}

func doWalk(path string, result chan *tlc.Container, errs chan error) {
	// TODO: allow custom filter function
	container, err := tlc.Walk(path, nil)
	if err != nil {
		errs <- err
	}

	result <- container
}

func parseSpec(spec string) (string, string, error) {
	tokens := strings.Split(spec, ":")

	if len(tokens) == 1 {
		return "", "", fmt.Errorf("invalid spec: %s, missing channel (examples: %s:windows-32-beta, %s:linux-64)", spec, spec, spec)
	} else if len(tokens) != 2 {
		return "", "", fmt.Errorf("invalid spec: %s, expected something of the form user/page:channel", spec)
	}

	return tokens[0], tokens[1], nil
}

func parseAddress(address string) string {
	if strings.Contains(address, ":") {
		return address
	} else {
		return fmt.Sprintf("%s:%d", address, defaultPort)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"errors"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"time"

	quic "github.com/phuslu/quic-go"
	"github.com/phuslu/quic-go/h2quic"
)

var errNoSuchBucket = []byte("<?xml version='1.0' encoding='UTF-8'?><Error>NoSuchBucket<Message>The specified bucket does not exist.</Message></Error>")

func testQuic(ip string, config *ScanConfig, record *ScanRecord) bool {
	addr := net.JoinHostPort(ip, "443")

	start := time.Now()
	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return false
	}
	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return false
	}
	udpConn.SetDeadline(time.Now().Add(config.ScanMaxRTT))
	defer udpConn.Close()

	quicCfg := &quic.Config{
		HandshakeTimeout: config.HandshakeTimeout,
		KeepAlive:        false,
	}

	var serverName string
	if len(config.ServerName) == 0 {
		serverName = randomHost()
	} else {
		serverName = config.ServerName[rand.Intn(len(config.ServerName))]
	}

	tlsCfg := &tls.Config{
		InsecureSkipVerify: true,
		ServerName:         serverName,
	}
	quicSessn, err := quic.Dial(udpConn, udpAddr, addr, tlsCfg, quicCfg)
	if err != nil {
		return false
	}
	defer quicSessn.Close(nil)

	// lv1 只会验证证书是否存在
	cs := quicSessn.ConnectionState()
	if cs == nil {
		return false
	}
	pcs := cs.PeerCertificates
	if pcs == nil && len(pcs) < 2 {
		return false
	}

	// lv2 验证证书是否正确
	if config.Level > 1 {
		pkp := pcs[1].RawSubjectPublicKeyInfo
		if !bytes.Equal(g2pkp, pkp) && !bytes.Equal(g3pkp, pkp) { // && !bytes.Equal(g3ecc, pkp[:]) {
			return false
		}
	}

	// lv3 使用 http 访问来验证
	if config.Level > 2 {
		tr := &h2quic.RoundTripper{DisableCompression: true}
		defer tr.Close()

		tr.DialAddr = func(hostname string, tlsConfig *tls.Config, config *quic.Config) (quic.Session, error) {
			return quicSessn, err
		}
		hclient := &http.Client{
			Transport: tr,
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				return errors.New("fuck redirect")
			},
		}
		url := "https://" + config.HTTPVerifyHosts[rand.Intn(len(config.HTTPVerifyHosts))]
		req, _ := http.NewRequest(http.MethodGet, url, nil)
		req.Close = true
		resp, _ := hclient.Do(req)
		if resp == nil || (resp.StatusCode < 200 || resp.StatusCode >= 400) || !strings.Contains(resp.Header.Get("Alt-Svc"), `quic=":443"`) {
			return false
		}
		if resp.Body != nil {
			defer resp.Body.Close()
			// lv4 验证是否是 NoSuchBucket 错误
			if config.Level > 3 && resp.Header.Get("Content-Type") == "application/xml; charset=UTF-8" { // 也许条件改为 || 更好
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil || bytes.Equal(body, errNoSuchBucket) {
					return false
				}
			} else {
				io.Copy(ioutil.Discard, resp.Body)
			}
		}
	}

	if rtt := time.Since(start); rtt > config.ScanMinRTT {
		record.RTT += rtt
		return true
	}
	return false
}
<commit_msg>修复一处QUIC证书判断问题<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"errors"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"time"

	quic "github.com/phuslu/quic-go"
	"github.com/phuslu/quic-go/h2quic"
)

var errNoSuchBucket = []byte("<?xml version='1.0' encoding='UTF-8'?><Error>NoSuchBucket<Message>The specified bucket does not exist.</Message></Error>")

func testQuic(ip string, config *ScanConfig, record *ScanRecord) bool {
	addr := net.JoinHostPort(ip, "443")

	start := time.Now()
	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return false
	}
	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return false
	}
	udpConn.SetDeadline(time.Now().Add(config.ScanMaxRTT))
	defer udpConn.Close()

	quicCfg := &quic.Config{
		HandshakeTimeout: config.HandshakeTimeout,
		KeepAlive:        false,
	}

	var serverName string
	if len(config.ServerName) == 0 {
		serverName = randomHost()
	} else {
		serverName = config.ServerName[rand.Intn(len(config.ServerName))]
	}

	tlsCfg := &tls.Config{
		InsecureSkipVerify: true,
		ServerName:         serverName,
	}
	quicSessn, err := quic.Dial(udpConn, udpAddr, addr, tlsCfg, quicCfg)
	if err != nil {
		return false
	}
	defer quicSessn.Close(nil)

	// lv1 只会验证证书是否存在
	cs := quicSessn.ConnectionState()
	if cs == nil {
		return false
	}
	pcs := cs.PeerCertificates
	if len(pcs) < 2 {
		return false
	}

	// lv2 验证证书是否正确
	if config.Level > 1 {
		pkp := pcs[1].RawSubjectPublicKeyInfo
		if !bytes.Equal(g2pkp, pkp) && !bytes.Equal(g3pkp, pkp) { // && !bytes.Equal(g3ecc, pkp[:]) {
			return false
		}
	}

	// lv3 使用 http 访问来验证
	if config.Level > 2 {
		tr := &h2quic.RoundTripper{DisableCompression: true}
		defer tr.Close()

		tr.DialAddr = func(hostname string, tlsConfig *tls.Config, config *quic.Config) (quic.Session, error) {
			return quicSessn, err
		}
		hclient := &http.Client{
			Transport: tr,
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				return errors.New("fuck redirect")
			},
		}
		url := "https://" + config.HTTPVerifyHosts[rand.Intn(len(config.HTTPVerifyHosts))]
		req, _ := http.NewRequest(http.MethodGet, url, nil)
		req.Close = true
		resp, _ := hclient.Do(req)
		if resp == nil || (resp.StatusCode < 200 || resp.StatusCode >= 400) || !strings.Contains(resp.Header.Get("Alt-Svc"), `quic=":443"`) {
			return false
		}
		if resp.Body != nil {
			defer resp.Body.Close()
			// lv4 验证是否是 NoSuchBucket 错误
			if config.Level > 3 && resp.Header.Get("Content-Type") == "application/xml; charset=UTF-8" { // 也许条件改为 || 更好
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil || bytes.Equal(body, errNoSuchBucket) {
					return false
				}
			} else {
				io.Copy(ioutil.Discard, resp.Body)
			}
		}
	}

	if rtt := time.Since(start); rtt > config.ScanMinRTT {
		record.RTT += rtt
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package terminfo

import (
	"errors"
	"io"
	"os"
	"strings"
	"sync"

	"github.com/nhooyr/terminfo/caps"
)

var (
	ErrSmallFile  = errors.New("terminfo: file too small")
	ErrBadString  = errors.New("terminfo: bad string")
	ErrBigSection = errors.New("terminfo: section too big")
	ErrBadHeader  = errors.New("terminfo: bad header")
)

// header represents a Terminfo file's header.
// It is only 5 shorts because we no don't need to store magic.
type header [5]int16

// The magic number of terminfo files.
const magic = 0x11a

// What each short means in the standard format.
const (
	lenNames   = iota // bytes
	lenBools          // bytes
	lenNumbers        // shorts
	lenStrings        // shorts
	lenTable          // bytes
)

// What each short means in the extended format.
// lenTable is the same in both so it was not repeated here.
const (
	lenExtBools   = iota // bytes
	lenExtNumbers        // shorts
	lenExtStrings        // shorts
	lenExtOff            // shorts
)

// lenCaps returns the length of all of the capabilies in bytes.
func (h header) lenCaps() int16 {
	return h[lenNames] +
		h[lenBools] +
		(h[lenNames]+h[lenBools])%2 +
		h[lenNumbers]*2 +
		h[lenStrings]*2 +
		h[lenTable]
}

// lenExtCaps returns the length of all the extended capabilities in bytes.
func (h header) lenExtCaps() int16 {
	return h[lenExtBools] +
		h[lenExtBools]%2 +
		h[lenExtNumbers]*2 +
		h[lenExtOff]*2 +
		h[lenTable]
}

// lenBytes returns the length of the header in bytes.
func (h header) lenBytes() int16 {
	return int16(len(h) * 2)
}

// littleEndian decodes a short starting at i in buf using little-endian byte order.
func littleEndian(i int16, buf []byte) int16 {
	return int16(buf[i+1])<<8 | int16(buf[i])
}

type reader struct {
	pos            int16
	extNameOffPos  int16 // position in the name offsets
	h              header
	buf            []byte
	extStringTable []byte
	extNameTable   []byte
	ti             *Terminfo
}

var readerPool = sync.Pool{
	New: func() interface{} {
		r := new(reader)
		// TODO: What is the max entry size talking about in terminfo(5)?
		r.buf = make([]byte, 4096)
		return r
	},
}

// sliceNext slices the next off bytes of r.buf.
// It also increments r.pos by off.
func (r *reader) sliceNext(off int16) []byte {
	// Just use off as ppos.
	off, r.pos = r.pos, r.pos+off
	return r.buf[off:r.pos]
}

// evenBoundary checks if we are on an uneven word boundary.
// If so, it will skip the next byte, which should be a null.
func (r *reader) evenBoundary() {
	if r.pos%2 == 1 {
		r.pos++
	}
}

// indexNull returns the position of the next null byte in buf.
// It is used to find the end of null terminated strings.
func indexNull(off int16, buf []byte) int16 {
	for ; buf[off] != 0; off++ {
		if off >= int16(len(buf)) {
			return -1
		}
	}
	return off
}

func (r *reader) read(f *os.File) error {
	fi, err := f.Stat()
	if err != nil {
		return err
	}
	s, hl := int16(fi.Size()), r.h.lenBytes()
	if s < hl+2 { // add 2 for the magic
		return ErrSmallFile
	}
	if s > int16(cap(r.buf)) {
		r.buf = make([]byte, s, s*2+1)
	} else if s > int16(len(r.buf)) {
		r.buf = r.buf[:s]
	}
	if _, err = io.ReadAtLeast(f, r.buf, int(s)); err != nil {
		return err
	}
	// Check magic.
	if littleEndian(0, r.buf) != magic {
		return ErrBadHeader
	}
	r.pos = 2 // Skip magic.
	if err = r.readHeader(); err != nil {
		return err
	}
	if s-r.pos < r.h.lenCaps() {
		return ErrSmallFile
	}
	r.ti = new(Terminfo)
	r.ti.Names = strings.Split(string(r.sliceNext(r.h[lenNames])), "|")
	r.readBools()
	r.evenBoundary()
	r.readNumbers()
	if err = r.readStrings(); err != nil || s <= r.pos {
		return err
	}
	// We have extended capabilities.
	r.evenBoundary()
	if s -= r.pos; s < hl {
		return ErrSmallFile
	}
	if err = r.readHeader(); err != nil {
		return err
	}
	if r.h[lenExtBools]+r.h[lenExtNumbers]+r.h[lenExtStrings]*2 != r.h[lenExtOff] {
		return ErrBadHeader
	}
	if s-hl < r.h.lenExtCaps() {
		return ErrSmallFile
	}
	if err = r.setExtNameTable(); err != nil {
		return err
	}
	if err = r.readExtBools(); err != nil {
		return err
	}
	r.evenBoundary()
	if err = r.readExtNumbers(); err != nil {
		return err
	}
	return r.readExtStrings()
}

func (r *reader) readHeader() error {
	hbuf := r.sliceNext(r.h.lenBytes())
	for i := 0; i < len(r.h); i++ {
		n := littleEndian(int16(i*2), hbuf)
		if n < 0 {
			return ErrBadHeader
		}
		r.h[i] = n
	}
	return nil
}

func (r *reader) readBools() {
	if r.h[lenBools] >= caps.BoolCount {
		r.h[lenBools] = caps.BoolCount
	}
	for i, b := range r.sliceNext(r.h[lenBools]) {
		if b == 1 {
			r.ti.Bools[i] = true
		}
	}
}

func (r *reader) readNumbers() {
	if r.h[lenNumbers] >= caps.NumberCount {
		r.h[lenNumbers] = caps.NumberCount
	}
	nbuf := r.sliceNext(r.h[lenNumbers] * 2)
	for i := int16(0); i < r.h[lenNumbers]; i++ {
		if n := littleEndian(i*2, nbuf); n > -1 {
			r.ti.Numbers[i] = n
		}
	}
}

// readStrings reads the string and string table sections.
func (r *reader) readStrings() error {
	if r.h[lenStrings] >= caps.StringCount {
		r.h[lenStrings] = caps.StringCount
	}
	sbuf := r.sliceNext(r.h[lenStrings] * 2)
	table := r.sliceNext(r.h[lenTable])
	for i := int16(0); i < r.h[lenStrings]; i++ {
		if off := littleEndian(i*2, sbuf); off > -1 {
			end := indexNull(off, table)
			if end == -1 {
				return ErrBadString
			}
			r.ti.Strings[i] = string(table[off:end])
		}
	}
	return nil
}

func (r *reader) setExtNameTable() error {
	// This works because
	// r.h[lenExtOff] == r.h[lenExtBools]+r.h[lenExtNumbers]+r.h[lenExtStrings]*2.
	// See the check in r.read.
	r.extNameOffPos = r.pos +
		r.h[lenExtBools]%2 +
		r.h[lenExtNumbers] +
		r.h[lenExtOff]
	lenNameOffs := (r.h[lenExtOff] - r.h[lenExtStrings]) * 2
	// Find last string offset.
	vpos, voff := r.extNameOffPos, int16(0)
	for {
		vpos -= 2
		if vpos < r.pos {
			return ErrBadString
		}
		r.h[lenExtStrings]--
		if voff = littleEndian(vpos, r.buf); voff > -1 {
			break
		}
	}
	// Read the capability value.
	r.extStringTable = r.buf[r.extNameOffPos+lenNameOffs:]
	vend := indexNull(voff, r.extStringTable)
	if vend == -1 {
		return ErrBadString
	}
	// The rest is the name table
	r.extNameTable = r.extStringTable[vend+1:]
	// Find the capability's key in the name table.
	koff := littleEndian(vpos+lenNameOffs, r.buf)
	kend := indexNull(koff, r.extNameTable)
	if kend == -1 {
		return ErrBadString
	}
	r.ti.ExtStrings = make(map[string]string)
	r.ti.ExtStrings[string(r.extNameTable[koff:kend])] = string(r.extStringTable[voff:vend])
	// Truncate the string table to only name values.
	r.extStringTable = r.extStringTable[:voff]
	return nil
}

func (r *reader) nextExtName() (string, error) {
	off := littleEndian(r.extNameOffPos, r.buf)
	end := indexNull(off, r.extNameTable)
	if end == -1 {
		return "", ErrBadString
	}
	r.extNameOffPos += 2
	return string(r.extNameTable[off:end]), nil
}

func (r *reader) readExtBools() error {
	r.ti.ExtBools = make(map[string]bool)
	for _, b := range r.sliceNext(r.h[lenExtBools]) {
		if b == 1 {
			key, err := r.nextExtName()
			if err != nil {
				return err
			}
			r.ti.ExtBools[key] = true
		}
	}
	return nil
}

func (r *reader) readExtNumbers() error {
	r.ti.ExtNumbers = make(map[string]int16)
	nbuf := r.sliceNext(r.h[lenExtNumbers] * 2)
	for i := int16(0); i < r.h[lenExtNumbers]; i++ {
		if n := littleEndian(i*2, nbuf); n > -1 {
			key, err := r.nextExtName()
			if err != nil {
				return err
			}
			r.ti.ExtNumbers[key] = n
		}
	}
	return nil
}

func (r *reader) readExtStrings() error {
	for lpos := r.pos + r.h[lenExtStrings]*2; r.pos < lpos; r.pos += 2 {
		if off := littleEndian(r.pos, r.buf); off > -1 {
			end := indexNull(off, r.extStringTable)
			if end == -1 {
				return ErrBadString
			}
			key, err := r.nextExtName()
			if err != nil {
				return err
			}
			r.ti.ExtStrings[key] = string(r.extStringTable[off:end])
		}
	}
	return nil
}
<commit_msg>todo always<commit_after>package terminfo

import (
	"errors"
	"io"
	"os"
	"strings"
	"sync"

	"github.com/nhooyr/terminfo/caps"
)

var (
	ErrSmallFile  = errors.New("terminfo: file too small")
	ErrBadString  = errors.New("terminfo: bad string")
	ErrBigSection = errors.New("terminfo: section too big")
	ErrBadHeader  = errors.New("terminfo: bad header")
)

// header represents a Terminfo file's header.
// It is only 5 shorts because we no don't need to store magic.
type header [5]int16

// The magic number of terminfo files.
const magic = 0x11a

// What each short means in the standard format.
const (
	lenNames   = iota // bytes
	lenBools          // bytes
	lenNumbers        // shorts
	lenStrings        // shorts
	lenTable          // bytes
)

// What each short means in the extended format.
// lenTable is the same in both so it was not repeated here.
const (
	lenExtBools   = iota // bytes
	lenExtNumbers        // shorts
	lenExtStrings        // shorts
	lenExtOff            // shorts
)

// lenCaps returns the length of all of the capabilies in bytes.
func (h header) lenCaps() int16 {
	return h[lenNames] +
		h[lenBools] +
		(h[lenNames]+h[lenBools])%2 +
		h[lenNumbers]*2 +
		h[lenStrings]*2 +
		h[lenTable]
}

// lenExtCaps returns the length of all the extended capabilities in bytes.
func (h header) lenExtCaps() int16 {
	return h[lenExtBools] +
		h[lenExtBools]%2 +
		h[lenExtNumbers]*2 +
		h[lenExtOff]*2 +
		h[lenTable]
}

// lenBytes returns the length of the header in bytes.
func (h header) lenBytes() int16 {
	return int16(len(h) * 2)
}

// littleEndian decodes a short starting at i in buf using little-endian byte order.
func littleEndian(i int16, buf []byte) int16 {
	return int16(buf[i+1])<<8 | int16(buf[i])
}

type reader struct {
	pos            int16
	extNameOffPos  int16 // position in the name offsets
	h              header
	buf            []byte
	extStringTable []byte
	extNameTable   []byte
	ti             *Terminfo
}

var readerPool = sync.Pool{
	New: func() interface{} {
		r := new(reader)
		// TODO: What is the max entry size talking about in terminfo(5)?
		r.buf = make([]byte, 4096)
		return r
	},
}

// sliceNext slices the next off bytes of r.buf.
// It also increments r.pos by off.
func (r *reader) sliceNext(off int16) []byte {
	// Just use off as ppos.
	off, r.pos = r.pos, r.pos+off
	return r.buf[off:r.pos]
}

// evenBoundary checks if we are on an uneven word boundary.
// If so, it will skip the next byte, which should be a null.
func (r *reader) evenBoundary() {
	if r.pos%2 == 1 {
		r.pos++
	}
}

// indexNull returns the position of the next null byte in buf.
// It is used to find the end of null terminated strings.
func indexNull(off int16, buf []byte) int16 {
	for ; buf[off] != 0; off++ {
		if off >= int16(len(buf)) {
			return -1
		}
	}
	return off
}

func (r *reader) read(f *os.File) error {
	fi, err := f.Stat()
	if err != nil {
		return err
	}
	s, hl := int16(fi.Size()), r.h.lenBytes()
	if s < hl+2 { // add 2 for the magic
		return ErrSmallFile
	}
	if s > int16(cap(r.buf)) {
		r.buf = make([]byte, s, s*2+1)
	} else if s > int16(len(r.buf)) {
		r.buf = r.buf[:s]
	}
	if _, err = io.ReadAtLeast(f, r.buf, int(s)); err != nil {
		return err
	}
	// Check magic.
	if littleEndian(0, r.buf) != magic {
		return ErrBadHeader
	}
	r.pos = 2 // Skip magic.
	if err = r.readHeader(); err != nil {
		return err
	}
	if s-r.pos < r.h.lenCaps() {
		return ErrSmallFile
	}
	r.ti = new(Terminfo)
	r.ti.Names = strings.Split(string(r.sliceNext(r.h[lenNames])), "|")
	r.readBools()
	r.evenBoundary()
	r.readNumbers()
	if err = r.readStrings(); err != nil || s <= r.pos {
		return err
	}
	// We have extended capabilities.
	r.evenBoundary()
	if s -= r.pos; s < hl {
		return ErrSmallFile
	}
	if err = r.readHeader(); err != nil {
		return err
	}
	if r.h[lenExtBools]+r.h[lenExtNumbers]+r.h[lenExtStrings]*2 != r.h[lenExtOff] {
		return ErrBadHeader
	}
	if s-hl < r.h.lenExtCaps() {
		return ErrSmallFile
	}
	if err = r.setExtNameTable(); err != nil {
		return err
	}
	if err = r.readExtBools(); err != nil {
		return err
	}
	r.evenBoundary()
	if err = r.readExtNumbers(); err != nil {
		return err
	}
	return r.readExtStrings()
}

func (r *reader) readHeader() error {
	hbuf := r.sliceNext(r.h.lenBytes())
	for i := 0; i < len(r.h); i++ {
		n := littleEndian(int16(i*2), hbuf)
		if n < 0 {
			return ErrBadHeader
		}
		r.h[i] = n
	}
	return nil
}

func (r *reader) readBools() {
	if r.h[lenBools] >= caps.BoolCount {
		r.h[lenBools] = caps.BoolCount
	}
	for i, b := range r.sliceNext(r.h[lenBools]) {
		if b == 1 {
			r.ti.Bools[i] = true
		}
	}
}

func (r *reader) readNumbers() {
	if r.h[lenNumbers] >= caps.NumberCount {
		r.h[lenNumbers] = caps.NumberCount
	}
	nbuf := r.sliceNext(r.h[lenNumbers] * 2)
	for i := int16(0); i < r.h[lenNumbers]; i++ {
		if n := littleEndian(i*2, nbuf); n > -1 {
			r.ti.Numbers[i] = n
		}
	}
}

// readStrings reads the string and string table sections.
func (r *reader) readStrings() error {
	if r.h[lenStrings] >= caps.StringCount {
		r.h[lenStrings] = caps.StringCount
	}
	sbuf := r.sliceNext(r.h[lenStrings] * 2)
	table := r.sliceNext(r.h[lenTable])
	for i := int16(0); i < r.h[lenStrings]; i++ {
		if off := littleEndian(i*2, sbuf); off > -1 {
			end := indexNull(off, table)
			if end == -1 {
				return ErrBadString
			}
			r.ti.Strings[i] = string(table[off:end])
		}
	}
	return nil
}

func (r *reader) setExtNameTable() error {
	// This works because
	// r.h[lenExtOff] == r.h[lenExtBools]+r.h[lenExtNumbers]+r.h[lenExtStrings]*2.
	// See the check in r.read.
	r.extNameOffPos = r.pos +
		r.h[lenExtBools]%2 +
		r.h[lenExtNumbers] +
		r.h[lenExtOff]
	lenNameOffs := (r.h[lenExtOff] - r.h[lenExtStrings]) * 2
	// Find last string offset.
	vpos, voff := r.extNameOffPos, int16(0)
	for {
		vpos -= 2
		if vpos < r.pos {
			return ErrBadString
		}
		r.h[lenExtStrings]--
		if voff = littleEndian(vpos, r.buf); voff > -1 {
			break
		}
	}
	// Read the capability value.
	r.extStringTable = r.buf[r.extNameOffPos+lenNameOffs:]
	vend := indexNull(voff, r.extStringTable)
	if vend == -1 {
		return ErrBadString
	}
	// The rest is the name table
	r.extNameTable = r.extStringTable[vend+1:]
	// Find the capability's key in the name table.
	koff := littleEndian(vpos+lenNameOffs, r.buf)
	kend := indexNull(koff, r.extNameTable)
	if kend == -1 {
		return ErrBadString
	}
	r.ti.ExtStrings = make(map[string]string)
	r.ti.ExtStrings[string(r.extNameTable[koff:kend])] = string(r.extStringTable[voff:vend])
	// Truncate the string table to only name values.
	r.extStringTable = r.extStringTable[:voff]
	return nil
}

// TODO should this always be called?
func (r *reader) nextExtName() (string, error) {
	off := littleEndian(r.extNameOffPos, r.buf)
	end := indexNull(off, r.extNameTable)
	if end == -1 {
		return "", ErrBadString
	}
	r.extNameOffPos += 2
	return string(r.extNameTable[off:end]), nil
}

func (r *reader) readExtBools() error {
	r.ti.ExtBools = make(map[string]bool)
	for _, b := range r.sliceNext(r.h[lenExtBools]) {
		if b == 1 {
			key, err := r.nextExtName()
			if err != nil {
				return err
			}
			r.ti.ExtBools[key] = true
		}
	}
	return nil
}

func (r *reader) readExtNumbers() error {
	r.ti.ExtNumbers = make(map[string]int16)
	nbuf := r.sliceNext(r.h[lenExtNumbers] * 2)
	for i := int16(0); i < r.h[lenExtNumbers]; i++ {
		if n := littleEndian(i*2, nbuf); n > -1 {
			key, err := r.nextExtName()
			if err != nil {
				return err
			}
			r.ti.ExtNumbers[key] = n
		}
	}
	return nil
}

func (r *reader) readExtStrings() error {
	for lpos := r.pos + r.h[lenExtStrings]*2; r.pos < lpos; r.pos += 2 {
		if off := littleEndian(r.pos, r.buf); off > -1 {
			end := indexNull(off, r.extStringTable)
			if end == -1 {
				return ErrBadString
			}
			key, err := r.nextExtName()
			if err != nil {
				return err
			}
			r.ti.ExtStrings[key] = string(r.extStringTable[off:end])
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2015 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/bemasher/rtlamr/idm"
	"github.com/bemasher/rtlamr/parse"
	"github.com/bemasher/rtlamr/r900"
	"github.com/bemasher/rtlamr/scm"
	"github.com/bemasher/rtltcp"
)

var rcvr Receiver

type Receiver struct {
	rtltcp.SDR
	p  parse.Parser
	fc parse.FilterChain
}

func (rcvr *Receiver) NewReceiver() {
	switch strings.ToLower(*msgType) {
	case "scm":
		rcvr.p = scm.NewParser(*symbolLength, *decimation)
	case "idm":
		rcvr.p = idm.NewParser(*symbolLength, *decimation)
	case "r900":
		rcvr.p = r900.NewParser(*symbolLength, *decimation)
	default:
		log.Fatalf("Invalid message type: %q\n", *msgType)
	}

	if !*quiet {
		rcvr.p.Log()
	}

	// Connect to rtl_tcp server.
	if err := rcvr.Connect(nil); err != nil {
		log.Fatal(err)
	}

	rcvr.HandleFlags()

	// Tell the user how many gain settings were reported by rtl_tcp.
	if !*quiet {
		log.Println("GainCount:", rcvr.SDR.Info.GainCount)
	}

	centerfreqFlagSet := false
	sampleRateFlagSet := false
	gainFlagSet := false
	flag.Visit(func(f *flag.Flag) {
		switch f.Name {
		case "centerfreq":
			centerfreqFlagSet = true
		case "samplerate":
			sampleRateFlagSet = true
		case "gainbyindex", "tunergainmode", "tunergain", "agcmode":
			gainFlagSet = true
		case "unique":
			rcvr.fc.Add(NewUniqueFilter())
		case "filterid":
			rcvr.fc.Add(meterID)
		case "filtertype":
			rcvr.fc.Add(meterType)
		default:
			fmt.Println(f.Name)
		}
	})
	fmt.Printf("%+v\n", rcvr.fc)

	// Set some parameters for listening.
	if centerfreqFlagSet {
		rcvr.SetCenterFreq(uint32(rcvr.Flags.CenterFreq))
	} else {
		rcvr.SetCenterFreq(rcvr.p.Cfg().CenterFreq)
	}

	if !sampleRateFlagSet {
		rcvr.SetSampleRate(uint32(rcvr.p.Cfg().SampleRate))
	}
	if !gainFlagSet {
		rcvr.SetGainMode(true)
	}

	return
}

func (rcvr *Receiver) Run() {
	// Setup signal channel for interruption.
	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint, os.Kill, os.Interrupt)

	// Setup time limit channel
	tLimit := make(<-chan time.Time, 1)
	if *timeLimit != 0 {
		tLimit = time.After(*timeLimit)
	}

	in, out := io.Pipe()

	go func() {
		tcpBlock := make([]byte, 16384)
		for {
			n, err := rcvr.Read(tcpBlock)
			if err != nil {
				return
			}
			out.Write(tcpBlock[:n])
		}
	}()

	block := make([]byte, rcvr.p.Cfg().BlockSize2)

	start := time.Now()
	for {
		// Exit on interrupt or time limit, otherwise receive.
		select {
		case <-sigint:
			return
		case <-tLimit:
			fmt.Println("Time Limit Reached:", time.Since(start))
			return
		default:
			// Read new sample block.
			_, err := io.ReadFull(in, block)
			if err != nil {
				log.Fatal("Error reading samples: ", err)
			}

			pktFound := false
			indices := rcvr.p.Dec().Decode(block)

			for _, pkt := range rcvr.p.Parse(indices) {
				if !rcvr.fc.Match(pkt) {
					continue
				}

				var msg parse.LogMessage
				msg.Time = time.Now()
				msg.Offset, _ = sampleFile.Seek(0, os.SEEK_CUR)
				msg.Length = rcvr.p.Cfg().BufferLength << 1
				msg.Message = pkt

				err = encoder.Encode(msg)
				if err != nil {
					log.Fatal("Error encoding message: ", err)
				}

				// The XML encoder doesn't write new lines after each
				// element, add them.
				if _, ok := encoder.(*xml.Encoder); ok {
					fmt.Fprintln(logFile)
				}

				pktFound = true
				if *single {
					break
				}
			}

			if pktFound {
				if *sampleFilename != os.DevNull {
					_, err = sampleFile.Write(rcvr.p.Dec().IQ)
					if err != nil {
						log.Fatal("Error writing raw samples to file:", err)
					}
				}
				if *single {
					return
				}
			}
		}
	}
}

func init() {
	log.SetFlags(log.Lshortfile | log.Lmicroseconds)
}

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to this file")

func main() {
	rcvr.RegisterFlags()
	RegisterFlags()

	flag.Parse()
	HandleFlags()

	rcvr.NewReceiver()

	defer logFile.Close()
	defer sampleFile.Close()
	defer rcvr.Close()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	rcvr.Run()
}
<commit_msg>Remove filter chain debug print.<commit_after>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2015 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/bemasher/rtlamr/idm"
	"github.com/bemasher/rtlamr/parse"
	"github.com/bemasher/rtlamr/r900"
	"github.com/bemasher/rtlamr/scm"
	"github.com/bemasher/rtltcp"
)

var rcvr Receiver

type Receiver struct {
	rtltcp.SDR
	p  parse.Parser
	fc parse.FilterChain
}

func (rcvr *Receiver) NewReceiver() {
	switch strings.ToLower(*msgType) {
	case "scm":
		rcvr.p = scm.NewParser(*symbolLength, *decimation)
	case "idm":
		rcvr.p = idm.NewParser(*symbolLength, *decimation)
	case "r900":
		rcvr.p = r900.NewParser(*symbolLength, *decimation)
	default:
		log.Fatalf("Invalid message type: %q\n", *msgType)
	}

	if !*quiet {
		rcvr.p.Log()
	}

	// Connect to rtl_tcp server.
	if err := rcvr.Connect(nil); err != nil {
		log.Fatal(err)
	}

	rcvr.HandleFlags()

	// Tell the user how many gain settings were reported by rtl_tcp.
	if !*quiet {
		log.Println("GainCount:", rcvr.SDR.Info.GainCount)
	}

	centerfreqFlagSet := false
	sampleRateFlagSet := false
	gainFlagSet := false
	flag.Visit(func(f *flag.Flag) {
		switch f.Name {
		case "centerfreq":
			centerfreqFlagSet = true
		case "samplerate":
			sampleRateFlagSet = true
		case "gainbyindex", "tunergainmode", "tunergain", "agcmode":
			gainFlagSet = true
		case "unique":
			rcvr.fc.Add(NewUniqueFilter())
		case "filterid":
			rcvr.fc.Add(meterID)
		case "filtertype":
			rcvr.fc.Add(meterType)
		default:
			fmt.Println(f.Name)
		}
	})

	// Set some parameters for listening.
	if centerfreqFlagSet {
		rcvr.SetCenterFreq(uint32(rcvr.Flags.CenterFreq))
	} else {
		rcvr.SetCenterFreq(rcvr.p.Cfg().CenterFreq)
	}

	if !sampleRateFlagSet {
		rcvr.SetSampleRate(uint32(rcvr.p.Cfg().SampleRate))
	}
	if !gainFlagSet {
		rcvr.SetGainMode(true)
	}

	return
}

func (rcvr *Receiver) Run() {
	// Setup signal channel for interruption.
	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint, os.Kill, os.Interrupt)

	// Setup time limit channel
	tLimit := make(<-chan time.Time, 1)
	if *timeLimit != 0 {
		tLimit = time.After(*timeLimit)
	}

	in, out := io.Pipe()

	go func() {
		tcpBlock := make([]byte, 16384)
		for {
			n, err := rcvr.Read(tcpBlock)
			if err != nil {
				return
			}
			out.Write(tcpBlock[:n])
		}
	}()

	block := make([]byte, rcvr.p.Cfg().BlockSize2)

	start := time.Now()
	for {
		// Exit on interrupt or time limit, otherwise receive.
		select {
		case <-sigint:
			return
		case <-tLimit:
			fmt.Println("Time Limit Reached:", time.Since(start))
			return
		default:
			// Read new sample block.
			_, err := io.ReadFull(in, block)
			if err != nil {
				log.Fatal("Error reading samples: ", err)
			}

			pktFound := false
			indices := rcvr.p.Dec().Decode(block)

			for _, pkt := range rcvr.p.Parse(indices) {
				if !rcvr.fc.Match(pkt) {
					continue
				}

				var msg parse.LogMessage
				msg.Time = time.Now()
				msg.Offset, _ = sampleFile.Seek(0, os.SEEK_CUR)
				msg.Length = rcvr.p.Cfg().BufferLength << 1
				msg.Message = pkt

				err = encoder.Encode(msg)
				if err != nil {
					log.Fatal("Error encoding message: ", err)
				}

				// The XML encoder doesn't write new lines after each
				// element, add them.
				if _, ok := encoder.(*xml.Encoder); ok {
					fmt.Fprintln(logFile)
				}

				pktFound = true
				if *single {
					break
				}
			}

			if pktFound {
				if *sampleFilename != os.DevNull {
					_, err = sampleFile.Write(rcvr.p.Dec().IQ)
					if err != nil {
						log.Fatal("Error writing raw samples to file:", err)
					}
				}
				if *single {
					return
				}
			}
		}
	}
}

func init() {
	log.SetFlags(log.Lshortfile | log.Lmicroseconds)
}

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to this file")

func main() {
	rcvr.RegisterFlags()
	RegisterFlags()

	flag.Parse()
	HandleFlags()

	rcvr.NewReceiver()

	defer logFile.Close()
	defer sampleFile.Close()
	defer rcvr.Close()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	rcvr.Run()
}
<|endoftext|>


<commit_before>//go:build !plan9
// +build !plan9

package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	. "github.com/candid82/joker/core"
	"github.com/candid82/liner"
)

var qualifiedSymbolRe *regexp.Regexp = regexp.MustCompile(`([0-9A-Za-z_\-\+\*\'\.]+)/([0-9A-Za-z_\-\+\*\']*$)`)
var callRe *regexp.Regexp = regexp.MustCompile(`\(\s*([0-9A-Za-z_\-\+\*\'\.]*$)`)

func completer(line string, pos int) (head string, c []string, tail string) {
	head = line[:pos]
	tail = line[pos:]
	var match []string
	var prefix string
	var ns *Namespace
	var addNamespaces bool
	if match = qualifiedSymbolRe.FindStringSubmatch(head); match != nil {
		nsName := match[1]
		prefix = match[2]
		ns = GLOBAL_ENV.NamespaceFor(GLOBAL_ENV.CurrentNamespace(), MakeSymbol(nsName+"/"+prefix))
	} else if match = callRe.FindStringSubmatch(head); match != nil {
		prefix = match[1]
		ns = GLOBAL_ENV.CurrentNamespace()
		addNamespaces = true
	}
	if ns == nil {
		return
	}
	for k, _ := range ns.Mappings() {
		if strings.HasPrefix(*k, prefix) {
			c = append(c, *k)
		}
	}
	if addNamespaces {
		for k, _ := range GLOBAL_ENV.Namespaces {
			if strings.HasPrefix(*k, prefix) {
				c = append(c, *k)
			}
		}
		for k, _ := range ns.Aliases() {
			if strings.HasPrefix(*k, prefix) {
				c = append(c, *k)
			}
		}
	}
	if len(c) > 0 {
		head = head[:len(head)-len(prefix)]
	}
	sort.Strings(c)
	return
}

func saveReplHistory(rl *liner.State, filename string) {
	if filename == "" {
		return
	}
	if f, err := os.Create(filename); err == nil {
		rl.WriteHistory(f)
		f.Close()
	}
}

func repl(phase Phase) {
	ProcessReplData()
	GLOBAL_ENV.FindNamespace(MakeSymbol("user")).ReferAll(GLOBAL_ENV.FindNamespace(MakeSymbol("joker.repl")))
	fmt.Printf("Welcome to joker %s. Use '(exit)', %s to exit.\n", VERSION, EXITERS)
	parseContext := &ParseContext{GlobalEnv: GLOBAL_ENV}
	replContext := NewReplContext(parseContext.GlobalEnv)

	var runeReader io.RuneReader
	var rl *liner.State
	var historyFilename string
	if noReadline {
		runeReader = bufio.NewReader(Stdin)
	} else {
		home := HomeDir()
		jokerd := filepath.Join(home, ".jokerd")
		if _, err := os.Stat(jokerd); os.IsNotExist(err) {
			if err := os.MkdirAll(jokerd, 0777); err != nil {
				fmt.Fprintf(Stderr, "WARNING: could not create %s \n", jokerd)
			}
		}
		if !noReplHistory {
			historyFilename = filepath.Join(jokerd, ".repl_history")
		}
		rl = liner.NewLiner()
		OnExit(func() {
			saveReplHistory(rl, historyFilename)
			rl.Close()
		})
		defer rl.Close()
		rl.SetCtrlCAborts(true)
		rl.SetWordCompleter(completer)
		rl.SetTabCompletionStyle(liner.TabPrints)

		if !noReplHistory {
			if f, err := os.Open(historyFilename); err == nil {
				rl.ReadHistory(f)
				f.Close()
			}
		}

		runeReader = NewLineRuneReader(rl)

		for _, line := range strings.Split(string(dataRead), "\n") {
			if strings.TrimSpace(line) != "" {
				rl.AppendHistory(line)
			}
		}
		dataRead = []rune{}
	}

	reader := NewReader(runeReader, "<repl>")

	for {
		if noReadline {
			print(GLOBAL_ENV.CurrentNamespace().Name.ToString(false) + "=> ")
		} else {
			runeReader.(*LineRuneReader).Prompt = (GLOBAL_ENV.CurrentNamespace().Name.ToString(false) + "=> ")
		}
		if processReplCommand(reader, phase, parseContext, replContext) {
			saveReplHistory(rl, historyFilename)
			return
		}
	}
}
<commit_msg>simplify repl<commit_after>//go:build !plan9
// +build !plan9

package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	. "github.com/candid82/joker/core"
	"github.com/candid82/liner"
)

var qualifiedSymbolRe *regexp.Regexp = regexp.MustCompile(`([0-9A-Za-z_\-\+\*\'\.]+)/([0-9A-Za-z_\-\+\*\']*$)`)
var callRe *regexp.Regexp = regexp.MustCompile(`\(\s*([0-9A-Za-z_\-\+\*\'\.]*$)`)

func completer(line string, pos int) (head string, c []string, tail string) {
	head = line[:pos]
	tail = line[pos:]
	var match []string
	var prefix string
	var ns *Namespace
	var addNamespaces bool
	if match = qualifiedSymbolRe.FindStringSubmatch(head); match != nil {
		nsName := match[1]
		prefix = match[2]
		ns = GLOBAL_ENV.NamespaceFor(GLOBAL_ENV.CurrentNamespace(), MakeSymbol(nsName+"/"+prefix))
	} else if match = callRe.FindStringSubmatch(head); match != nil {
		prefix = match[1]
		ns = GLOBAL_ENV.CurrentNamespace()
		addNamespaces = true
	}
	if ns == nil {
		return
	}
	for k, _ := range ns.Mappings() {
		if strings.HasPrefix(*k, prefix) {
			c = append(c, *k)
		}
	}
	if addNamespaces {
		for k, _ := range GLOBAL_ENV.Namespaces {
			if strings.HasPrefix(*k, prefix) {
				c = append(c, *k)
			}
		}
		for k, _ := range ns.Aliases() {
			if strings.HasPrefix(*k, prefix) {
				c = append(c, *k)
			}
		}
	}
	if len(c) > 0 {
		head = head[:len(head)-len(prefix)]
	}
	sort.Strings(c)
	return
}

func saveReplHistory(rl *liner.State, filename string) {
	if filename == "" {
		return
	}
	if f, err := os.Create(filename); err == nil {
		rl.WriteHistory(f)
		f.Close()
	}
}

func repl(phase Phase) {
	ProcessReplData()
	GLOBAL_ENV.FindNamespace(MakeSymbol("user")).ReferAll(GLOBAL_ENV.FindNamespace(MakeSymbol("joker.repl")))
	fmt.Printf("Welcome to joker %s. Use '(exit)', %s to exit.\n", VERSION, EXITERS)
	parseContext := &ParseContext{GlobalEnv: GLOBAL_ENV}
	replContext := NewReplContext(parseContext.GlobalEnv)

	var runeReader io.RuneReader
	var rl *liner.State
	var historyFilename string
	if noReadline {
		runeReader = bufio.NewReader(Stdin)
	} else {
		home := HomeDir()
		jokerd := filepath.Join(home, ".jokerd")
		if _, err := os.Stat(jokerd); os.IsNotExist(err) {
			if err := os.MkdirAll(jokerd, 0777); err != nil {
				fmt.Fprintf(Stderr, "WARNING: could not create %s \n", jokerd)
			}
		}
		if !noReplHistory {
			historyFilename = filepath.Join(jokerd, ".repl_history")
		}
		rl = liner.NewLiner()
		OnExit(func() {
			saveReplHistory(rl, historyFilename)
			rl.Close()
		})
		defer rl.Close()
		rl.SetCtrlCAborts(true)
		rl.SetWordCompleter(completer)
		rl.SetTabCompletionStyle(liner.TabPrints)

		if !noReplHistory {
			if f, err := os.Open(historyFilename); err == nil {
				rl.ReadHistory(f)
				f.Close()
			}
		}

		runeReader = NewLineRuneReader(rl)

		for _, line := range strings.Split(string(dataRead), "\n") {
			if strings.TrimSpace(line) != "" {
				rl.AppendHistory(line)
			}
		}
		dataRead = []rune{}
	}

	reader := NewReader(runeReader, "<repl>")

	for {
		namespace := GLOBAL_ENV.CurrentNamespace().Name.ToString(false)
		if noReadline {
			print(namespace + "=> ")
		} else {
			runeReader.(*LineRuneReader).Prompt = (namespace + "=> ")
		}
		if processReplCommand(reader, phase, parseContext, replContext) {
			saveReplHistory(rl, historyFilename)
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"bytes"
	"container/list"
	"errors"
	"os"
	"path"
	"path/filepath"
	"time"
)

// Repository represents a Git repository.
type Repository struct {
	Path string

	commitCache *ObjectCache
	tagCache    *ObjectCache
}

const prettyLogFormat = `--pretty=format:%H`

func (repo *Repository) parsePrettyFormatLogToList(logs []byte) (*list.List, error) {
	l := list.New()
	if len(logs) == 0 {
		return l, nil
	}

	parts := bytes.Split(logs, []byte{'\n'})

	for _, commitID := range parts {
		commit, err := repo.GetCommit(string(commitID))
		if err != nil {
			return nil, err
		}
		l.PushBack(commit)
	}

	return l, nil
}

// IsRepoURLAccessible checks if given repository URL is accessible.
func IsRepoURLAccessible(url string) bool {
	_, err := NewCommand("ls-remote", "-q", "-h", url, "HEAD").Run()
	if err != nil {
		return false
	}
	return true
}

// InitRepository initializes a new Git repository.
func InitRepository(repoPath string, bare bool) error {
	os.MkdirAll(repoPath, os.ModePerm)

	cmd := NewCommand("init")
	if bare {
		cmd.AddArguments("--bare")
	}
	_, err := cmd.RunInDir(repoPath)
	return err
}

// OpenRepository opens the repository at the given path.
func OpenRepository(repoPath string) (*Repository, error) {
	repoPath, err := filepath.Abs(repoPath)
	if err != nil {
		return nil, err
	} else if !isDir(repoPath) {
		return nil, errors.New("no such file or directory")
	}

	return &Repository{
		Path:        repoPath,
		commitCache: newObjectCache(),
		tagCache:    newObjectCache(),
	}, nil
}

// CloneRepoOptions options when clone a repository
type CloneRepoOptions struct {
	Timeout time.Duration
	Mirror  bool
	Bare    bool
	Quiet   bool
	Branch  string
}

// Clone clones original repository to target path.
func Clone(from, to string, opts CloneRepoOptions) (err error) {
	toDir := path.Dir(to)
	if err = os.MkdirAll(toDir, os.ModePerm); err != nil {
		return err
	}

	cmd := NewCommand("clone")
	if opts.Mirror {
		cmd.AddArguments("--mirror")
	}
	if opts.Bare {
		cmd.AddArguments("--bare")
	}
	if opts.Quiet {
		cmd.AddArguments("--quiet")
	}
	if len(opts.Branch) > 0 {
		cmd.AddArguments("-b", opts.Branch)
	}
	cmd.AddArguments(from, to)

	if opts.Timeout <= 0 {
		opts.Timeout = -1
	}

	_, err = cmd.RunTimeout(opts.Timeout)
	return err
}

// PullRemoteOptions options when pull from remote
type PullRemoteOptions struct {
	Timeout time.Duration
	All     bool
	Rebase  bool
	Remote  string
	Branch  string
}

// Pull pulls changes from remotes.
func Pull(repoPath string, opts PullRemoteOptions) error {
	cmd := NewCommand("pull")
	if opts.Rebase {
		cmd.AddArguments("--rebase")
	}
	if opts.All {
		cmd.AddArguments("--all")
	} else {
		cmd.AddArguments(opts.Remote)
		cmd.AddArguments(opts.Branch)
	}

	if opts.Timeout <= 0 {
		opts.Timeout = -1
	}

	_, err := cmd.RunInDirTimeout(opts.Timeout, repoPath)
	return err
}

// Push pushs local commits to given remote branch.
func Push(repoPath, remote, branch string) error {
	_, err := NewCommand("push", remote, branch).RunInDir(repoPath)
	return err
}

// CheckoutOptions options when heck out some branch
type CheckoutOptions struct {
	Timeout   time.Duration
	Branch    string
	OldBranch string
}

// Checkout checkouts a branch
func Checkout(repoPath string, opts CheckoutOptions) error {
	cmd := NewCommand("checkout")
	if len(opts.OldBranch) > 0 {
		cmd.AddArguments("-b")
	}

	if opts.Timeout <= 0 {
		opts.Timeout = -1
	}

	cmd.AddArguments(opts.Branch)

	if len(opts.OldBranch) > 0 {
		cmd.AddArguments(opts.OldBranch)
	}

	_, err := cmd.RunInDirTimeout(opts.Timeout, repoPath)
	return err
}

// ResetHEAD resets HEAD to given revision or head of branch.
func ResetHEAD(repoPath string, hard bool, revision string) error {
	cmd := NewCommand("reset")
	if hard {
		cmd.AddArguments("--hard")
	}
	_, err := cmd.AddArguments(revision).RunInDir(repoPath)
	return err
}

// MoveFile moves a file to another file or directory.
func MoveFile(repoPath, oldTreeName, newTreeName string) error {
	_, err := NewCommand("mv").AddArguments(oldTreeName, newTreeName).RunInDir(repoPath)
	return err
}
<commit_msg>Add GetRepoSize implemented using count-objects (#40)<commit_after>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"bytes"
	"container/list"
	"errors"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/Unknwon/com"
)

// Repository represents a Git repository.
type Repository struct {
	Path string

	commitCache *ObjectCache
	tagCache    *ObjectCache
}

const prettyLogFormat = `--pretty=format:%H`

func (repo *Repository) parsePrettyFormatLogToList(logs []byte) (*list.List, error) {
	l := list.New()
	if len(logs) == 0 {
		return l, nil
	}

	parts := bytes.Split(logs, []byte{'\n'})

	for _, commitID := range parts {
		commit, err := repo.GetCommit(string(commitID))
		if err != nil {
			return nil, err
		}
		l.PushBack(commit)
	}

	return l, nil
}

// IsRepoURLAccessible checks if given repository URL is accessible.
func IsRepoURLAccessible(url string) bool {
	_, err := NewCommand("ls-remote", "-q", "-h", url, "HEAD").Run()
	if err != nil {
		return false
	}
	return true
}

// InitRepository initializes a new Git repository.
func InitRepository(repoPath string, bare bool) error {
	os.MkdirAll(repoPath, os.ModePerm)

	cmd := NewCommand("init")
	if bare {
		cmd.AddArguments("--bare")
	}
	_, err := cmd.RunInDir(repoPath)
	return err
}

// OpenRepository opens the repository at the given path.
func OpenRepository(repoPath string) (*Repository, error) {
	repoPath, err := filepath.Abs(repoPath)
	if err != nil {
		return nil, err
	} else if !isDir(repoPath) {
		return nil, errors.New("no such file or directory")
	}

	return &Repository{
		Path:        repoPath,
		commitCache: newObjectCache(),
		tagCache:    newObjectCache(),
	}, nil
}

// CloneRepoOptions options when clone a repository
type CloneRepoOptions struct {
	Timeout time.Duration
	Mirror  bool
	Bare    bool
	Quiet   bool
	Branch  string
}

// Clone clones original repository to target path.
func Clone(from, to string, opts CloneRepoOptions) (err error) {
	toDir := path.Dir(to)
	if err = os.MkdirAll(toDir, os.ModePerm); err != nil {
		return err
	}

	cmd := NewCommand("clone")
	if opts.Mirror {
		cmd.AddArguments("--mirror")
	}
	if opts.Bare {
		cmd.AddArguments("--bare")
	}
	if opts.Quiet {
		cmd.AddArguments("--quiet")
	}
	if len(opts.Branch) > 0 {
		cmd.AddArguments("-b", opts.Branch)
	}
	cmd.AddArguments(from, to)

	if opts.Timeout <= 0 {
		opts.Timeout = -1
	}

	_, err = cmd.RunTimeout(opts.Timeout)
	return err
}

// PullRemoteOptions options when pull from remote
type PullRemoteOptions struct {
	Timeout time.Duration
	All     bool
	Rebase  bool
	Remote  string
	Branch  string
}

// Pull pulls changes from remotes.
func Pull(repoPath string, opts PullRemoteOptions) error {
	cmd := NewCommand("pull")
	if opts.Rebase {
		cmd.AddArguments("--rebase")
	}
	if opts.All {
		cmd.AddArguments("--all")
	} else {
		cmd.AddArguments(opts.Remote)
		cmd.AddArguments(opts.Branch)
	}

	if opts.Timeout <= 0 {
		opts.Timeout = -1
	}

	_, err := cmd.RunInDirTimeout(opts.Timeout, repoPath)
	return err
}

// Push pushs local commits to given remote branch.
func Push(repoPath, remote, branch string) error {
	_, err := NewCommand("push", remote, branch).RunInDir(repoPath)
	return err
}

// CheckoutOptions options when heck out some branch
type CheckoutOptions struct {
	Timeout   time.Duration
	Branch    string
	OldBranch string
}

// Checkout checkouts a branch
func Checkout(repoPath string, opts CheckoutOptions) error {
	cmd := NewCommand("checkout")
	if len(opts.OldBranch) > 0 {
		cmd.AddArguments("-b")
	}

	if opts.Timeout <= 0 {
		opts.Timeout = -1
	}

	cmd.AddArguments(opts.Branch)

	if len(opts.OldBranch) > 0 {
		cmd.AddArguments(opts.OldBranch)
	}

	_, err := cmd.RunInDirTimeout(opts.Timeout, repoPath)
	return err
}

// ResetHEAD resets HEAD to given revision or head of branch.
func ResetHEAD(repoPath string, hard bool, revision string) error {
	cmd := NewCommand("reset")
	if hard {
		cmd.AddArguments("--hard")
	}
	_, err := cmd.AddArguments(revision).RunInDir(repoPath)
	return err
}

// MoveFile moves a file to another file or directory.
func MoveFile(repoPath, oldTreeName, newTreeName string) error {
	_, err := NewCommand("mv").AddArguments(oldTreeName, newTreeName).RunInDir(repoPath)
	return err
}

// CountObject represents repository count objects report
type CountObject struct {
	Count       int64
	Size        int64
	InPack      int64
	Packs       int64
	SizePack    int64
	PrunePack   int64
	Garbage     int64
	SizeGarbage int64
}

const (
	statCount        = "count: "
	statSize         = "size: "
	statInpack       = "in-pack: "
	statPacks        = "packs: "
	statSizePack     = "size-pack: "
	statPrunePackage = "prune-package: "
	statGarbage      = "garbage: "
	statSizeGarbage  = "size-garbage: "
)

// GetRepoSize returns disk consumption for repo in path
func GetRepoSize(repoPath string) (*CountObject, error) {
	cmd := NewCommand("count-objects", "-v")
	stdout, err := cmd.RunInDir(repoPath)
	if err != nil {
		return nil, err
	}

	return parseSize(stdout), nil
}

// parseSize parses the output from count-objects and return a CountObject
func parseSize(objects string) *CountObject {
	repoSize := new(CountObject)
	for _, line := range strings.Split(objects, "\n") {
		switch {
		case strings.HasPrefix(line, statCount):
			repoSize.Count = com.StrTo(line[7:]).MustInt64()
		case strings.HasPrefix(line, statSize):
			repoSize.Size = com.StrTo(line[6:]).MustInt64() * 1024
		case strings.HasPrefix(line, statInpack):
			repoSize.InPack = com.StrTo(line[9:]).MustInt64()
		case strings.HasPrefix(line, statPacks):
			repoSize.Packs = com.StrTo(line[7:]).MustInt64()
		case strings.HasPrefix(line, statSizePack):
			repoSize.SizePack = com.StrTo(line[11:]).MustInt64() * 1024
		case strings.HasPrefix(line, statPrunePackage):
			repoSize.PrunePack = com.StrTo(line[16:]).MustInt64()
		case strings.HasPrefix(line, statGarbage):
			repoSize.Garbage = com.StrTo(line[9:]).MustInt64()
		case strings.HasPrefix(line, statSizeGarbage):
			repoSize.SizeGarbage = com.StrTo(line[14:]).MustInt64() * 1024
		}
	}
	return repoSize
}
<|endoftext|>


<commit_before>package parse

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"reflect"
	"strings"
	"time"
)

const (
	ParseVersion       = "1"
	AppIdHeader        = "X-Parse-Application-Id"
	RestKeyHeader      = "X-Parse-REST-API-Key"
	MasterKeyHeader    = "X-Parse-Master-Key"
	SessionTokenHeader = "X-Parse-Session-Token"
)

var fieldNameCache map[reflect.Type]map[string]string = make(map[reflect.Type]map[string]string)

type requestT interface {
	method() string
	endpoint() (string, error)
	body() (string, error)
	useMasterKey() bool
	session() *sessionT
}

type ParseError interface {
	error
	Code() int
	Message() string
}

type parseErrorT struct {
	Code    int    `json:"code" parse:"code"`
	Message string `json:"error" parse:"error"`
}

func (e *parseErrorT) Error() string {
	return fmt.Sprintf("error %d - %s", e.Code, e.Message)
}

type Client struct {
	appId     string
	restKey   string
	masterKey string
}

var defaultClient *Client

func (c *Client) logs() {
	req, err := http.NewRequest("GET", "https://api.parse.com/logs", nil)
	if err != nil {
		fmt.Printf("Error: %v\n", err)
		return
	}

	req.Header.Add(AppIdHeader, defaultClient.appId)
	req.Header.Add(MasterKeyHeader, c.masterKey)

	fmt.Printf("Executing request: [%+v]\n", req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Printf("Error: %v\n", err)
		return
	}

	fmt.Printf("resp code: %d\n", resp.StatusCode)
}

func Initialize(appId, restKey, masterKey string) {
	defaultClient = &Client{
		appId:     appId,
		restKey:   restKey,
		masterKey: masterKey,
	}
}

func (c *Client) doRequest(op requestT, dst interface{}) error {
	rv := reflect.ValueOf(dst)
	if rv.Kind() != reflect.Ptr || rv.IsNil() {
		return errors.New("v must be a non-nil pointer")
	}

	ep, err := op.endpoint()
	if err != nil {
		return err
	}

	method := op.method()
	var body io.Reader
	if method == "POST" || method == "PUT" {
		b, err := op.body()
		if err != nil {
			return err
		}
		body = strings.NewReader(b)
	}

	req, err := http.NewRequest(method, ep, body)
	if err != nil {
		return err
	}

	req.Header.Add(AppIdHeader, defaultClient.appId)
	if op.useMasterKey() && c.masterKey != "" && op.session() == nil {
		req.Header.Add(MasterKeyHeader, c.masterKey)
	} else {
		req.Header.Add(RestKeyHeader, c.restKey)
		if s := op.session(); s != nil {
			req.Header.Add(SessionTokenHeader, s.sessionToken)
		}
	}

	fmt.Printf("Executing request: [%+v]\n", req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	return handleResponse(resp, op, dst)
}

func getFields(t reflect.Type) []reflect.StructField {
	fields := make([]reflect.StructField, 0)

	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}

	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		ft := f.Type
		if ft.Kind() == reflect.Ptr {
			ft = ft.Elem()
		}
		switch ft.Kind() {
		case reflect.Struct:
			fields = append(fields, getFields(ft)...)
		default:
			fields = append(fields, f)
		}
	}

	return fields
}

func getFieldNameMap(v reflect.Value) map[string]string {
	// Get the actual type we care about. Indirect any pointers, and handle
	ind := reflect.Indirect(v)
	t := ind.Type()
	if t.Kind() == reflect.Slice || t.Kind() == reflect.Array {
		t = t.Elem()
		if t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
	}

	if f, ok := fieldNameCache[t]; ok {
		return f
	}

	fields := getFields(t)

	fieldMap := make(map[string]string)
	for _, f := range fields {
		if tag := f.Tag.Get("parse"); tag != "" {
			fieldMap[tag] = f.Name
		}
	}

	fieldNameCache[t] = fieldMap
	return fieldMap
}

func handleResponse(resp *http.Response, op requestT, dst interface{}) error {
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	// Error formats are consistent. If the response is an error,
	// return a ParseError
	if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
		ret := parseErrorT{}
		if err = json.Unmarshal(body, &ret); err != nil {
			return err
		}
		return &ret
	}

	data := make(map[string]interface{})
	err = json.Unmarshal(body, &data)
	if err != nil {
		return err
	}

	if c, ok := data["count"]; ok {
		return populateValue(dst, c)
	} else if r, ok := data["results"]; ok {
		// Handle query results
		return populateValue(dst, r)
	} else {
		return populateValue(dst, data)
	}
}

func populateValue(dst interface{}, src interface{}) error {
	dv := reflect.ValueOf(dst)
	dvi := reflect.Indirect(dv)

	sv := reflect.ValueOf(src)

	switch dvi.Kind() {
	case reflect.Slice, reflect.Array:
		if sv.Kind() == reflect.Slice || sv.Kind() == reflect.Array {
			dt := dvi.Type().Elem()
			for i := 0; i < sv.Len(); i++ {
				var newV reflect.Value
				if dt.Kind() == reflect.Ptr {
					newV = reflect.New(dt.Elem())
				} else {
					newV = reflect.New(dt)
				}

				err := populateValue(newV.Interface(), sv.Index(i).Interface())
				if err != nil {
					return err
				}
				if dt.Kind() == reflect.Ptr {
					dvi = reflect.Append(dvi, newV)
				} else {
					dvi = reflect.Append(dvi, reflect.Indirect(newV))
				}
				dv.Elem().Set(dvi)
			}
		} else {
			return fmt.Errorf("expected slice, got %s", sv.Kind())
		}
	case reflect.Struct: // TODO: Handle other Parse object types ?
		if dvi.Type() == reflect.TypeOf(time.Time{}) || dvi.Type() == reflect.TypeOf(Date{}) {
			// TODO: handle Parse "Date" type
			if s, ok := src.(string); ok {
				if t, err := parseTime(s); err != nil {
					return err
				} else {
					dvi.Set(reflect.ValueOf(t).Convert(dvi.Type()))
				}
			} else if m, ok := src.(map[string]interface{}); ok {
				if t, ok := m["__type"]; ok {
					if t == "Date" {
						if ds, ok := m["iso"]; ok {
							if t, err := parseTime(ds.(string)); err != nil {
								return err
							} else {
								dvi.Set(reflect.ValueOf(t).Convert(dvi.Type()))
							}
						} else {
							return fmt.Errorf("malformed Date type: %v", m)
						}
					} else {
						return fmt.Errorf("expected Date type got %s", t)
					}
				} else {
					return fmt.Errorf("no __type in object: %v", m)
				}
			} else {
				return fmt.Errorf("expected string or Date type, got %s", sv.Type())
			}
		} else if sv.Kind() == reflect.Map {
			fieldNameMap := getFieldNameMap(dvi)
			if m, ok := src.(map[string]interface{}); ok {
				if f := dvi.FieldByName("Extra"); f.IsValid() && f.CanSet() && f.IsNil() {
					f.Set(reflect.ValueOf(make(map[string]interface{})))
				}

				for k, v := range m {
					if k == "__type" {
						continue
					}

					if nk, ok := fieldNameMap[k]; ok {
						k = nk
					}

					k = strings.Title(k) // Make first letter uppercase. TODO: find better way

					if f := dvi.FieldByName(k); f.IsValid() {
						if f.Kind() == reflect.Ptr {
							if f.IsNil() {
								f.Set(reflect.New(f.Type().Elem()))
							}
						}

						fi := reflect.Indirect(f)
						if fi.CanSet() {
							var err error
							if f.Kind() == reflect.Ptr {
								err = populateValue(f.Interface(), v)
							} else {
								fptr := f.Addr()
								err = populateValue(fptr.Interface(), v)
							}
							if err != nil {
								return fmt.Errorf("can not set field %s - %s", k, err)
							}
						}
					} else if f := dvi.FieldByName("Extra"); f.IsValid() && f.Kind() == reflect.Map {
						f.SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(v))
					}
				}
			} else {
				return fmt.Errorf("expected map[string]interface{} got %s", sv.Type())
			}
		} else if sv.Kind() == reflect.Slice && sv.Len() == 1{
			return populateValue(dst, sv.Index(0).Interface())
		} else {
			return fmt.Errorf("expected map, got %s", sv.Kind())
		}
	default:
		if dvi.Kind() == reflect.Ptr {
			if dvi.IsNil() {
				dvi = reflect.New(dvi.Type())
			}

			dvi = dvi.Elem()
		}

		if sv.Type().AssignableTo(dvi.Type()) {
			if dvi.CanSet() {
				dvi.Set(sv)
			}
			return nil
		} else if sv.Type().ConvertibleTo(dvi.Type()) {
			newV := sv.Convert(dvi.Type())
			if dvi.CanSet() {
				dvi.Set(newV)
			}
			return nil
		}
	}

	return nil
}

func parseTime(s string) (time.Time, error) {
	return time.Parse(time.RFC3339Nano, s)
}
<commit_msg>remove experimental code<commit_after>package parse

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"reflect"
	"strings"
	"time"
)

const (
	ParseVersion       = "1"
	AppIdHeader        = "X-Parse-Application-Id"
	RestKeyHeader      = "X-Parse-REST-API-Key"
	MasterKeyHeader    = "X-Parse-Master-Key"
	SessionTokenHeader = "X-Parse-Session-Token"
)

var fieldNameCache map[reflect.Type]map[string]string = make(map[reflect.Type]map[string]string)

type requestT interface {
	method() string
	endpoint() (string, error)
	body() (string, error)
	useMasterKey() bool
	session() *sessionT
}

type ParseError interface {
	error
	Code() int
	Message() string
}

type parseErrorT struct {
	Code    int    `json:"code" parse:"code"`
	Message string `json:"error" parse:"error"`
}

func (e *parseErrorT) Error() string {
	return fmt.Sprintf("error %d - %s", e.Code, e.Message)
}

type Client struct {
	appId     string
	restKey   string
	masterKey string
}

var defaultClient *Client

func Initialize(appId, restKey, masterKey string) {
	defaultClient = &Client{
		appId:     appId,
		restKey:   restKey,
		masterKey: masterKey,
	}
}

func (c *Client) doRequest(op requestT, dst interface{}) error {
	rv := reflect.ValueOf(dst)
	if rv.Kind() != reflect.Ptr || rv.IsNil() {
		return errors.New("v must be a non-nil pointer")
	}

	ep, err := op.endpoint()
	if err != nil {
		return err
	}

	method := op.method()
	var body io.Reader
	if method == "POST" || method == "PUT" {
		b, err := op.body()
		if err != nil {
			return err
		}
		body = strings.NewReader(b)
	}

	req, err := http.NewRequest(method, ep, body)
	if err != nil {
		return err
	}

	req.Header.Add(AppIdHeader, defaultClient.appId)
	if op.useMasterKey() && c.masterKey != "" && op.session() == nil {
		req.Header.Add(MasterKeyHeader, c.masterKey)
	} else {
		req.Header.Add(RestKeyHeader, c.restKey)
		if s := op.session(); s != nil {
			req.Header.Add(SessionTokenHeader, s.sessionToken)
		}
	}

	fmt.Printf("Executing request: [%+v]\n", req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	return handleResponse(resp, op, dst)
}

func getFields(t reflect.Type) []reflect.StructField {
	fields := make([]reflect.StructField, 0)

	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}

	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		ft := f.Type
		if ft.Kind() == reflect.Ptr {
			ft = ft.Elem()
		}
		switch ft.Kind() {
		case reflect.Struct:
			fields = append(fields, getFields(ft)...)
		default:
			fields = append(fields, f)
		}
	}

	return fields
}

func getFieldNameMap(v reflect.Value) map[string]string {
	// Get the actual type we care about. Indirect any pointers, and handle
	ind := reflect.Indirect(v)
	t := ind.Type()
	if t.Kind() == reflect.Slice || t.Kind() == reflect.Array {
		t = t.Elem()
		if t.Kind() == reflect.Ptr {
			t = t.Elem()
		}
	}

	if f, ok := fieldNameCache[t]; ok {
		return f
	}

	fields := getFields(t)

	fieldMap := make(map[string]string)
	for _, f := range fields {
		if tag := f.Tag.Get("parse"); tag != "" {
			fieldMap[tag] = f.Name
		}
	}

	fieldNameCache[t] = fieldMap
	return fieldMap
}

func handleResponse(resp *http.Response, op requestT, dst interface{}) error {
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	// Error formats are consistent. If the response is an error,
	// return a ParseError
	if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
		ret := parseErrorT{}
		if err = json.Unmarshal(body, &ret); err != nil {
			return err
		}
		return &ret
	}

	data := make(map[string]interface{})
	err = json.Unmarshal(body, &data)
	if err != nil {
		return err
	}

	if c, ok := data["count"]; ok {
		return populateValue(dst, c)
	} else if r, ok := data["results"]; ok {
		// Handle query results
		return populateValue(dst, r)
	} else {
		return populateValue(dst, data)
	}
}

func populateValue(dst interface{}, src interface{}) error {
	dv := reflect.ValueOf(dst)
	dvi := reflect.Indirect(dv)

	sv := reflect.ValueOf(src)

	switch dvi.Kind() {
	case reflect.Slice, reflect.Array:
		if sv.Kind() == reflect.Slice || sv.Kind() == reflect.Array {
			dt := dvi.Type().Elem()
			for i := 0; i < sv.Len(); i++ {
				var newV reflect.Value
				if dt.Kind() == reflect.Ptr {
					newV = reflect.New(dt.Elem())
				} else {
					newV = reflect.New(dt)
				}

				err := populateValue(newV.Interface(), sv.Index(i).Interface())
				if err != nil {
					return err
				}
				if dt.Kind() == reflect.Ptr {
					dvi = reflect.Append(dvi, newV)
				} else {
					dvi = reflect.Append(dvi, reflect.Indirect(newV))
				}
				dv.Elem().Set(dvi)
			}
		} else {
			return fmt.Errorf("expected slice, got %s", sv.Kind())
		}
	case reflect.Struct: // TODO: Handle other Parse object types ?
		if dvi.Type() == reflect.TypeOf(time.Time{}) || dvi.Type() == reflect.TypeOf(Date{}) {
			// TODO: handle Parse "Date" type
			if s, ok := src.(string); ok {
				if t, err := parseTime(s); err != nil {
					return err
				} else {
					dvi.Set(reflect.ValueOf(t).Convert(dvi.Type()))
				}
			} else if m, ok := src.(map[string]interface{}); ok {
				if t, ok := m["__type"]; ok {
					if t == "Date" {
						if ds, ok := m["iso"]; ok {
							if t, err := parseTime(ds.(string)); err != nil {
								return err
							} else {
								dvi.Set(reflect.ValueOf(t).Convert(dvi.Type()))
							}
						} else {
							return fmt.Errorf("malformed Date type: %v", m)
						}
					} else {
						return fmt.Errorf("expected Date type got %s", t)
					}
				} else {
					return fmt.Errorf("no __type in object: %v", m)
				}
			} else {
				return fmt.Errorf("expected string or Date type, got %s", sv.Type())
			}
		} else if sv.Kind() == reflect.Map {
			fieldNameMap := getFieldNameMap(dvi)
			if m, ok := src.(map[string]interface{}); ok {
				if f := dvi.FieldByName("Extra"); f.IsValid() && f.CanSet() && f.IsNil() {
					f.Set(reflect.ValueOf(make(map[string]interface{})))
				}

				for k, v := range m {
					if k == "__type" {
						continue
					}

					if nk, ok := fieldNameMap[k]; ok {
						k = nk
					}

					k = strings.Title(k) // Make first letter uppercase. TODO: find better way

					if f := dvi.FieldByName(k); f.IsValid() {
						if f.Kind() == reflect.Ptr {
							if f.IsNil() {
								f.Set(reflect.New(f.Type().Elem()))
							}
						}

						fi := reflect.Indirect(f)
						if fi.CanSet() {
							var err error
							if f.Kind() == reflect.Ptr {
								err = populateValue(f.Interface(), v)
							} else {
								fptr := f.Addr()
								err = populateValue(fptr.Interface(), v)
							}
							if err != nil {
								return fmt.Errorf("can not set field %s - %s", k, err)
							}
						}
					} else if f := dvi.FieldByName("Extra"); f.IsValid() && f.Kind() == reflect.Map {
						f.SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(v))
					}
				}
			} else {
				return fmt.Errorf("expected map[string]interface{} got %s", sv.Type())
			}
		} else if sv.Kind() == reflect.Slice && sv.Len() == 1{
			return populateValue(dst, sv.Index(0).Interface())
		} else {
			return fmt.Errorf("expected map, got %s", sv.Kind())
		}
	default:
		if dvi.Kind() == reflect.Ptr {
			if dvi.IsNil() {
				dvi = reflect.New(dvi.Type())
			}

			dvi = dvi.Elem()
		}

		if sv.Type().AssignableTo(dvi.Type()) {
			if dvi.CanSet() {
				dvi.Set(sv)
			}
			return nil
		} else if sv.Type().ConvertibleTo(dvi.Type()) {
			newV := sv.Convert(dvi.Type())
			if dvi.CanSet() {
				dvi.Set(newV)
			}
			return nil
		}
	}

	return nil
}

func parseTime(s string) (time.Time, error) {
	return time.Parse(time.RFC3339Nano, s)
}
<|endoftext|>