<commit_before>package fixchain

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/certificate-transparency/go/x509"
)

// Limiter is an interface to allow different rate limiters to be used with the
// Logger.
type Limiter interface {
	Wait()
}

// Logger contains methods to asynchronously log certificate chains to a
// Certificate Transparency log and properties to store information about each
// attempt that is made to post a certificate chain to said log.
type Logger struct {
	url    string
	client *http.Client
	roots  *x509.CertPool
	toPost chan *toPost
	errors chan<- *FixError

	active uint32

	queued        uint32 // How many chains have been queued to be posted.
	posted        uint32 // How many chains have been posted.
	reposted      uint32 // How many chains for an already-posted cert have been queued.
	chainReposted uint32 // How many chains have been queued again.

	// Note that wg counts the number of active requests, not
	// active servers, because we can't close it to signal the
	// end, because of retries.
	wg      sync.WaitGroup
	limiter Limiter

	postCertCache  *lockedMap
	postChainCache *lockedMap
}

// IsPosted tells the caller whether a chain for the given certificate has
// already been successfully posted to the log by this Logger.
func (l *Logger) IsPosted(cert *x509.Certificate) bool {
	return l.postCertCache.get(hash(cert))
}

// QueueChain adds the given chain to the queue to be posted to the log.
func (l *Logger) QueueChain(chain []*x509.Certificate) {
	if chain == nil {
		return
	}

	atomic.AddUint32(&l.queued, 1)
	// Has a chain for the cert this chain if for already been successfully
	//posted to the log by this Logger?
	h := hash(chain[0]) // Chains are cert -> root
	if l.postCertCache.get(h) {
		atomic.AddUint32(&l.reposted, 1)
		return // Don't post chain for a cert that has already had a chain posted.
	}
	// If we assume all chains for the same cert are equally
	// likely to succeed, then we could mark the cert as posted
	// here. However, bugs might cause a log to refuse one chain
	// and accept another, so try each unique chain.

	// Has this Logger already tried to post this chain?
	h = hashChain(chain)
	if l.postChainCache.get(h) {
		atomic.AddUint32(&l.chainReposted, 1)
		return
	}
	l.postChainCache.set(h, true)

	p := &toPost{chain: chain, retries: 5}
	l.postToLog(p)
}

// Wait for all of the active requests to finish being processed.
func (l *Logger) Wait() {
	l.wg.Wait()
}

// RootCerts returns the root certificates that the log accepts.
func (l *Logger) RootCerts() *x509.CertPool {
	if l.roots == nil {
		l.roots = l.getRoots()
	}
	return l.roots
}

func (l *Logger) getRoots() *x509.CertPool {
	rootsJSON, err := l.client.Get(l.url + "/ct/v1/get-roots")
	if err != nil {
		log.Fatalf("can't get roots from %s: %s", l.url, err)
	}
	defer rootsJSON.Body.Close()
	if rootsJSON.StatusCode != 200 {
		log.Fatalf("can't deal with status other than 200 from %s: %d", l.url, rootsJSON.StatusCode)
	}
	j, err := ioutil.ReadAll(rootsJSON.Body)
	if err != nil {
		log.Fatalf("can't read roots from %s: %s", l.url, err)
	}
	type Certificates struct {
		Certificates [][]byte
	}
	var certs Certificates
	err = json.Unmarshal(j, &certs)
	if err != nil {
		log.Fatalf("can't parse json (%s) from %s: %s", err, l.url, j)
	}
	ret := x509.NewCertPool()
	for i := 0; i < len(certs.Certificates); i++ {
		r, err := x509.ParseCertificate(certs.Certificates[i])
		switch err.(type) {
		case nil, x509.NonFatalErrors:
			// ignore
		default:
			log.Fatalf("can't parse certificate from %s: %s %#v", l.url, err, certs.Certificates[i])
		}
		ret.AddCert(r)
	}
	return ret
}

type toPost struct {
	chain   []*x509.Certificate
	retries uint8
}

// postToLog(), rather than its asynchronous couterpart asyncPostToLog(), is
// used during the initial queueing of chains to avoid spinning up an excessive
// number of goroutines, and unecessarily using up memory. If asyncPostToLog()
// was called instead, then every time a new chain was queued, a new goroutine
// would be created, each holding their own chain - regardless of whether there
// were postServers available to process them or not.  If a large number of
// chains were queued in a short period of time, this could lead to a large
// number of these additional goroutines being created, resulting in excessive
// memory usage.
func (l *Logger) postToLog(p *toPost) {
	l.wg.Add(1) // Add to the wg as we are adding a new active request to the logger queue.
	l.toPost <- p
}

// asyncPostToLog(), rather than its synchronous couterpart postToLog(), is used
// during retries to avoid deadlock. Without the separate goroutine created in
// asyncPostToLog(), deadlock can occur in the following situation:
//
// Suppose there is only one postServer() goroutine running, and it is blocked
// waiting for a toPost on the toPost chan.  A toPost gets added to the chan,
// which causes the following to happen:
// - the postServer takes the toPost from the chan.
// - the postServer calls l.postChain(toPost), and waits for
//   l.postChain() to return before going back to the toPost
//   chan for another toPost.
// - l.postChain() begins execution.  Suppose the first post
//   attempt of the toPost fails for some network-related
//   reason.
// - l.postChain retries and calls l.postToLog() to queue up the
//   toPost to try to post it again.
// - l.postToLog() tries to put the toPost on the toPost chan,
//   and blocks until a postServer takes it off the chan.
// But the one and only postServer is still waiting for l.postChain (and
// therefore l.postToLog) to return, and will not go to take another toPost off
// the toPost chan until that happens.
// Thus, deadlock.
//
// Similar situations with multiple postServers can easily be imagined.
func (l *Logger) asyncPostToLog(p *toPost) {
	l.wg.Add(1) // Add to the wg as we are adding a new active request to the logger queue.
	go func() {
		l.toPost <- p
	}()
}

func (l *Logger) postChain(p *toPost) {
	h := hash(p.chain[0])
	if l.postCertCache.get(h) {
		atomic.AddUint32(&l.reposted, 1)
		return
	}

	l.limiter.Wait()
	ferr := PostChainToLog(p.chain, l.client, l.url)
	atomic.AddUint32(&l.posted, 1)
	if ferr != nil {
		switch ferr.Type {
		case PostFailed:
			if p.retries == 0 {
				l.errors <- ferr
			} else {
				log.Printf(ferr.Error.Error())
				p.retries--
				l.asyncPostToLog(p)
			}
			return
		case LogPostFailed:
			// If the http error code is 502, we retry.
			// TODO(katjoyce): Are there any other error codes for which the
			// post should be retried?
			if p.retries == 0 || ferr.Code != 502 {
				l.errors <- ferr
			} else {
				p.retries--
				l.asyncPostToLog(p)
			}
			return
		default:
			log.Fatalf("Unexpected FixError type: %s", ferr.TypeString())
		}
	}

	// If the post was successful, cache.
	l.postCertCache.set(h, true)
}

func (l *Logger) postServer() {
	for {
		c := <-l.toPost
		atomic.AddUint32(&l.active, 1)
		l.postChain(c)
		atomic.AddUint32(&l.active, ^uint32(0))
		l.wg.Done()
	}
}

func (l *Logger) logStats() {
	t := time.NewTicker(time.Second)
	go func() {
		for _ = range t.C {
			log.Printf("posters: %d active, %d posted, %d queued, %d certs requeued, %d chains requeued",
				l.active, l.posted, l.queued, l.reposted, l.chainReposted)
		}
	}()
}

// NewLogger creates a new asynchronous logger to log chains to the
// Certificate Transparency log at the given url.  It starts up a pool of
// workerCount workers.  Errors are pushed to the errors channel.  client is
// used to post the chains to the log.
func NewLogger(workerCount int, url string, errors chan<- *FixError, client *http.Client, limiter Limiter, logStats bool) *Logger {
	l := &Logger{
		url:            url,
		client:         client,
		errors:         errors,
		toPost:         make(chan *toPost),
		postCertCache:  newLockedMap(),
		postChainCache: newLockedMap(),
		limiter:        limiter,
	}

	// Start post server pool.
	for i := 0; i < workerCount; i++ {
		go l.postServer()
	}

	if logStats {
		l.logStats()
	}
	return l
}
<commit_msg>Add retries when trying to fetch roots.<commit_after>package fixchain

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/certificate-transparency/go/x509"
)

// Limiter is an interface to allow different rate limiters to be used with the
// Logger.
type Limiter interface {
	Wait()
}

// Logger contains methods to asynchronously log certificate chains to a
// Certificate Transparency log and properties to store information about each
// attempt that is made to post a certificate chain to said log.
type Logger struct {
	url    string
	client *http.Client
	roots  *x509.CertPool
	toPost chan *toPost
	errors chan<- *FixError

	active uint32

	queued        uint32 // How many chains have been queued to be posted.
	posted        uint32 // How many chains have been posted.
	reposted      uint32 // How many chains for an already-posted cert have been queued.
	chainReposted uint32 // How many chains have been queued again.

	// Note that wg counts the number of active requests, not
	// active servers, because we can't close it to signal the
	// end, because of retries.
	wg      sync.WaitGroup
	limiter Limiter

	postCertCache  *lockedMap
	postChainCache *lockedMap
}

// IsPosted tells the caller whether a chain for the given certificate has
// already been successfully posted to the log by this Logger.
func (l *Logger) IsPosted(cert *x509.Certificate) bool {
	return l.postCertCache.get(hash(cert))
}

// QueueChain adds the given chain to the queue to be posted to the log.
func (l *Logger) QueueChain(chain []*x509.Certificate) {
	if chain == nil {
		return
	}

	atomic.AddUint32(&l.queued, 1)
	// Has a chain for the cert this chain if for already been successfully
	//posted to the log by this Logger?
	h := hash(chain[0]) // Chains are cert -> root
	if l.postCertCache.get(h) {
		atomic.AddUint32(&l.reposted, 1)
		return // Don't post chain for a cert that has already had a chain posted.
	}
	// If we assume all chains for the same cert are equally
	// likely to succeed, then we could mark the cert as posted
	// here. However, bugs might cause a log to refuse one chain
	// and accept another, so try each unique chain.

	// Has this Logger already tried to post this chain?
	h = hashChain(chain)
	if l.postChainCache.get(h) {
		atomic.AddUint32(&l.chainReposted, 1)
		return
	}
	l.postChainCache.set(h, true)

	p := &toPost{chain: chain, retries: 5}
	l.postToLog(p)
}

// Wait for all of the active requests to finish being processed.
func (l *Logger) Wait() {
	l.wg.Wait()
}

// RootCerts returns the root certificates that the log accepts.
func (l *Logger) RootCerts() *x509.CertPool {
	if l.roots == nil {
		// Retry if unable to get roots.
		for i := 0; i < 10; i++ {
			roots, err := l.getRoots()
			if err == nil {
				l.roots = roots
				return l.roots
			}
			log.Println(err)
		}
		log.Fatalf("Can't get roots from %s", l.url)
	}
	return l.roots
}

func (l *Logger) getRoots() (*x509.CertPool, error) {
	rootsJSON, err := l.client.Get(l.url + "/ct/v1/get-roots")
	if err != nil {
		return nil, fmt.Errorf("can't get roots from %s: %s", l.url, err)
	}
	defer rootsJSON.Body.Close()
	j, err := ioutil.ReadAll(rootsJSON.Body)
	if err != nil {
		return nil, fmt.Errorf("can't read body from %s: %s", l.url, err)
	}
	if rootsJSON.StatusCode != 200 {
		return nil, fmt.Errorf("can't deal with status other than 200 from %s: %d\nbody: %s", l.url, rootsJSON.StatusCode, string(j))
	}
	type Certificates struct {
		Certificates [][]byte
	}
	var certs Certificates
	err = json.Unmarshal(j, &certs)
	if err != nil {
		return nil, fmt.Errorf("can't parse json (%s) from %s: %s", err, l.url, j)
	}
	ret := x509.NewCertPool()
	for i := 0; i < len(certs.Certificates); i++ {
		r, err := x509.ParseCertificate(certs.Certificates[i])
		switch err.(type) {
		case nil, x509.NonFatalErrors:
			// ignore
		default:
			return nil, fmt.Errorf("can't parse certificate from %s: %s %#v", l.url, err, certs.Certificates[i])
		}
		ret.AddCert(r)
	}
	return ret, nil
}

type toPost struct {
	chain   []*x509.Certificate
	retries uint8
}

// postToLog(), rather than its asynchronous couterpart asyncPostToLog(), is
// used during the initial queueing of chains to avoid spinning up an excessive
// number of goroutines, and unecessarily using up memory. If asyncPostToLog()
// was called instead, then every time a new chain was queued, a new goroutine
// would be created, each holding their own chain - regardless of whether there
// were postServers available to process them or not.  If a large number of
// chains were queued in a short period of time, this could lead to a large
// number of these additional goroutines being created, resulting in excessive
// memory usage.
func (l *Logger) postToLog(p *toPost) {
	l.wg.Add(1) // Add to the wg as we are adding a new active request to the logger queue.
	l.toPost <- p
}

// asyncPostToLog(), rather than its synchronous couterpart postToLog(), is used
// during retries to avoid deadlock. Without the separate goroutine created in
// asyncPostToLog(), deadlock can occur in the following situation:
//
// Suppose there is only one postServer() goroutine running, and it is blocked
// waiting for a toPost on the toPost chan.  A toPost gets added to the chan,
// which causes the following to happen:
// - the postServer takes the toPost from the chan.
// - the postServer calls l.postChain(toPost), and waits for
//   l.postChain() to return before going back to the toPost
//   chan for another toPost.
// - l.postChain() begins execution.  Suppose the first post
//   attempt of the toPost fails for some network-related
//   reason.
// - l.postChain retries and calls l.postToLog() to queue up the
//   toPost to try to post it again.
// - l.postToLog() tries to put the toPost on the toPost chan,
//   and blocks until a postServer takes it off the chan.
// But the one and only postServer is still waiting for l.postChain (and
// therefore l.postToLog) to return, and will not go to take another toPost off
// the toPost chan until that happens.
// Thus, deadlock.
//
// Similar situations with multiple postServers can easily be imagined.
func (l *Logger) asyncPostToLog(p *toPost) {
	l.wg.Add(1) // Add to the wg as we are adding a new active request to the logger queue.
	go func() {
		l.toPost <- p
	}()
}

func (l *Logger) postChain(p *toPost) {
	h := hash(p.chain[0])
	if l.postCertCache.get(h) {
		atomic.AddUint32(&l.reposted, 1)
		return
	}

	l.limiter.Wait()
	ferr := PostChainToLog(p.chain, l.client, l.url)
	atomic.AddUint32(&l.posted, 1)
	if ferr != nil {
		switch ferr.Type {
		case PostFailed:
			if p.retries == 0 {
				l.errors <- ferr
			} else {
				log.Printf(ferr.Error.Error())
				p.retries--
				l.asyncPostToLog(p)
			}
			return
		case LogPostFailed:
			// If the http error code is 502, we retry.
			// TODO(katjoyce): Are there any other error codes for which the
			// post should be retried?
			if p.retries == 0 || ferr.Code != 502 {
				l.errors <- ferr
			} else {
				p.retries--
				l.asyncPostToLog(p)
			}
			return
		default:
			log.Fatalf("Unexpected FixError type: %s", ferr.TypeString())
		}
	}

	// If the post was successful, cache.
	l.postCertCache.set(h, true)
}

func (l *Logger) postServer() {
	for {
		c := <-l.toPost
		atomic.AddUint32(&l.active, 1)
		l.postChain(c)
		atomic.AddUint32(&l.active, ^uint32(0))
		l.wg.Done()
	}
}

func (l *Logger) logStats() {
	t := time.NewTicker(time.Second)
	go func() {
		for _ = range t.C {
			log.Printf("posters: %d active, %d posted, %d queued, %d certs requeued, %d chains requeued",
				l.active, l.posted, l.queued, l.reposted, l.chainReposted)
		}
	}()
}

// NewLogger creates a new asynchronous logger to log chains to the
// Certificate Transparency log at the given url.  It starts up a pool of
// workerCount workers.  Errors are pushed to the errors channel.  client is
// used to post the chains to the log.
func NewLogger(workerCount int, url string, errors chan<- *FixError, client *http.Client, limiter Limiter, logStats bool) *Logger {
	l := &Logger{
		url:            url,
		client:         client,
		errors:         errors,
		toPost:         make(chan *toPost),
		postCertCache:  newLockedMap(),
		postChainCache: newLockedMap(),
		limiter:        limiter,
	}

	// Start post server pool.
	for i := 0; i < workerCount; i++ {
		go l.postServer()
	}

	if logStats {
		l.logStats()
	}
	return l
}
<|endoftext|>


<commit_before>package yorm

import (
	"database/sql"
	"errors"
	"reflect"
	"strings"
	"sync"
	"time"
)

type tableSetter struct {
	table    string
	dests    []interface{}
	columns  []*column
	pkColumn *column
}

var (
	//TimeType time's reflect type.
	TimeType   = reflect.TypeOf(time.Time{})
	BoolType   = reflect.TypeOf(true)
	Int64Type  = reflect.TypeOf(int64(0))
	StringType = reflect.TypeOf("")

	// one struct reflect to a table query setter
	tableMap = map[reflect.Value]*tableSetter{}
	//table lock
	tableRWLock sync.RWMutex
)

func newTableSetter(ri reflect.Value) (*tableSetter, error) {
	if q, ok := tableMap[ri]; ok {
		//		if t, ok := ri.Interface().(YormTableStruct); ok {
		//			returnValue := *q
		//			returnValue.table = t.YormTableName()
		//			return &returnValue, nil
		//		}
		return q, nil
	}
	tableRWLock.Lock()
	defer tableRWLock.Unlock()
	if q, ok := tableMap[ri]; ok {
		return q, nil
	}
	if ri.Kind() != reflect.Ptr {
		return nil, ErrNonPtr
	}
	// todo maybe a bug here
	if ri.IsNil() {
		return nil, ErrNotSupported
	}
	q := new(tableSetter)
	table, cs := structToTable(reflect.Indirect(ri).Interface())
	if len(cs) == 0 {
		return nil, ErrNotSupported
	}
	q.pkColumn, _ = findPkColumn(cs)
	q.table = table
	q.columns = cs
	q.dests = make([]interface{}, len(cs))
	for k, v := range cs {
		q.dests[k] = newPtrInterface(v.typ)
	}
	tableMap[ri] = q
	return q, nil
}

func findPkColumn(cs []*column) (*column, error) {
	var c *column
	var idColumn *column
	isPk := false

	for _, v := range cs {
		if strings.ToLower(v.name) == "id" {
			idColumn = v
		}
		if v.isPK {
			if isPk {
				return c, ErrDuplicatePkColumn
			}
			isPk = true
			c = v
		}
	}
	if c == nil && idColumn != nil {
		idColumn.isPK = true
		idColumn.isAuto = true
		c = idColumn
	}
	if c == nil {
		return nil, ErrNonePkColumn
	}
	return c, nil
}

func newPtrInterface(t reflect.Type) interface{} {
	k := t.Kind()
	var ti interface{}
	switch k {
	case reflect.Bool:
		fallthrough
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		fallthrough
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		ti = new(sql.NullInt64)
	case reflect.String:
		ti = new(sql.NullString)
	case reflect.Float32, reflect.Float64:
		ti = new(sql.NullFloat64)
	case reflect.Struct:
		switch t {
		case TimeType:
			ti = new(sql.NullString)
		}
	}
	return ti
}

func scanValue(sc sqlScanner, q *tableSetter, st reflect.Value) error {
	err := sc.Scan(q.dests...)
	if err != nil {
		return err
	}
	for idx, c := range q.columns {
		// different assign func here
		fv := st.Field(c.fieldNum)
		fi := q.dests[idx]
		err := setValue(fv, fi)
		if err != nil {
			continue
		}
	}
	return nil
}

func setValue(fv reflect.Value, fi interface{}) error {
	switch typ := fv.Type().Kind(); typ {

	case reflect.Bool:
		sqlValue := sql.NullInt64(*(fi.(*sql.NullInt64)))
		if !sqlValue.Valid {
			fv.SetBool(false)
			return errors.New("sqlValue is invalid")
		}
		fv.SetBool(sqlValue.Int64 > 0)

	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		fallthrough

	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		sqlValue := sql.NullInt64(*(fi.(*sql.NullInt64)))
		if !sqlValue.Valid {
			fv.SetInt(0)
			return errors.New("sqlValue is invalid")
		}
		fv.SetInt(sqlValue.Int64)

	case reflect.String:
		sqlValue := sql.NullString(*(fi.(*sql.NullString)))
		if !sqlValue.Valid {
			fv.SetString("")
			return errors.New("sqlValue is invalid")
		}
		fv.SetString(sqlValue.String)

	case reflect.Float32, reflect.Float64:
		sqlValue := sql.NullFloat64(*(fi.(*sql.NullFloat64)))
		if !sqlValue.Valid {
			fv.SetFloat(0.0)
			return errors.New("sqlValue is invalid")
		}
		fv.SetFloat(sqlValue.Float64)

	case reflect.Struct:
		switch fv.Type() {
		case TimeType:
			sqlValue := sql.NullString(*(fi.(*sql.NullString)))
			if !sqlValue.Valid {
				return errors.New("sqlValue is invalid")
			}
			timeStr := sqlValue.String
			var layout string
			if len(timeStr) == 10 {
				layout = shortSimpleTimeFormat
			}
			if len(timeStr) == 19 {
				layout = longSimpleTimeFormat
			}
			timeTime, err := time.ParseInLocation(layout, timeStr, time.Local)
			if timeTime.IsZero() {
				return err
			}
			fv.Set(reflect.ValueOf(timeTime))
		}
	}
	return nil
}
<commit_msg>cache reflect.Type not reflect.Value<commit_after>package yorm

import (
	"database/sql"
	"errors"
	"reflect"
	"strings"
	"sync"
	"time"
)

type tableSetter struct {
	table    string
	dests    []interface{}
	columns  []*column
	pkColumn *column
}

var (
	//TimeType time's reflect type.
	TimeType   = reflect.TypeOf(time.Time{})
	BoolType   = reflect.TypeOf(true)
	Int64Type  = reflect.TypeOf(int64(0))
	StringType = reflect.TypeOf("")

	// one struct reflect to a table query setter
	tableMap = map[reflect.Type]*tableSetter{}
	//table lock
	tableRWLock sync.RWMutex
)

func newTableSetter(ri reflect.Value) (*tableSetter, error) {
	if q, ok := tableMap[ri.Type()]; ok {
		//		if t, ok := ri.Interface().(YormTableStruct); ok {
		//			returnValue := *q
		//			returnValue.table = t.YormTableName()
		//			return &returnValue, nil
		//		}
		return q, nil
	}
	tableRWLock.Lock()
	defer tableRWLock.Unlock()
	if q, ok := tableMap[ri.Type()]; ok {
		return q, nil
	}
	if ri.Kind() != reflect.Ptr {
		return nil, ErrNonPtr
	}
	// todo maybe a bug here
	if ri.IsNil() {
		return nil, ErrNotSupported
	}
	q := new(tableSetter)
	table, cs := structToTable(reflect.Indirect(ri).Interface())
	if len(cs) == 0 {
		return nil, ErrNotSupported
	}
	q.pkColumn, _ = findPkColumn(cs)
	q.table = table
	q.columns = cs
	q.dests = make([]interface{}, len(cs))
	for k, v := range cs {
		q.dests[k] = newPtrInterface(v.typ)
	}
	tableMap[ri.Type()] = q
	return q, nil
}

func findPkColumn(cs []*column) (*column, error) {
	var c *column
	var idColumn *column
	isPk := false

	for _, v := range cs {
		if strings.ToLower(v.name) == "id" {
			idColumn = v
		}
		if v.isPK {
			if isPk {
				return c, ErrDuplicatePkColumn
			}
			isPk = true
			c = v
		}
	}
	if c == nil && idColumn != nil {
		idColumn.isPK = true
		idColumn.isAuto = true
		c = idColumn
	}
	if c == nil {
		return nil, ErrNonePkColumn
	}
	return c, nil
}

func newPtrInterface(t reflect.Type) interface{} {
	k := t.Kind()
	var ti interface{}
	switch k {
	case reflect.Bool:
		fallthrough
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		fallthrough
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		ti = new(sql.NullInt64)
	case reflect.String:
		ti = new(sql.NullString)
	case reflect.Float32, reflect.Float64:
		ti = new(sql.NullFloat64)
	case reflect.Struct:
		switch t {
		case TimeType:
			ti = new(sql.NullString)
		}
	}
	return ti
}

func scanValue(sc sqlScanner, q *tableSetter, st reflect.Value) error {
	err := sc.Scan(q.dests...)
	if err != nil {
		return err
	}
	for idx, c := range q.columns {
		// different assign func here
		fv := st.Field(c.fieldNum)
		fi := q.dests[idx]
		err := setValue(fv, fi)
		if err != nil {
			continue
		}
	}
	return nil
}

func setValue(fv reflect.Value, fi interface{}) error {
	switch typ := fv.Type().Kind(); typ {

	case reflect.Bool:
		sqlValue := sql.NullInt64(*(fi.(*sql.NullInt64)))
		if !sqlValue.Valid {
			fv.SetBool(false)
			return errors.New("sqlValue is invalid")
		}
		fv.SetBool(sqlValue.Int64 > 0)

	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		fallthrough

	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		sqlValue := sql.NullInt64(*(fi.(*sql.NullInt64)))
		if !sqlValue.Valid {
			fv.SetInt(0)
			return errors.New("sqlValue is invalid")
		}
		fv.SetInt(sqlValue.Int64)

	case reflect.String:
		sqlValue := sql.NullString(*(fi.(*sql.NullString)))
		if !sqlValue.Valid {
			fv.SetString("")
			return errors.New("sqlValue is invalid")
		}
		fv.SetString(sqlValue.String)

	case reflect.Float32, reflect.Float64:
		sqlValue := sql.NullFloat64(*(fi.(*sql.NullFloat64)))
		if !sqlValue.Valid {
			fv.SetFloat(0.0)
			return errors.New("sqlValue is invalid")
		}
		fv.SetFloat(sqlValue.Float64)

	case reflect.Struct:
		switch fv.Type() {
		case TimeType:
			sqlValue := sql.NullString(*(fi.(*sql.NullString)))
			if !sqlValue.Valid {
				return errors.New("sqlValue is invalid")
			}
			timeStr := sqlValue.String
			var layout string
			if len(timeStr) == 10 {
				layout = shortSimpleTimeFormat
			}
			if len(timeStr) == 19 {
				layout = longSimpleTimeFormat
			}
			timeTime, err := time.ParseInLocation(layout, timeStr, time.Local)
			if timeTime.IsZero() {
				return err
			}
			fv.Set(reflect.ValueOf(timeTime))
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package storage

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"

	"utils"

	"github.com/golang/protobuf/proto"
)

// AOF ...
type AOF struct {
	file   *os.File
	buffer *bufio.ReadWriter
	lock   sync.RWMutex
	in     chan *Entry
}

// NewAOF ...
func NewAOF(path string) *AOF {
	file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600)
	utils.PanicOnError(err)
	rdr := bufio.NewReader(file)
	wtr := bufio.NewWriter(file)
	in := make(chan *Entry)
	return &AOF{file, bufio.NewReadWriter(rdr, wtr), sync.RWMutex{}, in}
}

// Run ...
func (aof *AOF) Run() {
	go func() {
		for {
			select {
			case e := <-aof.in:
				aof.write(e)
			}
		}
	}()
}

func (aof *AOF) write(e *Entry) {
	line := fmt.Sprintf("%d|%s/", e.op, string(e.raw))
	if _, err := aof.buffer.WriteString(line); err != nil {
		fmt.Println(err)
	}
	if err := aof.buffer.Flush(); err != nil {
		fmt.Println(err)
	}
}

// Append ...
func (aof *AOF) Append(op uint8, msg proto.Message) error {
	raw, err := proto.Marshal(msg)
	if err != nil {
		return err
	}
	e := &Entry{op, msg, raw}
	aof.in <- e
	return nil
}

// Read ...
func (aof *AOF) Read() (*Entry, error) {
	line, err := aof.buffer.ReadBytes('/')
	if err != nil {
		return nil, err
	}
	line = line[:len(line)-1]
	rs := strings.Split(string(line), "|")
	msg := rs[1]
	op, err := strconv.Atoi(rs[0])
	if err != nil {
		return nil, err
	}
	e := &Entry{uint8(op), nil, []byte(msg)}
	return e, nil
}
<commit_msg>Make the AOF in: channel sized to 100<commit_after>package storage

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"

	"utils"

	"github.com/golang/protobuf/proto"
)

// AOF ...
type AOF struct {
	file   *os.File
	buffer *bufio.ReadWriter
	lock   sync.RWMutex
	in     chan *Entry
}

// NewAOF ...
func NewAOF(path string) *AOF {
	file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600)
	utils.PanicOnError(err)
	rdr := bufio.NewReader(file)
	wtr := bufio.NewWriter(file)
	in := make(chan *Entry, 100)
	return &AOF{file, bufio.NewReadWriter(rdr, wtr), sync.RWMutex{}, in}
}

// Run ...
func (aof *AOF) Run() {
	go func() {
		for {
			select {
			case e := <-aof.in:
				aof.write(e)
			}
		}
	}()
}

func (aof *AOF) write(e *Entry) {
	line := fmt.Sprintf("%d|%s/", e.op, string(e.raw))
	if _, err := aof.buffer.WriteString(line); err != nil {
		fmt.Println(err)
	}
	if err := aof.buffer.Flush(); err != nil {
		fmt.Println(err)
	}
}

// Append ...
func (aof *AOF) Append(op uint8, msg proto.Message) error {
	raw, err := proto.Marshal(msg)
	if err != nil {
		return err
	}
	e := &Entry{op, msg, raw}
	aof.in <- e
	return nil
}

// Read ...
func (aof *AOF) Read() (*Entry, error) {
	line, err := aof.buffer.ReadBytes('/')
	if err != nil {
		return nil, err
	}
	line = line[:len(line)-1]
	rs := strings.Split(string(line), "|")
	msg := rs[1]
	op, err := strconv.Atoi(rs[0])
	if err != nil {
		return nil, err
	}
	e := &Entry{uint8(op), nil, []byte(msg)}
	return e, nil
}
<|endoftext|>


<commit_before>package scraper

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"sort"
	"time"

	"github.com/EVE-Tools/emdr-to-nsq/lib/emds"
	"github.com/EVE-Tools/goesi"
	"github.com/EVE-Tools/goesi/v1"
	"github.com/EVE-Tools/market-streamer/lib/locations/citadels"
	"github.com/EVE-Tools/market-streamer/lib/locations/locationCache"
	"github.com/EVE-Tools/market-streamer/lib/marketTypes"
	"github.com/Sirupsen/logrus"
	"github.com/klauspost/compress/zlib"
	"golang.org/x/net/context"
)

type esiOrder goesiv1.GetMarketsRegionIdOrders200Ok

var esiClient goesi.APIClient
var esiPublicContext context.Context

// Initialize initializes the scraper
func Initialize(clientID string, secretKey string, refreshToken string) {
	httpClient := &http.Client{
		Timeout: time.Duration(time.Second * 10),
	}

	// Requests to citadel's markets are authenticated - we're just using a default key for retrieving public markets
	esiAuthenticator := goesi.NewSSOAuthenticator(
		httpClient,
		clientID,
		secretKey,
		"eveauth-e43://market-streamer",
		[]string{"esi-universe.read_structures.v1",
			"esi-search.search_structures.v1",
			"esi-markets.structure_markets.v1"})

	// Build token source for auto-refreshing tokens
	token := &goesi.CRESTToken{
		AccessToken:  "",
		TokenType:    "Bearer",
		RefreshToken: refreshToken,
		Expiry:       time.Now().AddDate(0, 0, -1),
	}

	esiPublicToken, err := esiAuthenticator.TokenSource(token)
	if err != nil {
		log.Fatalf("Error starting bootstrap ESI client: %v", err)
	}

	esiPublicContext = context.WithValue(context.TODO(), goesi.ContextOAuth2, esiPublicToken)
	esiClient = *goesi.NewAPIClient(httpClient, "Element43/market-streamer (element-43.com)")
}

// ScrapeMarket gets a market from ESI and pushes it to supported backends
func ScrapeMarket(regionID int64) ([]byte, *time.Time, error) {
	// Prepare empty rowsets with all market types
	rowsets := generateRowsetsForRegion(regionID)

	//
	// Fetch public region Orders
	//
	// // First page -> re-schedule
	params := make(map[string]interface{})
	params["page"] = int32(1)

	esiOrdersRegion, response, err := esiClient.V1.MarketApi.GetMarketsRegionIdOrders("all", int32(regionID), params)
	if err != nil {
		return nil, nil, err
	}

	expiry, err := time.Parse(time.RFC1123, response.Header.Get("expires"))
	if err != nil {
		// Will run in 10 minutes, anyway
		logrus.WithError(err).Warn("Could not parse ESI expires timestamp!")
	}

	// Re-schedule self with 5 second safety margin
	runAgain := expiry.Add(time.Second * 5)

	// Add orders to rowset
	err = appendResponseRegion(rowsets, esiOrdersRegion, response)
	if err != nil {
		return nil, nil, err
	}

	// Fetch all other pages
	for len(esiOrdersRegion) > 0 {
		params["page"] = params["page"].(int32) + 1
		esiOrdersRegion, response, err = esiClient.V1.MarketApi.GetMarketsRegionIdOrders("all", int32(regionID), params)
		if err != nil {
			return nil, nil, err
		}

		// Add orders to rowset
		err = appendResponseRegion(rowsets, esiOrdersRegion, response)
		if err != nil {
			return nil, nil, err
		}
	}

	//
	// Fetch orders in citadels
	//
	citadelIDs := citadels.GetCitadelsInRegion(regionID)

	for _, citadelID := range citadelIDs {
		params["page"] = int32(1)
		esiOrdersCitadel, response, err := esiClient.V1.MarketApi.GetMarketsStructuresStructureId(esiPublicContext, citadelID, params)
		if err != nil {
			// Blacklist and skip these citadels
			if (response != nil) && (response.StatusCode == 403) {
				citadels.BlacklistCitadel(citadelID)
				continue
			}
			return nil, nil, err
		}

		// Add orders to rowset
		err = appendResponseCitadel(rowsets, esiOrdersCitadel, response)
		if err != nil {
			return nil, nil, err
		}

		// Fetch all other pages
		for len(esiOrdersCitadel) > 0 {
			params["page"] = params["page"].(int32) + 1
			esiOrdersCitadel, response, err = esiClient.V1.MarketApi.GetMarketsStructuresStructureId(esiPublicContext, citadelID, params)
			if err != nil {
				return nil, nil, err
			}

			// Add orders to rowset
			err = appendResponseCitadel(rowsets, esiOrdersCitadel, response)
			if err != nil {
				return nil, nil, err
			}
		}
	}

	// Set generatedAt, sort slices within rowsets and deduplicate orders
	for _, rowset := range rowsets {
		// Sort
		sort.Sort(emds.ByOrderID(rowset.Rows))

		if len(rowset.Rows) > 0 {
			rowset.GeneratedAt = rowset.Rows[0].GeneratedAt
		}

		// Dedup by ID
		inititalLength := len(rowset.Rows)
		deduplicated := rowset.Rows[:0]
		numRemoved := 0
		for index, row := range rowset.Rows {
			if index > 0 && (rowset.Rows[index-1].OrderID == row.OrderID) {
				numRemoved++
				logrus.WithField("order", fmt.Sprintf("%+v", rowset.Rows[index-1])).Debug("A: ")
				logrus.WithField("order", fmt.Sprintf("%+v", rowset.Rows[index])).Debug("B: ")
			} else {
				deduplicated = append(deduplicated, row)
			}
		}

		rowset.Rows = deduplicated

		if numRemoved > 0 {
			logrus.WithFields(logrus.Fields{
				"numDuplicates": numRemoved,
				"regionID":      rowset.RegionID,
				"typeID":        rowset.TypeID,
				"lengthBefore":  inititalLength,
				"lengthAfter":   len(rowset.Rows),
			}).Debug("Removed duplicate orders.")
		}
	}

	// Serialize rowsets
	rowsetSlice := make([]emds.Rowset, len(rowsets))
	rowsetIndex := 0
	numOrders := 0
	for _, rowset := range rowsets {
		numOrders += len(rowset.Rows)
		rowsetSlice[rowsetIndex] = *rowset
		rowsetIndex++
	}

	rowsetJSON, err := emds.RowsetsToUUDIF(rowsetSlice, "Element43/market-streamer", "0.1")
	if err != nil {
		return nil, nil, err
	}

	var compressedJSONBuffer bytes.Buffer
	compressionWriter := zlib.NewWriter(&compressedJSONBuffer)
	_, err = compressionWriter.Write(rowsetJSON)
	if err != nil {
		return nil, nil, err
	}

	err = compressionWriter.Close()
	if err != nil {
		return nil, nil, err
	}

	compressedJSON := compressedJSONBuffer.Bytes()

	logrus.WithFields(logrus.Fields{
		"regionID":          regionID,
		"numOrders":         numOrders,
		"bytesUncompressed": len(rowsetJSON),
		"bytesCompressed":   len(compressedJSON),
	}).Info("Uploading market.")

	return compressedJSON, &runAgain, nil
}

// Type conversion for regions
func appendResponseRegion(rowsets map[int64]*emds.Rowset, regionOrders []goesiv1.GetMarketsRegionIdOrders200Ok, response *http.Response) error {
	var orders []esiOrder

	for _, regionOrder := range regionOrders {
		orders = append(orders, esiOrder(regionOrder))
	}

	return appendResponse(rowsets, orders, response)
}

// Type conversion for citadels
func appendResponseCitadel(rowsets map[int64]*emds.Rowset, citadelOrders []goesiv1.GetMarketsStructuresStructureId200Ok, response *http.Response) error {
	var orders []esiOrder

	for _, citadelOrder := range citadelOrders {
		orders = append(orders, esiOrder(citadelOrder))
	}

	return appendResponse(rowsets, orders, response)
}

func appendResponse(rowsets map[int64]*emds.Rowset, esiOrders []esiOrder, response *http.Response) error {
	lastModified, err := time.Parse(time.RFC1123, response.Header.Get("last-modified"))
	if err != nil {
		// Default to now
		logrus.WithError(err).Warn("Could not parse ESI last-modified timestamp!")
		lastModified = time.Now()
	}

	generatedAt := lastModified.Format(time.RFC3339)

	return appendOrders(rowsets, esiOrders, generatedAt)
}

func appendOrders(rowsets map[int64]*emds.Rowset, esiOrders []esiOrder, generatedAt string) error {
	// Collect locations
	var locationIDs []int64
	for _, order := range esiOrders {
		locationIDs = append(locationIDs, order.LocationId)
	}

	locations, err := locationCache.GetLocations(locationIDs)
	if err != nil {
		return err
	}

	// Add orders including location info
	for _, order := range esiOrders {
		if location, ok := locations[order.LocationId]; ok {
			typeID := int64(order.TypeId)

			orderRange, err := emds.ConvertRange(order.Range_)
			if err != nil {
				logrus.WithError(err).Error("Could not parse range! Skipping order.")
				continue
			}

			// Create rowset for types which should not be there
			if _, ok := rowsets[typeID]; !ok {
				logrus.WithField("typeID", typeID).WithField("order", fmt.Sprintf("%+v", order)).Debug("Type not in marketTypes but in orders!")

				rowsets[typeID] = &emds.Rowset{
					GeneratedAt: generatedAt,
					RegionID:    location.Region.ID,
					TypeID:      typeID,
				}
			}

			rowset := rowsets[typeID]
			rowset.Rows = append(rowset.Rows, emds.Order{
				OrderID:       order.OrderId,
				RegionID:      rowset.RegionID,
				TypeID:        int64(order.TypeId),
				GeneratedAt:   generatedAt,
				Price:         float64(order.Price),
				VolRemaining:  int64(order.VolumeRemain),
				OrderRange:    orderRange,
				VolEntered:    int64(order.VolumeTotal),
				MinVolume:     int64(order.MinVolume),
				Bid:           order.IsBuyOrder,
				IssueDate:     order.Issued.Format(time.RFC3339),
				Duration:      int64(order.Duration),
				StationID:     order.LocationId,
				SolarSystemID: location.SolarSystem.ID,
			})

		} else {
			logrus.WithField("locationID", order.LocationId).Warn("Unknown location.")
		}
	}

	return nil
}

// Generates empty rowsets for population by scraper
func generateRowsetsForRegion(regionID int64) map[int64]*emds.Rowset {
	rowsets := map[int64]*emds.Rowset{}
	now := time.Now().Format(time.RFC3339)
	types := marketTypes.GetMarketTypes()

	for _, typeID := range types {
		rowsets[typeID] = &emds.Rowset{
			GeneratedAt: now,
			RegionID:    regionID,
			TypeID:      typeID,
		}
	}

	return rowsets
}
<commit_msg>Add simple backoff for markets expired in the past<commit_after>package scraper

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"sort"
	"time"

	"github.com/EVE-Tools/emdr-to-nsq/lib/emds"
	"github.com/EVE-Tools/goesi"
	"github.com/EVE-Tools/goesi/v1"
	"github.com/EVE-Tools/market-streamer/lib/locations/citadels"
	"github.com/EVE-Tools/market-streamer/lib/locations/locationCache"
	"github.com/EVE-Tools/market-streamer/lib/marketTypes"
	"github.com/Sirupsen/logrus"
	"github.com/klauspost/compress/zlib"
	"golang.org/x/net/context"
)

type esiOrder goesiv1.GetMarketsRegionIdOrders200Ok

var esiClient goesi.APIClient
var esiPublicContext context.Context

// Initialize initializes the scraper
func Initialize(clientID string, secretKey string, refreshToken string) {
	httpClient := &http.Client{
		Timeout: time.Duration(time.Second * 10),
	}

	// Requests to citadel's markets are authenticated - we're just using a default key for retrieving public markets
	esiAuthenticator := goesi.NewSSOAuthenticator(
		httpClient,
		clientID,
		secretKey,
		"eveauth-e43://market-streamer",
		[]string{"esi-universe.read_structures.v1",
			"esi-search.search_structures.v1",
			"esi-markets.structure_markets.v1"})

	// Build token source for auto-refreshing tokens
	token := &goesi.CRESTToken{
		AccessToken:  "",
		TokenType:    "Bearer",
		RefreshToken: refreshToken,
		Expiry:       time.Now().AddDate(0, 0, -1),
	}

	esiPublicToken, err := esiAuthenticator.TokenSource(token)
	if err != nil {
		log.Fatalf("Error starting bootstrap ESI client: %v", err)
	}

	esiPublicContext = context.WithValue(context.TODO(), goesi.ContextOAuth2, esiPublicToken)
	esiClient = *goesi.NewAPIClient(httpClient, "Element43/market-streamer (element-43.com)")
}

// ScrapeMarket gets a market from ESI and pushes it to supported backends
func ScrapeMarket(regionID int64) ([]byte, *time.Time, error) {
	// Prepare empty rowsets with all market types
	rowsets := generateRowsetsForRegion(regionID)

	//
	// Fetch public region Orders
	//
	// // First page -> re-schedule
	params := make(map[string]interface{})
	params["page"] = int32(1)

	esiOrdersRegion, response, err := esiClient.V1.MarketApi.GetMarketsRegionIdOrders("all", int32(regionID), params)
	if err != nil {
		return nil, nil, err
	}

	expiry, err := time.Parse(time.RFC1123, response.Header.Get("expires"))
	if err != nil {
		// Will run in 10 minutes, anyway
		logrus.WithError(err).Warn("Could not parse ESI expires timestamp!")
	}

	// If expired in the past check back in five seconds (see next line) as the CDN might take some time to refresh
	if expiry.Before(time.Now()) {
		expiry = time.Now()
	}

	// Re-schedule self with 5 second safety margin
	runAgain := expiry.Add(time.Second * 5)

	// Add orders to rowset
	err = appendResponseRegion(rowsets, esiOrdersRegion, response)
	if err != nil {
		return nil, nil, err
	}

	// Fetch all other pages
	for len(esiOrdersRegion) > 0 {
		params["page"] = params["page"].(int32) + 1
		esiOrdersRegion, response, err = esiClient.V1.MarketApi.GetMarketsRegionIdOrders("all", int32(regionID), params)
		if err != nil {
			return nil, nil, err
		}

		// Add orders to rowset
		err = appendResponseRegion(rowsets, esiOrdersRegion, response)
		if err != nil {
			return nil, nil, err
		}
	}

	//
	// Fetch orders in citadels
	//
	citadelIDs := citadels.GetCitadelsInRegion(regionID)

	for _, citadelID := range citadelIDs {
		params["page"] = int32(1)
		esiOrdersCitadel, response, err := esiClient.V1.MarketApi.GetMarketsStructuresStructureId(esiPublicContext, citadelID, params)
		if err != nil {
			// Blacklist and skip these citadels
			if (response != nil) && (response.StatusCode == 403) {
				citadels.BlacklistCitadel(citadelID)
				continue
			}
			return nil, nil, err
		}

		// Add orders to rowset
		err = appendResponseCitadel(rowsets, esiOrdersCitadel, response)
		if err != nil {
			return nil, nil, err
		}

		// Fetch all other pages
		for len(esiOrdersCitadel) > 0 {
			params["page"] = params["page"].(int32) + 1
			esiOrdersCitadel, response, err = esiClient.V1.MarketApi.GetMarketsStructuresStructureId(esiPublicContext, citadelID, params)
			if err != nil {
				return nil, nil, err
			}

			// Add orders to rowset
			err = appendResponseCitadel(rowsets, esiOrdersCitadel, response)
			if err != nil {
				return nil, nil, err
			}
		}
	}

	// Set generatedAt, sort slices within rowsets and deduplicate orders
	for _, rowset := range rowsets {
		// Sort
		sort.Sort(emds.ByOrderID(rowset.Rows))

		if len(rowset.Rows) > 0 {
			rowset.GeneratedAt = rowset.Rows[0].GeneratedAt
		}

		// Dedup by ID
		inititalLength := len(rowset.Rows)
		deduplicated := rowset.Rows[:0]
		numRemoved := 0
		for index, row := range rowset.Rows {
			if index > 0 && (rowset.Rows[index-1].OrderID == row.OrderID) {
				numRemoved++
				logrus.WithField("order", fmt.Sprintf("%+v", rowset.Rows[index-1])).Debug("A: ")
				logrus.WithField("order", fmt.Sprintf("%+v", rowset.Rows[index])).Debug("B: ")
			} else {
				deduplicated = append(deduplicated, row)
			}
		}

		rowset.Rows = deduplicated

		if numRemoved > 0 {
			logrus.WithFields(logrus.Fields{
				"numDuplicates": numRemoved,
				"regionID":      rowset.RegionID,
				"typeID":        rowset.TypeID,
				"lengthBefore":  inititalLength,
				"lengthAfter":   len(rowset.Rows),
			}).Debug("Removed duplicate orders.")
		}
	}

	// Serialize rowsets
	rowsetSlice := make([]emds.Rowset, len(rowsets))
	rowsetIndex := 0
	numOrders := 0
	for _, rowset := range rowsets {
		numOrders += len(rowset.Rows)
		rowsetSlice[rowsetIndex] = *rowset
		rowsetIndex++
	}

	rowsetJSON, err := emds.RowsetsToUUDIF(rowsetSlice, "Element43/market-streamer", "0.1")
	if err != nil {
		return nil, nil, err
	}

	var compressedJSONBuffer bytes.Buffer
	compressionWriter := zlib.NewWriter(&compressedJSONBuffer)
	_, err = compressionWriter.Write(rowsetJSON)
	if err != nil {
		return nil, nil, err
	}

	err = compressionWriter.Close()
	if err != nil {
		return nil, nil, err
	}

	compressedJSON := compressedJSONBuffer.Bytes()

	logrus.WithFields(logrus.Fields{
		"regionID":          regionID,
		"numOrders":         numOrders,
		"bytesUncompressed": len(rowsetJSON),
		"bytesCompressed":   len(compressedJSON),
	}).Info("Uploading market.")

	return compressedJSON, &runAgain, nil
}

// Type conversion for regions
func appendResponseRegion(rowsets map[int64]*emds.Rowset, regionOrders []goesiv1.GetMarketsRegionIdOrders200Ok, response *http.Response) error {
	var orders []esiOrder

	for _, regionOrder := range regionOrders {
		orders = append(orders, esiOrder(regionOrder))
	}

	return appendResponse(rowsets, orders, response)
}

// Type conversion for citadels
func appendResponseCitadel(rowsets map[int64]*emds.Rowset, citadelOrders []goesiv1.GetMarketsStructuresStructureId200Ok, response *http.Response) error {
	var orders []esiOrder

	for _, citadelOrder := range citadelOrders {
		orders = append(orders, esiOrder(citadelOrder))
	}

	return appendResponse(rowsets, orders, response)
}

func appendResponse(rowsets map[int64]*emds.Rowset, esiOrders []esiOrder, response *http.Response) error {
	lastModified, err := time.Parse(time.RFC1123, response.Header.Get("last-modified"))
	if err != nil {
		// Default to now
		logrus.WithError(err).Warn("Could not parse ESI last-modified timestamp!")
		lastModified = time.Now()
	}

	generatedAt := lastModified.Format(time.RFC3339)

	return appendOrders(rowsets, esiOrders, generatedAt)
}

func appendOrders(rowsets map[int64]*emds.Rowset, esiOrders []esiOrder, generatedAt string) error {
	// Collect locations
	var locationIDs []int64
	for _, order := range esiOrders {
		locationIDs = append(locationIDs, order.LocationId)
	}

	locations, err := locationCache.GetLocations(locationIDs)
	if err != nil {
		return err
	}

	// Add orders including location info
	for _, order := range esiOrders {
		if location, ok := locations[order.LocationId]; ok {
			typeID := int64(order.TypeId)

			orderRange, err := emds.ConvertRange(order.Range_)
			if err != nil {
				logrus.WithError(err).Error("Could not parse range! Skipping order.")
				continue
			}

			// Create rowset for types which should not be there
			if _, ok := rowsets[typeID]; !ok {
				logrus.WithField("typeID", typeID).WithField("order", fmt.Sprintf("%+v", order)).Debug("Type not in marketTypes but in orders!")

				rowsets[typeID] = &emds.Rowset{
					GeneratedAt: generatedAt,
					RegionID:    location.Region.ID,
					TypeID:      typeID,
				}
			}

			rowset := rowsets[typeID]
			rowset.Rows = append(rowset.Rows, emds.Order{
				OrderID:       order.OrderId,
				RegionID:      rowset.RegionID,
				TypeID:        int64(order.TypeId),
				GeneratedAt:   generatedAt,
				Price:         float64(order.Price),
				VolRemaining:  int64(order.VolumeRemain),
				OrderRange:    orderRange,
				VolEntered:    int64(order.VolumeTotal),
				MinVolume:     int64(order.MinVolume),
				Bid:           order.IsBuyOrder,
				IssueDate:     order.Issued.Format(time.RFC3339),
				Duration:      int64(order.Duration),
				StationID:     order.LocationId,
				SolarSystemID: location.SolarSystem.ID,
			})

		} else {
			logrus.WithField("locationID", order.LocationId).Warn("Unknown location.")
		}
	}

	return nil
}

// Generates empty rowsets for population by scraper
func generateRowsetsForRegion(regionID int64) map[int64]*emds.Rowset {
	rowsets := map[int64]*emds.Rowset{}
	now := time.Now().Format(time.RFC3339)
	types := marketTypes.GetMarketTypes()

	for _, typeID := range types {
		rowsets[typeID] = &emds.Rowset{
			GeneratedAt: now,
			RegionID:    regionID,
			TypeID:      typeID,
		}
	}

	return rowsets
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

func main() {
	start := time.Now()
	// 処理
	var i float64
	i = 0
	for {
		i++
		end := time.Now()
		diff := float64(end.Sub(start) / 1000 / 1000 / 1000)
		m := int(i / diff)
		out := 0
		if m < 0 {
		} else {
			out = m
		}
		fmt.Printf("\r")
		fmt.Printf("\r%v回試行  %v秒経過 %v回/秒", i, diff, out)
		if submain() {
			break
		}
	}
	fmt.Printf("\n")
}

func submain() bool {
	// 136枚の中から14枚返したい
	list := shuffled_cards()

	// 出力
	result := solve(list)
	string_output(list)
	return result
}

// http://d.hatena.ne.jp/hake/20150930/p1
func shuffle(list []int) {
	for i := len(list); i > 1; i-- {
		j := rand.Intn(i) // 0　.. i-1 の乱数発生
		list[i-1], list[j] = list[j], list[i-1]
	}
}

func shuffled_cards() []int {
	rand.Seed(time.Now().UnixNano())

	// データ要素数指定、および初期データ作成
	size := 136
	list := make([]int, size, size)
	for i := 0; i < size; i++ {
		list[i] = i / 4
	}

	// シャッフル
	shuffle(list)

	return list[:14]
}

// 牌の出力
func string_output(list []int) {
	// http://qiita.com/ruiu/items/2bb83b29baeae2433a79
	// サイズ0、内部バッファの長さ69の[]byteの値を割り当てる
	b := make([]byte, 0, 70)

	// bに文字列を追加
	for j := 0; j < 14; j++ {
		// 126976 is 'ton'
		// https://codepoints.net/U+1F000
		b = append(b, string(list[j]+126976)...) // ...が必要
		b = append(b, string(32)...)             // ...が必要
	}
	fmt.Printf("%v", string(b))
}

// solve
// 字マンソーピンのリストをつくる
func solve(list []int) bool {
	if is_chitoitsu(list) {
		return true
	}
	matrix := [][]int{{}, {}, {}, {}}
	for _, value := range list {
		group(matrix, value)
	}
	//fmt.Printf("%v", matrix)

	return group_scan(matrix)
}

// スート分類してくれる
func group(m [][]int, i int) {
	switch {
	case i < 7:
		m[0] = append(m[0], i)
	case i < 7+(9*1):
		m[1] = append(m[1], i-7)
	case i < 7+(9*2):
		m[2] = append(m[2], i-7-(9*1))
	case i < 7+(9*3):
		m[3] = append(m[3], i-7-(9*2))
	}
}

func group_scan(m [][]int) bool {
	if !valid_mod3(m) {
		return false
	}
	if !valid_33332(m) {
		return false
	}
	return true
}

func valid_mod3(m [][]int) bool {
	//スートのサイズを3で割った時
	//あまりが2であるすーとグループが1つであること
	c := 0
	for _, a := range m {
		switch len(a) % 3 {
		case 0:
			// noop
		case 1:
			return false
		case 2:
			c++
		}
	}
	return c == 1
}

func valid_33332(m [][]int) bool {
	for i := 0; i < 4; i++ {
		if !valid_suit_group(m[i], i) {
			return false
		}
	}
	return true
}

func valid_suit_group(a []int, i int) bool {
	// 第二引数は字牌のとき0

	//ソート
	sort.Ints(a)
	if len(a)%3 == 2 {
		//ペアを探す
		pair_numbers := pairable_numbers(a)
		//ペア候補がなかったらぬける
		if len(pair_numbers) == 0 {
			return false
		}
		//ペア候補毎に繰り返し処理
		for _, v := range pair_numbers {
			//ペアとなる２枚を除去
			rest := []int{}
			c := 2
			for _, w := range a {
				// ペア候補以外は新スライスに入れる
				// ペア候補は３枚目以降は新スライスに入れる
				if w != v || c <= 0 {
					rest = append(rest, w)
				}
				if w == v {
					c--
				}
			}
			if valid_3cards(rest, i) {
				return true
			} else {
				continue
			}
		}
		return false
	} else if len(a)%3 == 0 {
		return valid_3cards(a, i)
	}
	// 到達しないはず
	panic("到達しないはず")
}

func valid_3cards(a []int, i int) bool {
	// 刻子や順子のみで構成されている場合true
	// a is sorted
	// a.size % 3 is0
	// 第二引数は字牌のとき0
	ok := false
	for {
		a, ok = remove_kotsu(a)
		if ok {
			continue
		}
		if i > 0 {
			a, ok = remove_shuntsu(a)
			if ok {
				continue
			}
		}
		return len(a) == 0
	}
}

func remove_kotsu(a []int) ([]int, bool) {
	// 刻子を除去できればtrue
	// a is sorted
	retval := a
	if len(a) < 3 {
		return retval, false
	}
	if a[0] == a[1] && a[0] == a[2] {
		retval = a[3:]
		return retval, true
	}
	return retval, false
}

func remove_shuntsu(a []int) ([]int, bool) {
	// 順子を除去できればtrue
	// a is sorted
	rest := []int{}
	first := -1
	second := -1
	found := false
	for _, v := range a {
		if found {
			rest = append(rest, v)
			continue
		}
		if first == -1 {
			first = v
		} else if second == -1 && first+1 == v {
			second = v
		} else if second != -1 && first+2 == v {
			//flush
			first = -1
			second = -1
			found = true
		} else {
			rest = append(rest, v)
		}
	}
	return rest, found
}

func pairable_numbers(sorted []int) []int {
	//カウンタ
	retval := []int{}
	a := 999 // 2つ前
	b := 999 // 1つ前
	for _, v := range sorted {
		if b == v && a != v {
			retval = append(retval, v)
		} else {
			b = v
		}
	}
	return retval
}

func is_chitoitsu(list []int) bool {
	//カウンタ
	c := map[int]int{}

	//コピー
	l := list

	for _, v := range l {
		count, ok := c[v]
		if ok {
			if count == 1 {
				c[v] = 2
			} else {
				// c[v] == 2
				return false
			}
		} else {
			c[v] = 1
		}
		//8個チェック
		if len(c) >= 8 {
			return false
		}
	}
	return true
}
<commit_msg>Refactor ひどい関数名<commit_after>package main

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

func main() {
	start := time.Now()
	// 処理
	var i float64
	i = 0
	for {
		i++
		end := time.Now()
		diff := float64(end.Sub(start) / 1000 / 1000 / 1000)
		m := int(i / diff)
		out := 0
		if m < 0 {
		} else {
			out = m
		}
		fmt.Printf("\r")
		fmt.Printf("\r%v回試行  %v秒経過 %v回/秒", i, diff, out)
		if try_once() {
			break
		}
	}
	fmt.Printf("\n")
}

func try_once() bool {
	// 136枚の中から14枚返したい
	list := shuffled_cards()

	// 出力
	result := solve(list)
	string_output(list)
	return result
}

// http://d.hatena.ne.jp/hake/20150930/p1
func shuffle(list []int) {
	for i := len(list); i > 1; i-- {
		j := rand.Intn(i) // 0　.. i-1 の乱数発生
		list[i-1], list[j] = list[j], list[i-1]
	}
}

func shuffled_cards() []int {
	rand.Seed(time.Now().UnixNano())

	// データ要素数指定、および初期データ作成
	size := 136
	list := make([]int, size, size)
	for i := 0; i < size; i++ {
		list[i] = i / 4
	}

	// シャッフル
	shuffle(list)

	return list[:14]
}

// 牌の出力
func string_output(list []int) {
	// http://qiita.com/ruiu/items/2bb83b29baeae2433a79
	// サイズ0、内部バッファの長さ69の[]byteの値を割り当てる
	b := make([]byte, 0, 70)

	// bに文字列を追加
	for j := 0; j < 14; j++ {
		// 126976 is 'ton'
		// https://codepoints.net/U+1F000
		b = append(b, string(list[j]+126976)...) // ...が必要
		b = append(b, string(32)...)             // ...が必要
	}
	fmt.Printf("%v", string(b))
}

// solve
// 字マンソーピンのリストをつくる
func solve(list []int) bool {
	if is_chitoitsu(list) {
		return true
	}
	matrix := [][]int{{}, {}, {}, {}}
	for _, value := range list {
		group(matrix, value)
	}
	//fmt.Printf("%v", matrix)

	return group_scan(matrix)
}

// スート分類してくれる
func group(m [][]int, i int) {
	switch {
	case i < 7:
		m[0] = append(m[0], i)
	case i < 7+(9*1):
		m[1] = append(m[1], i-7)
	case i < 7+(9*2):
		m[2] = append(m[2], i-7-(9*1))
	case i < 7+(9*3):
		m[3] = append(m[3], i-7-(9*2))
	}
}

func group_scan(m [][]int) bool {
	if !valid_mod3(m) {
		return false
	}
	if !valid_33332(m) {
		return false
	}
	return true
}

func valid_mod3(m [][]int) bool {
	//スートのサイズを3で割った時
	//あまりが2であるすーとグループが1つであること
	c := 0
	for _, a := range m {
		switch len(a) % 3 {
		case 0:
			// noop
		case 1:
			return false
		case 2:
			c++
		}
	}
	return c == 1
}

func valid_33332(m [][]int) bool {
	for i := 0; i < 4; i++ {
		if !valid_suit_group(m[i], i) {
			return false
		}
	}
	return true
}

func valid_suit_group(a []int, i int) bool {
	// 第二引数は字牌のとき0

	//ソート
	sort.Ints(a)
	if len(a)%3 == 2 {
		//ペアを探す
		pair_numbers := pairable_numbers(a)
		//ペア候補がなかったらぬける
		if len(pair_numbers) == 0 {
			return false
		}
		//ペア候補毎に繰り返し処理
		for _, v := range pair_numbers {
			//ペアとなる２枚を除去
			rest := []int{}
			c := 2
			for _, w := range a {
				// ペア候補以外は新スライスに入れる
				// ペア候補は３枚目以降は新スライスに入れる
				if w != v || c <= 0 {
					rest = append(rest, w)
				}
				if w == v {
					c--
				}
			}
			if valid_3cards(rest, i) {
				return true
			} else {
				continue
			}
		}
		return false
	} else if len(a)%3 == 0 {
		return valid_3cards(a, i)
	}
	// 到達しないはず
	panic("到達しないはず")
}

func valid_3cards(a []int, i int) bool {
	// 刻子や順子のみで構成されている場合true
	// a is sorted
	// a.size % 3 is0
	// 第二引数は字牌のとき0
	ok := false
	for {
		a, ok = remove_kotsu(a)
		if ok {
			continue
		}
		if i > 0 {
			a, ok = remove_shuntsu(a)
			if ok {
				continue
			}
		}
		return len(a) == 0
	}
}

func remove_kotsu(a []int) ([]int, bool) {
	// 刻子を除去できればtrue
	// a is sorted
	retval := a
	if len(a) < 3 {
		return retval, false
	}
	if a[0] == a[1] && a[0] == a[2] {
		retval = a[3:]
		return retval, true
	}
	return retval, false
}

func remove_shuntsu(a []int) ([]int, bool) {
	// 順子を除去できればtrue
	// a is sorted
	rest := []int{}
	first := -1
	second := -1
	found := false
	for _, v := range a {
		if found {
			rest = append(rest, v)
			continue
		}
		if first == -1 {
			first = v
		} else if second == -1 && first+1 == v {
			second = v
		} else if second != -1 && first+2 == v {
			//flush
			first = -1
			second = -1
			found = true
		} else {
			rest = append(rest, v)
		}
	}
	return rest, found
}

func pairable_numbers(sorted []int) []int {
	//カウンタ
	retval := []int{}
	a := 999 // 2つ前
	b := 999 // 1つ前
	for _, v := range sorted {
		if b == v && a != v {
			retval = append(retval, v)
		} else {
			b = v
		}
	}
	return retval
}

func is_chitoitsu(list []int) bool {
	//カウンタ
	c := map[int]int{}

	//コピー
	l := list

	for _, v := range l {
		count, ok := c[v]
		if ok {
			if count == 1 {
				c[v] = 2
			} else {
				// c[v] == 2
				return false
			}
		} else {
			c[v] = 1
		}
		//8個チェック
		if len(c) >= 8 {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"math/rand"
	"time"
)

type Deck []Card

var suits = []string{"Clubs", "Diamonds", "Hearts", "Spades"}

func NewDeck() Deck {
	d := make(Deck, 0, 52)
	for _, s := range suits {
		d = append(d,
			Card{"Ace", s, 11, 1},
			Card{"Two", s, 2, 2},
			Card{"Three", s, 3, 3},
			Card{"Four", s, 4, 4},
			Card{"Five", s, 5, 5},
			Card{"Six", s, 6, 6},
			Card{"Seven", s, 7, 7},
			Card{"Eight", s, 8, 8},
			Card{"Nine", s, 9, 9},
			Card{"Ten", s, 10, 10},
			Card{"Jack", s, 10, 10},
			Card{"Queen", s, 10, 10},
			Card{"King", s, 10, 10},
		)
	}
	return d
}

func init() {
	rand.Seed(time.Now().UnixNano())
}

//TODO: empty deck
func (deck *Deck) Pull() Card {
	d := *deck
	i := rand.Intn(len(d))
	last := len(d) - 1
	if i != last {
		d[i], d[last] = d[last], d[i]
	}
	card := d[last]
	*deck = d[:last]
	return card
}
<commit_msg>Panic if deck is empty, as it shouldn't ever happen<commit_after>package main

import (
	"errors"
	"math/rand"
	"time"
)

type Deck []Card

var suits = []string{"Clubs", "Diamonds", "Hearts", "Spades"}

func NewDeck() Deck {
	d := make(Deck, 0, 52)
	for _, s := range suits {
		d = append(d,
			Card{"Ace", s, 11, 1},
			Card{"Two", s, 2, 2},
			Card{"Three", s, 3, 3},
			Card{"Four", s, 4, 4},
			Card{"Five", s, 5, 5},
			Card{"Six", s, 6, 6},
			Card{"Seven", s, 7, 7},
			Card{"Eight", s, 8, 8},
			Card{"Nine", s, 9, 9},
			Card{"Ten", s, 10, 10},
			Card{"Jack", s, 10, 10},
			Card{"Queen", s, 10, 10},
			Card{"King", s, 10, 10},
		)
	}
	return d
}

func init() {
	rand.Seed(time.Now().UnixNano())
}

func (deck *Deck) Pull() Card {
	if len(*deck) < 1 {
		panic(errors.New("Run out of cards"))
	}
	d := *deck
	i := rand.Intn(len(d))
	last := len(d) - 1
	if i != last {
		d[i], d[last] = d[last], d[i]
	}
	card := d[last]
	*deck = d[:last]
	return card
}
<|endoftext|>


<commit_before>package blockchain

import (
	"bytes"
	"context"
	"errors"
	"reflect"
    "time"
	"net/http"

	wire "github.com/tendermint/go-wire"
	"github.com/bytom/p2p"
	"github.com/bytom/types"
    "github.com/bytom/protocol/bc/legacy"
    "github.com/bytom/protocol"
	cmn "github.com/tendermint/tmlibs/common"
	"github.com/bytom/blockchain/txdb"
	"github.com/bytom/blockchain/account"
	"github.com/bytom/log"
	//"github.com/bytom/net/http/gzip"
	"github.com/bytom/net/http/httpjson"
	//"github.com/bytom/net/http/limit"
	"github.com/bytom/net/http/static"
	"github.com/bytom/generated/dashboard"
)

const (
	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
	BlockchainChannel = byte(0x40)

	defaultChannelCapacity = 100
	defaultSleepIntervalMS = 500
	trySyncIntervalMS      = 100
	// stop syncing when last block's time is
	// within this much of the system time.
	// stopSyncingDurationMinutes = 10

	// ask for best height every 10s
	statusUpdateIntervalSeconds = 10
	// check if we should switch to consensus reactor
	switchToConsensusIntervalSeconds = 1
	maxBlockchainResponseSize        = types.MaxBlockSize + 2
	crosscoreRPCPrefix = "/rpc/"
)

/*
type consensusReactor interface {
	// for when we switch from blockchain reactor and fast sync to
	// the consensus machine
	SwitchToConsensus(*sm.State)
}
*/

// BlockchainReactor handles long-term catchup syncing.
type BlockchainReactor struct {
	p2p.BaseReactor

//	state        *sm.State
//	proxyAppConn proxy.AppConnConsensus // same as consensus.proxyAppConn
//	store        *MemStore
    chain        *protocol.Chain
	store        *txdb.Store
	accounts	 *account.Manager
	pool         *BlockPool
	mux          *http.ServeMux
	handler      http.Handler
	fastSync     bool
	requestsCh   chan BlockRequest
	timeoutsCh   chan string
//	lastBlock    *types.Block

	evsw types.EventSwitch
}

func jsonHandler(f interface{}) http.Handler {
    h, err := httpjson.Handler(f, errorFormatter.Write)
	if err != nil {
		panic(err)
	}
	return h
}

func alwaysError(err error) http.Handler {
	return jsonHandler(func() error { return err })
}

func (bcr *BlockchainReactor) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
    bcr.handler.ServeHTTP(rw, req)
}

func (bcr *BlockchainReactor) info(ctx context.Context) (map[string]interface{}, error) {
    //if a.config == nil {
		// never configured
	log.Printf(ctx, "-------info-----")
	return map[string]interface{}{
		"is_configured": false,
		"version":       "0.001",
		"build_commit":  "----",
		"build_date":    "------",
		"build_config":  "---------",
	}, nil
	//}
}

func (bcr *BlockchainReactor) createblockkey(ctx context.Context) {
	log.Printf(ctx,"creat-block-key")
}

func webAssetsHandler(next http.Handler) http.Handler {
	mux := http.NewServeMux()
	mux.Handle("/dashboard/", http.StripPrefix("/dashboard/", static.Handler{
		Assets:  dashboard.Files,
		Default: "index.html",
	}))
	mux.Handle("/", next)
	return mux
}

func maxBytes(h http.Handler) http.Handler {
    const maxReqSize = 1e7 // 10MB
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		// A block can easily be bigger than maxReqSize, but everything
		// else should be pretty small.
		if req.URL.Path != crosscoreRPCPrefix+"signer/sign-block" {
			req.Body = http.MaxBytesReader(w, req.Body, maxReqSize)
		}
		h.ServeHTTP(w, req)
	})
}

func (bcr *BlockchainReactor) BuildHander() {
	m := bcr.mux
	m.Handle("/create-account", jsonHandler(bcr.createAccount))
	m.Handle("/", alwaysError(errors.New("not Found")))
	m.Handle("/info", jsonHandler(bcr.info))
	m.Handle("/create-block-key", jsonHandler(bcr.createblockkey))

    latencyHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if l := latency(m, req); l != nil {
			defer l.RecordSince(time.Now())
		m.ServeHTTP(w, req)
		})
	handler := maxBytes(latencyHandler) // TODO(tessr): consider moving this to non-core specific mux
	handler = webAssetsHandler(handler)
/*	handler = healthHandler(handler)
	for _, l := range a.requestLimits {
		handler = limit.Handler(handler, alwaysError(errRateLimited), l.perSecond, l.burst, l.key)
	}
	handler = gzip.Handler{Handler: handler}
	handler = coreCounter(handler)
	handler = timeoutContextHandler(handler)
	if a.config != nil && a.config.BlockchainId != nil {
		handler = blockchainIDHandler(handler, a.config.BlockchainId.String())
	}
	*/
	bcr.handler = handler
}

func NewBlockchainReactor(store *txdb.Store, chain *protocol.Chain, accounts *account.Manager, fastSync bool) *BlockchainReactor {
    requestsCh    := make(chan BlockRequest, defaultChannelCapacity)
    timeoutsCh    := make(chan string, defaultChannelCapacity)
    pool := NewBlockPool(
        store.Height()+1,
        requestsCh,
        timeoutsCh,
    )
    bcR := &BlockchainReactor {
        chain:         chain,
        store:         store,
		accounts:      accounts,
        pool:          pool,
		mux:           http.NewServeMux(),
        fastSync:      fastSync,
        requestsCh:    requestsCh,
        timeoutsCh:   timeoutsCh,
    }
    bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR)
    return bcR
}

// OnStart implements BaseService
func (bcR *BlockchainReactor) OnStart() error {
	bcR.BaseReactor.OnStart()
	bcR.BuildHander()
    if bcR.fastSync {
        _, err := bcR.pool.Start()
        if err != nil {
            return err
        }
        go bcR.poolRoutine()
    }
	return nil
}

// OnStop implements BaseService
func (bcR *BlockchainReactor) OnStop() {
	bcR.BaseReactor.OnStop()
}

// GetChannels implements Reactor
func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
	return []*p2p.ChannelDescriptor{
		&p2p.ChannelDescriptor{
			ID:                BlockchainChannel,
			Priority:          5,
			SendQueueCapacity: 100,
		},
	}
}

// AddPeer implements Reactor by sending our state to peer.
func (bcR *BlockchainReactor) AddPeer(peer *p2p.Peer) {
	if !peer.Send(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) {
		// doing nothing, will try later in `poolRoutine`
	}
}

// RemovePeer implements Reactor by removing peer from the pool.
func (bcR *BlockchainReactor) RemovePeer(peer *p2p.Peer, reason interface{}) {
	bcR.pool.RemovePeer(peer.Key)
}

// Receive implements Reactor by handling 4 types of messages (look below).
func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) {
	_, msg, err := DecodeMessage(msgBytes)
	if err != nil {
		bcR.Logger.Error("Error decoding message", "error", err)
		return
	}

	bcR.Logger.Info("Receive", "src", src, "chID", chID, "msg", msg)

	switch msg := msg.(type) {
	case *bcBlockRequestMessage:
		// Got a request for a block. Respond with block if we have it.
		block, _:= bcR.store.GetBlock(msg.Height)
		if block != nil {
			msg := &bcBlockResponseMessage{Block: block}
			queued := src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{msg})
			if !queued {
				// queue is full, just ignore.
			}
		} else {
			// TODO peer is asking for things we don't have.
		}
	case *bcBlockResponseMessage:
		// Got a block.
		bcR.pool.AddBlock(src.Key, msg.Block, len(msgBytes))
	case *bcStatusRequestMessage:
		// Send peer our state.
		queued := src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}})
		if !queued {
			// sorry
		}
	case *bcStatusResponseMessage:
		// Got a peer status. Unverified.
		bcR.pool.SetPeerHeight(src.Key, msg.Height)
	default:
		bcR.Logger.Error(cmn.Fmt("Unknown message type %v", reflect.TypeOf(msg)))
	}
}

// Handle messages from the poolReactor telling the reactor what to do.
// NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down!
// (Except for the SYNC_LOOP, which is the primary purpose and must be synchronous.)
func (bcR *BlockchainReactor) poolRoutine() {

	trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond)
	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
	//switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second)

FOR_LOOP:
	for {
		select {
		case request := <-bcR.requestsCh: // chan BlockRequest
			peer := bcR.Switch.Peers().Get(request.PeerID)
			if peer == nil {
				continue FOR_LOOP // Peer has since been disconnected.
			}
			msg := &bcBlockRequestMessage{request.Height}
			queued := peer.TrySend(BlockchainChannel, struct{ BlockchainMessage }{msg})
			if !queued {
				// We couldn't make the request, send-queue full.
				// The pool handles timeouts, just let it go.
				continue FOR_LOOP
			}
		case peerID := <-bcR.timeoutsCh: // chan string
			// Peer timed out.
			peer := bcR.Switch.Peers().Get(peerID)
			if peer != nil {
				bcR.Switch.StopPeerForError(peer, errors.New("BlockchainReactor Timeout"))
			}
		case _ = <-statusUpdateTicker.C:
			// ask for status updates
			go bcR.BroadcastStatusRequest()
		/*case _ = <-switchToConsensusTicker.C:
			height, numPending, _ := bcR.pool.GetStatus()
			outbound, inbound, _ := bcR.Switch.NumPeers()
			bcR.Logger.Info("Consensus ticker", "numPending", numPending, "total", len(bcR.pool.requesters),
				"outbound", outbound, "inbound", inbound)
			if bcR.pool.IsCaughtUp() {
				bcR.Logger.Info("Time to switch to consensus reactor!", "height", height)
				bcR.pool.Stop()

				conR := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
				conR.SwitchToConsensus(bcR.state)

				break FOR_LOOP
			}*/
		case _ = <-trySyncTicker.C: // chan time
			// This loop can be slow as long as it's doing syncing work.
		SYNC_LOOP:
			for i := 0; i < 10; i++ {
				// See if there are any blocks to sync.
				first, second := bcR.pool.PeekTwoBlocks()
				bcR.Logger.Info("TrySync peeked", "first", first, "second", second)
				if first == nil || second == nil {
					// We need both to sync the first block.
					break SYNC_LOOP
				}
			    bcR.pool.PopRequest()
                bcR.store.SaveBlock(first)
			}
			continue FOR_LOOP
		case <-bcR.Quit:
			break FOR_LOOP
		}
	}
}

// BroadcastStatusRequest broadcasts `BlockStore` height.
func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
	bcR.Switch.Broadcast(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusRequestMessage{bcR.store.Height()}})
	return nil
}

// SetEventSwitch implements events.Eventable
func (bcR *BlockchainReactor) SetEventSwitch(evsw types.EventSwitch) {
	bcR.evsw = evsw
}

//-----------------------------------------------------------------------------
// Messages

const (
	msgTypeBlockRequest   = byte(0x10)
	msgTypeBlockResponse  = byte(0x11)
	msgTypeStatusResponse = byte(0x20)
	msgTypeStatusRequest  = byte(0x21)
)

// BlockchainMessage is a generic message for this reactor.
type BlockchainMessage interface{}

var _ = wire.RegisterInterface(
	struct{ BlockchainMessage }{},
	wire.ConcreteType{&bcBlockRequestMessage{}, msgTypeBlockRequest},
	wire.ConcreteType{&bcBlockResponseMessage{}, msgTypeBlockResponse},
	wire.ConcreteType{&bcStatusResponseMessage{}, msgTypeStatusResponse},
	wire.ConcreteType{&bcStatusRequestMessage{}, msgTypeStatusRequest},
)

// DecodeMessage decodes BlockchainMessage.
// TODO: ensure that bz is completely read.
func DecodeMessage(bz []byte) (msgType byte, msg BlockchainMessage, err error) {
	msgType = bz[0]
	n := int(0)
	r := bytes.NewReader(bz)
	msg = wire.ReadBinary(struct{ BlockchainMessage }{}, r, maxBlockchainResponseSize, &n, &err).(struct{ BlockchainMessage }).BlockchainMessage
	if err != nil && n != len(bz) {
		err = errors.New("DecodeMessage() had bytes left over")
	}
	return
}

//-------------------------------------

type bcBlockRequestMessage struct {
	Height uint64
}

func (m *bcBlockRequestMessage) String() string {
	return cmn.Fmt("[bcBlockRequestMessage %v]", m.Height)
}

//-------------------------------------

// NOTE: keep up-to-date with maxBlockchainResponseSize
type bcBlockResponseMessage struct {
	Block *legacy.Block
}

func (m *bcBlockResponseMessage) String() string {
	return cmn.Fmt("[bcBlockResponseMessage %v]", m.Block.Height)
}

//-------------------------------------

type bcStatusRequestMessage struct {
	Height uint64
}

func (m *bcStatusRequestMessage) String() string {
	return cmn.Fmt("[bcStatusRequestMessage %v]", m.Height)
}

//-------------------------------------

type bcStatusResponseMessage struct {
	Height uint64
}

func (m *bcStatusResponseMessage) String() string {
	return cmn.Fmt("[bcStatusResponseMessage %v]", m.Height)
}
<commit_msg>Update reactor.go<commit_after>package blockchain

import (
	"bytes"
	"context"
	"errors"
	"reflect"
    "time"
	"net/http"

	wire "github.com/tendermint/go-wire"
	"github.com/bytom/p2p"
	"github.com/bytom/types"
    "github.com/bytom/protocol/bc/legacy"
    "github.com/bytom/protocol"
	cmn "github.com/tendermint/tmlibs/common"
	"github.com/bytom/blockchain/txdb"
	"github.com/bytom/blockchain/account"
	"github.com/bytom/log"
	//"github.com/bytom/net/http/gzip"
	"github.com/bytom/net/http/httpjson"
	//"github.com/bytom/net/http/limit"
	"github.com/bytom/net/http/static"
	"github.com/bytom/generated/dashboard"
)

const (
	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
	BlockchainChannel = byte(0x40)

	defaultChannelCapacity = 100
	defaultSleepIntervalMS = 500
	trySyncIntervalMS      = 100
	// stop syncing when last block's time is
	// within this much of the system time.
	// stopSyncingDurationMinutes = 10

	// ask for best height every 10s
	statusUpdateIntervalSeconds = 10
	// check if we should switch to consensus reactor
	switchToConsensusIntervalSeconds = 1
	maxBlockchainResponseSize        = types.MaxBlockSize + 2
	crosscoreRPCPrefix = "/rpc/"
)

/*
type consensusReactor interface {
	// for when we switch from blockchain reactor and fast sync to
	// the consensus machine
	SwitchToConsensus(*sm.State)
}
*/

// BlockchainReactor handles long-term catchup syncing.
type BlockchainReactor struct {
	p2p.BaseReactor

//	state        *sm.State
//	proxyAppConn proxy.AppConnConsensus // same as consensus.proxyAppConn
//	store        *MemStore
    chain        *protocol.Chain
	store        *txdb.Store
	accounts	 *account.Manager
	pool         *BlockPool
	mux          *http.ServeMux
	handler      http.Handler
	fastSync     bool
	requestsCh   chan BlockRequest
	timeoutsCh   chan string
//	lastBlock    *types.Block

	evsw types.EventSwitch
}

func jsonHandler(f interface{}) http.Handler {
    h, err := httpjson.Handler(f, errorFormatter.Write)
	if err != nil {
		panic(err)
	}
	return h
}

func alwaysError(err error) http.Handler {
	return jsonHandler(func() error { return err })
}

func (bcr *BlockchainReactor) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
    bcr.handler.ServeHTTP(rw, req)
}

func (bcr *BlockchainReactor) info(ctx context.Context) (map[string]interface{}, error) {
    //if a.config == nil {
		// never configured
	log.Printf(ctx, "-------info-----")
	return map[string]interface{}{
		"is_configured": false,
		"version":       "0.001",
		"build_commit":  "----",
		"build_date":    "------",
		"build_config":  "---------",
	}, nil
	//}
}

func (bcr *BlockchainReactor) createblockkey(ctx context.Context) {
	log.Printf(ctx,"creat-block-key")
}

func webAssetsHandler(next http.Handler) http.Handler {
	mux := http.NewServeMux()
	mux.Handle("/dashboard/", http.StripPrefix("/dashboard/", static.Handler{
		Assets:  dashboard.Files,
		Default: "index.html",
	}))
	mux.Handle("/", next)
	return mux
}

func maxBytes(h http.Handler) http.Handler {
    const maxReqSize = 1e7 // 10MB
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		// A block can easily be bigger than maxReqSize, but everything
		// else should be pretty small.
		if req.URL.Path != crosscoreRPCPrefix+"signer/sign-block" {
			req.Body = http.MaxBytesReader(w, req.Body, maxReqSize)
		}
		h.ServeHTTP(w, req)
	})
}

func (bcr *BlockchainReactor) BuildHander() {
	m := bcr.mux
	m.Handle("/create-account", jsonHandler(bcr.createAccount))
	m.Handle("/", alwaysError(errors.New("not Found")))
	m.Handle("/info", jsonHandler(bcr.info))
	m.Handle("/create-block-key", jsonHandler(bcr.createblockkey))

    latencyHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if l := latency(m, req); l != nil {
			defer l.RecordSince(time.Now())
		}
		m.ServeHTTP(w, req)
		})
	handler := maxBytes(latencyHandler) // TODO(tessr): consider moving this to non-core specific mux
	handler = webAssetsHandler(handler)
/*	handler = healthHandler(handler)
	for _, l := range a.requestLimits {
		handler = limit.Handler(handler, alwaysError(errRateLimited), l.perSecond, l.burst, l.key)
	}
	handler = gzip.Handler{Handler: handler}
	handler = coreCounter(handler)
	handler = timeoutContextHandler(handler)
	if a.config != nil && a.config.BlockchainId != nil {
		handler = blockchainIDHandler(handler, a.config.BlockchainId.String())
	}
	*/
	bcr.handler = handler
}

func NewBlockchainReactor(store *txdb.Store, chain *protocol.Chain, accounts *account.Manager, fastSync bool) *BlockchainReactor {
    requestsCh    := make(chan BlockRequest, defaultChannelCapacity)
    timeoutsCh    := make(chan string, defaultChannelCapacity)
    pool := NewBlockPool(
        store.Height()+1,
        requestsCh,
        timeoutsCh,
    )
    bcR := &BlockchainReactor {
        chain:         chain,
        store:         store,
		accounts:      accounts,
        pool:          pool,
		mux:           http.NewServeMux(),
        fastSync:      fastSync,
        requestsCh:    requestsCh,
        timeoutsCh:   timeoutsCh,
    }
    bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR)
    return bcR
}

// OnStart implements BaseService
func (bcR *BlockchainReactor) OnStart() error {
	bcR.BaseReactor.OnStart()
	bcR.BuildHander()
    if bcR.fastSync {
        _, err := bcR.pool.Start()
        if err != nil {
            return err
        }
        go bcR.poolRoutine()
    }
	return nil
}

// OnStop implements BaseService
func (bcR *BlockchainReactor) OnStop() {
	bcR.BaseReactor.OnStop()
}

// GetChannels implements Reactor
func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
	return []*p2p.ChannelDescriptor{
		&p2p.ChannelDescriptor{
			ID:                BlockchainChannel,
			Priority:          5,
			SendQueueCapacity: 100,
		},
	}
}

// AddPeer implements Reactor by sending our state to peer.
func (bcR *BlockchainReactor) AddPeer(peer *p2p.Peer) {
	if !peer.Send(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}}) {
		// doing nothing, will try later in `poolRoutine`
	}
}

// RemovePeer implements Reactor by removing peer from the pool.
func (bcR *BlockchainReactor) RemovePeer(peer *p2p.Peer, reason interface{}) {
	bcR.pool.RemovePeer(peer.Key)
}

// Receive implements Reactor by handling 4 types of messages (look below).
func (bcR *BlockchainReactor) Receive(chID byte, src *p2p.Peer, msgBytes []byte) {
	_, msg, err := DecodeMessage(msgBytes)
	if err != nil {
		bcR.Logger.Error("Error decoding message", "error", err)
		return
	}

	bcR.Logger.Info("Receive", "src", src, "chID", chID, "msg", msg)

	switch msg := msg.(type) {
	case *bcBlockRequestMessage:
		// Got a request for a block. Respond with block if we have it.
		block, _:= bcR.store.GetBlock(msg.Height)
		if block != nil {
			msg := &bcBlockResponseMessage{Block: block}
			queued := src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{msg})
			if !queued {
				// queue is full, just ignore.
			}
		} else {
			// TODO peer is asking for things we don't have.
		}
	case *bcBlockResponseMessage:
		// Got a block.
		bcR.pool.AddBlock(src.Key, msg.Block, len(msgBytes))
	case *bcStatusRequestMessage:
		// Send peer our state.
		queued := src.TrySend(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusResponseMessage{bcR.store.Height()}})
		if !queued {
			// sorry
		}
	case *bcStatusResponseMessage:
		// Got a peer status. Unverified.
		bcR.pool.SetPeerHeight(src.Key, msg.Height)
	default:
		bcR.Logger.Error(cmn.Fmt("Unknown message type %v", reflect.TypeOf(msg)))
	}
}

// Handle messages from the poolReactor telling the reactor what to do.
// NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down!
// (Except for the SYNC_LOOP, which is the primary purpose and must be synchronous.)
func (bcR *BlockchainReactor) poolRoutine() {

	trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond)
	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
	//switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second)

FOR_LOOP:
	for {
		select {
		case request := <-bcR.requestsCh: // chan BlockRequest
			peer := bcR.Switch.Peers().Get(request.PeerID)
			if peer == nil {
				continue FOR_LOOP // Peer has since been disconnected.
			}
			msg := &bcBlockRequestMessage{request.Height}
			queued := peer.TrySend(BlockchainChannel, struct{ BlockchainMessage }{msg})
			if !queued {
				// We couldn't make the request, send-queue full.
				// The pool handles timeouts, just let it go.
				continue FOR_LOOP
			}
		case peerID := <-bcR.timeoutsCh: // chan string
			// Peer timed out.
			peer := bcR.Switch.Peers().Get(peerID)
			if peer != nil {
				bcR.Switch.StopPeerForError(peer, errors.New("BlockchainReactor Timeout"))
			}
		case _ = <-statusUpdateTicker.C:
			// ask for status updates
			go bcR.BroadcastStatusRequest()
		/*case _ = <-switchToConsensusTicker.C:
			height, numPending, _ := bcR.pool.GetStatus()
			outbound, inbound, _ := bcR.Switch.NumPeers()
			bcR.Logger.Info("Consensus ticker", "numPending", numPending, "total", len(bcR.pool.requesters),
				"outbound", outbound, "inbound", inbound)
			if bcR.pool.IsCaughtUp() {
				bcR.Logger.Info("Time to switch to consensus reactor!", "height", height)
				bcR.pool.Stop()

				conR := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
				conR.SwitchToConsensus(bcR.state)

				break FOR_LOOP
			}*/
		case _ = <-trySyncTicker.C: // chan time
			// This loop can be slow as long as it's doing syncing work.
		SYNC_LOOP:
			for i := 0; i < 10; i++ {
				// See if there are any blocks to sync.
				first, second := bcR.pool.PeekTwoBlocks()
				bcR.Logger.Info("TrySync peeked", "first", first, "second", second)
				if first == nil || second == nil {
					// We need both to sync the first block.
					break SYNC_LOOP
				}
			    bcR.pool.PopRequest()
                bcR.store.SaveBlock(first)
			}
			continue FOR_LOOP
		case <-bcR.Quit:
			break FOR_LOOP
		}
	}
}

// BroadcastStatusRequest broadcasts `BlockStore` height.
func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
	bcR.Switch.Broadcast(BlockchainChannel, struct{ BlockchainMessage }{&bcStatusRequestMessage{bcR.store.Height()}})
	return nil
}

// SetEventSwitch implements events.Eventable
func (bcR *BlockchainReactor) SetEventSwitch(evsw types.EventSwitch) {
	bcR.evsw = evsw
}

//-----------------------------------------------------------------------------
// Messages

const (
	msgTypeBlockRequest   = byte(0x10)
	msgTypeBlockResponse  = byte(0x11)
	msgTypeStatusResponse = byte(0x20)
	msgTypeStatusRequest  = byte(0x21)
)

// BlockchainMessage is a generic message for this reactor.
type BlockchainMessage interface{}

var _ = wire.RegisterInterface(
	struct{ BlockchainMessage }{},
	wire.ConcreteType{&bcBlockRequestMessage{}, msgTypeBlockRequest},
	wire.ConcreteType{&bcBlockResponseMessage{}, msgTypeBlockResponse},
	wire.ConcreteType{&bcStatusResponseMessage{}, msgTypeStatusResponse},
	wire.ConcreteType{&bcStatusRequestMessage{}, msgTypeStatusRequest},
)

// DecodeMessage decodes BlockchainMessage.
// TODO: ensure that bz is completely read.
func DecodeMessage(bz []byte) (msgType byte, msg BlockchainMessage, err error) {
	msgType = bz[0]
	n := int(0)
	r := bytes.NewReader(bz)
	msg = wire.ReadBinary(struct{ BlockchainMessage }{}, r, maxBlockchainResponseSize, &n, &err).(struct{ BlockchainMessage }).BlockchainMessage
	if err != nil && n != len(bz) {
		err = errors.New("DecodeMessage() had bytes left over")
	}
	return
}

//-------------------------------------

type bcBlockRequestMessage struct {
	Height uint64
}

func (m *bcBlockRequestMessage) String() string {
	return cmn.Fmt("[bcBlockRequestMessage %v]", m.Height)
}

//-------------------------------------

// NOTE: keep up-to-date with maxBlockchainResponseSize
type bcBlockResponseMessage struct {
	Block *legacy.Block
}

func (m *bcBlockResponseMessage) String() string {
	return cmn.Fmt("[bcBlockResponseMessage %v]", m.Block.Height)
}

//-------------------------------------

type bcStatusRequestMessage struct {
	Height uint64
}

func (m *bcStatusRequestMessage) String() string {
	return cmn.Fmt("[bcStatusRequestMessage %v]", m.Height)
}

//-------------------------------------

type bcStatusResponseMessage struct {
	Height uint64
}

func (m *bcStatusResponseMessage) String() string {
	return cmn.Fmt("[bcStatusResponseMessage %v]", m.Height)
}
<|endoftext|>


<commit_before>package logger

import (
	"github.com/ian-kent/go-log/appenders"
	"github.com/ian-kent/go-log/layout"
	"github.com/ian-kent/go-log/levels"
	"strings"
)

type Logger interface {
	Level() levels.LogLevel
	Name() string
	FullName() string
	Enabled() map[levels.LogLevel]bool
	Appender() Appender
	Children() []Logger
	Parent() Logger
	GetLogger(string) Logger
	SetLevel(levels.LogLevel)
	Log(levels.LogLevel, ...interface{})
}

type logger struct {
	Logger
	level    levels.LogLevel
	name     string
	enabled  map[levels.LogLevel]bool
	appender Appender
	children []Logger
	parent   Logger
}

type Appender interface {
	Write(level levels.LogLevel, message string, args ...interface{})
	SetLayout(layout layout.Layout)
	Layout() layout.Layout
}

func New(name string) Logger {
	l := Logger(&logger{
		level:    levels.DEBUG,
		name:     name,
		enabled:  make(map[levels.LogLevel]bool),
		appender: appenders.Console(),
		children: make([]Logger, 0),
		parent:   nil,
	})
	l.SetLevel(levels.DEBUG)
	return l
}

func unwrap(args ...interface{}) []interface{} {
	head := args[0]
	switch head.(type) {
	case func() (string, []interface{}):
		msg, args := head.(func() (string, []interface{}))()
		args = unwrap(args...)
		return append([]interface{}{msg}, args...)
	case func() []interface{}:
		args = unwrap(head.(func() []interface{})()...)
	case func(...interface{}) []interface{}:
		args = unwrap(head.(func(...interface{}) []interface{})(args[1:]...)...)
	}
	return args
}

func (l *logger) New(name string) Logger {
	lg := Logger(&logger{
		level:    levels.INHERIT,
		name:     name,
		enabled:  make(map[levels.LogLevel]bool),
		appender: nil,
		children: make([]Logger, 0),
		parent:   l,
	})
	l.children = append(l.children, lg)
	return lg
}

func (l *logger) GetLogger(name string) Logger {
	bits := strings.Split(name, ".")

	if l.name == bits[0] {
		if len(bits) == 1 {
			return l
		}

		child := bits[1]
		n := strings.Join(bits[1:], ".")
		for _, c := range l.children {
			if c.Name() == child {
				return c.GetLogger(n)
			}
		}

		lg := l.New(child)
		return lg.GetLogger(n)
	}
	lg := l.New(bits[0])
	return lg.GetLogger(name)
}

func (l *logger) write(level levels.LogLevel, params ...interface{}) {
	a := l.Appender()
	if a != nil {
		a.Write(level, params[0].(string), params[1:]...)
	}
}

func (l *logger) Appender() Appender {
	if a := l.appender; a != nil {
		return a
	}
	if l.parent != nil {
		if a := l.parent.Appender(); a != nil {
			return a
		}
	}
	return nil
}

func (l *logger) Log(level levels.LogLevel, params ...interface{}) {
	if !l.Enabled()[level] {
		return
	}
	l.write(level, unwrap(params...)...)
}

func (l *logger) Level() levels.LogLevel {
	if l.level == levels.INHERIT {
		return l.parent.Level()
	}
	return l.level
}

func (l *logger) Enabled() map[levels.LogLevel]bool {
	if l.level == levels.INHERIT {
		return l.parent.Enabled()
	}
	return l.enabled
}

func (l *logger) Name() string {
	return l.name
}

func (l *logger) FullName() string {
	n := l.name
	if l.parent != nil {
		p := l.parent.FullName()
		if len(p) > 0 {
			n = l.parent.FullName() + "." + n
		}
	}
	return n
}

func (l *logger) SetLevel(level levels.LogLevel) {
	l.level = level
	for k, _ := range levels.LogLevelsToString {
		if k <= level {
			l.enabled[k] = true
		} else {
			l.enabled[k] = false
		}
	}
}

func (l *logger) SetAppender(appender Appender) {
	l.appender = appender
}

func (l *logger) Debug(params ...interface{})   { l.Log(levels.DEBUG, params...) }
func (l *logger) Info(params ...interface{})    { l.Log(levels.INFO, params...) }
func (l *logger) Warn(params ...interface{})    { l.Log(levels.WARN, params...) }
func (l *logger) Error(params ...interface{})   { l.Log(levels.ERROR, params...) }
func (l *logger) Trace(params ...interface{})   { l.Log(levels.TRACE, params...) }
func (l *logger) Printf(params ...interface{})  { l.Log(levels.INFO, params...) }
func (l *logger) Println(params ...interface{}) { l.Log(levels.INFO, params...) }
func (l *logger) Fatalf(params ...interface{})  { l.Log(levels.FATAL, params...) }
<commit_msg>Add missing method<commit_after>package logger

import (
	"github.com/ian-kent/go-log/appenders"
	"github.com/ian-kent/go-log/layout"
	"github.com/ian-kent/go-log/levels"
	"strings"
)

type Logger interface {
	Level() levels.LogLevel
	Name() string
	FullName() string
	Enabled() map[levels.LogLevel]bool
	Appender() Appender
	Children() []Logger
	Parent() Logger
	GetLogger(string) Logger
	SetLevel(levels.LogLevel)
	Log(levels.LogLevel, ...interface{})
        SetAppender(appender Appender)
}

type logger struct {
	Logger
	level    levels.LogLevel
	name     string
	enabled  map[levels.LogLevel]bool
	appender Appender
	children []Logger
	parent   Logger
}

type Appender interface {
	Write(level levels.LogLevel, message string, args ...interface{})
	SetLayout(layout layout.Layout)
	Layout() layout.Layout
}

func New(name string) Logger {
	l := Logger(&logger{
		level:    levels.DEBUG,
		name:     name,
		enabled:  make(map[levels.LogLevel]bool),
		appender: appenders.Console(),
		children: make([]Logger, 0),
		parent:   nil,
	})
	l.SetLevel(levels.DEBUG)
	return l
}

func unwrap(args ...interface{}) []interface{} {
	head := args[0]
	switch head.(type) {
	case func() (string, []interface{}):
		msg, args := head.(func() (string, []interface{}))()
		args = unwrap(args...)
		return append([]interface{}{msg}, args...)
	case func() []interface{}:
		args = unwrap(head.(func() []interface{})()...)
	case func(...interface{}) []interface{}:
		args = unwrap(head.(func(...interface{}) []interface{})(args[1:]...)...)
	}
	return args
}

func (l *logger) New(name string) Logger {
	lg := Logger(&logger{
		level:    levels.INHERIT,
		name:     name,
		enabled:  make(map[levels.LogLevel]bool),
		appender: nil,
		children: make([]Logger, 0),
		parent:   l,
	})
	l.children = append(l.children, lg)
	return lg
}

func (l *logger) GetLogger(name string) Logger {
	bits := strings.Split(name, ".")

	if l.name == bits[0] {
		if len(bits) == 1 {
			return l
		}

		child := bits[1]
		n := strings.Join(bits[1:], ".")
		for _, c := range l.children {
			if c.Name() == child {
				return c.GetLogger(n)
			}
		}

		lg := l.New(child)
		return lg.GetLogger(n)
	}
	lg := l.New(bits[0])
	return lg.GetLogger(name)
}

func (l *logger) write(level levels.LogLevel, params ...interface{}) {
	a := l.Appender()
	if a != nil {
		a.Write(level, params[0].(string), params[1:]...)
	}
}

func (l *logger) Appender() Appender {
	if a := l.appender; a != nil {
		return a
	}
	if l.parent != nil {
		if a := l.parent.Appender(); a != nil {
			return a
		}
	}
	return nil
}

func (l *logger) Log(level levels.LogLevel, params ...interface{}) {
	if !l.Enabled()[level] {
		return
	}
	l.write(level, unwrap(params...)...)
}

func (l *logger) Level() levels.LogLevel {
	if l.level == levels.INHERIT {
		return l.parent.Level()
	}
	return l.level
}

func (l *logger) Enabled() map[levels.LogLevel]bool {
	if l.level == levels.INHERIT {
		return l.parent.Enabled()
	}
	return l.enabled
}

func (l *logger) Name() string {
	return l.name
}

func (l *logger) FullName() string {
	n := l.name
	if l.parent != nil {
		p := l.parent.FullName()
		if len(p) > 0 {
			n = l.parent.FullName() + "." + n
		}
	}
	return n
}

func (l *logger) SetLevel(level levels.LogLevel) {
	l.level = level
	for k, _ := range levels.LogLevelsToString {
		if k <= level {
			l.enabled[k] = true
		} else {
			l.enabled[k] = false
		}
	}
}

func (l *logger) SetAppender(appender Appender) {
	l.appender = appender
}

func (l *logger) Debug(params ...interface{})   { l.Log(levels.DEBUG, params...) }
func (l *logger) Info(params ...interface{})    { l.Log(levels.INFO, params...) }
func (l *logger) Warn(params ...interface{})    { l.Log(levels.WARN, params...) }
func (l *logger) Error(params ...interface{})   { l.Log(levels.ERROR, params...) }
func (l *logger) Trace(params ...interface{})   { l.Log(levels.TRACE, params...) }
func (l *logger) Printf(params ...interface{})  { l.Log(levels.INFO, params...) }
func (l *logger) Println(params ...interface{}) { l.Log(levels.INFO, params...) }
func (l *logger) Fatalf(params ...interface{})  { l.Log(levels.FATAL, params...) }
<|endoftext|>