<commit_before>package gorb

import (
	"database/sql"
	"errors"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/go-gorp/gorp"
)

// Balancer embeds multiple connections to physical db and automatically distributes
// queries with a round-robin scheduling around a master/slave replication.
// Write queries are executed by the Master.
// Read queries(SELECTs) are executed by the slaves.
type Balancer struct {
	*gorp.DbMap   // master
	slaves        []*gorp.DbMap
	count         uint64
	mu            sync.Mutex
	masterCanRead bool
}

// NewBalancer opens a connection to each physical db.
// dataSourceNames must be a semi-comma separated list of DSNs with the first
// one being used as the master and the rest as slaves.
func NewBalancer(driverName string, dialect gorp.Dialect, sources string) (*Balancer, error) {
	conns := strings.Split(sources, ";")
	if len(conns) == 0 {
		return nil, errors.New("empty servers list")

	}
	b := &Balancer{}
	for i, c := range conns {
		s, err := sql.Open(driverName, c)
		if err != nil {
			return nil, err
		}
		mapper := &gorp.DbMap{Db: s, Dialect: dialect}
		if i == 0 { // first is the master
			b.DbMap = mapper
		} else {
			b.slaves = append(b.slaves, mapper)
		}
	}
	if len(b.slaves) == 0 {
		b.slaves = append(b.slaves, b.DbMap)
		b.masterCanRead = true
	}

	return b, nil
}

// MasterCanRead adds the master physical database to the slaves list if read==true
// so that the master can perform WRITE queries AND READ queries .
func (b *Balancer) MasterCanRead(read bool) {
	b.mu.Lock()
	defer b.mu.Unlock()
	if b.masterCanRead && read == false {
		b.slaves = b.slaves[:len(b.slaves)-2]
	}
	if !b.masterCanRead && read == true {
		b.slaves = append(b.slaves, b.DbMap)
	}
	b.masterCanRead = read
}

// Ping verifies if a connection to each physical database is still alive, establishing a connection if necessary.
func (b *Balancer) Ping() error {
	var err, innerErr error
	for _, db := range b.GetAllDbs() {
		innerErr = db.Db.Ping()
		if innerErr != nil {
			err = innerErr
		}
	}
	return err
}

// SetMaxIdleConns sets the maximum number of connections
// If MaxOpenConns is greater than 0 but less than the new MaxIdleConns then the
// new MaxIdleConns will be reduced to match the MaxOpenConns limit
// If n <= 0, no idle connections are retained.
func (b *Balancer) SetMaxIdleConns(n int) {
	for _, db := range b.GetAllDbs() {
		db.Db.SetMaxIdleConns(n)
	}
}

// SetMaxOpenConns sets the maximum number of open connections
// If MaxIdleConns is greater than 0 and the new MaxOpenConns
// is less than MaxIdleConns, then MaxIdleConns will be reduced to match
// the new MaxOpenConns limit. If n <= 0, then there is no limit on the number
// of open connections. The default is 0 (unlimited).
func (b *Balancer) SetMaxOpenConns(n int) {
	for _, db := range b.GetAllDbs() {
		db.Db.SetMaxOpenConns(n)
	}
}

// SetConnMaxLifetime sets the maximum amount of time a connection may be reused.
// Expired connections may be closed lazily before reuse.
// If d <= 0, connections are reused forever.
func (b *Balancer) SetConnMaxLifetime(d time.Duration) {
	for _, db := range b.GetAllDbs() {
		db.Db.SetConnMaxLifetime(d)
	}
}

// Master returns the master database
func (b *Balancer) Master() *gorp.DbMap {
	return b.DbMap
}

// Slave returns one of the slaves databases
func (b *Balancer) Slave() *gorp.DbMap {
	b.mu.Lock()
	b.mu.Unlock()
	return b.slaves[b.slave()]
}

// GetAllDbs returns each underlying physical database,
// the first one is the master
func (b *Balancer) GetAllDbs() []*gorp.DbMap {
	dbs := []*gorp.DbMap{}
	dbs = append(dbs, b.DbMap)
	dbs = append(dbs, b.slaves...)
	return dbs
}

// Close closes all physical databases
func (b *Balancer) Close() error {
	var err, innerErr error
	for _, db := range b.GetAllDbs() {
		innerErr = db.Db.Close()
		if innerErr != nil {
			err = innerErr
		}

	}
	return err
}

func (b *Balancer) slave() int {
	if len(b.slaves) == 1 {
		return 0
	}
	return int((atomic.AddUint64(&b.count, 1) % uint64(len(b.slaves))))
}
<commit_msg>sync.Mutex -> sync.RWMutex<commit_after>package gorb

import (
	"database/sql"
	"errors"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/go-gorp/gorp"
)

// Balancer embeds multiple connections to physical db and automatically distributes
// queries with a round-robin scheduling around a master/slave replication.
// Write queries are executed by the Master.
// Read queries(SELECTs) are executed by the slaves.
type Balancer struct {
	*gorp.DbMap   // master
	slaves        []*gorp.DbMap
	count         uint64
	mu            sync.RWMutex
	masterCanRead bool
}

// NewBalancer opens a connection to each physical db.
// dataSourceNames must be a semi-comma separated list of DSNs with the first
// one being used as the master and the rest as slaves.
func NewBalancer(driverName string, dialect gorp.Dialect, sources string) (*Balancer, error) {
	conns := strings.Split(sources, ";")
	if len(conns) == 0 {
		return nil, errors.New("empty servers list")

	}
	b := &Balancer{}
	for i, c := range conns {
		s, err := sql.Open(driverName, c)
		if err != nil {
			return nil, err
		}
		mapper := &gorp.DbMap{Db: s, Dialect: dialect}
		if i == 0 { // first is the master
			b.DbMap = mapper
		} else {
			b.slaves = append(b.slaves, mapper)
		}
	}
	if len(b.slaves) == 0 {
		b.slaves = append(b.slaves, b.DbMap)
		b.masterCanRead = true
	}

	return b, nil
}

// MasterCanRead adds the master physical database to the slaves list if read==true
// so that the master can perform WRITE queries AND READ queries .
func (b *Balancer) MasterCanRead(read bool) {
	b.mu.Lock()
	defer b.mu.Unlock()
	if b.masterCanRead && read == false {
		b.slaves = b.slaves[:len(b.slaves)-2]
	}
	if !b.masterCanRead && read == true {
		b.slaves = append(b.slaves, b.DbMap)
	}
	b.masterCanRead = read
}

// Ping verifies if a connection to each physical database is still alive, establishing a connection if necessary.
func (b *Balancer) Ping() error {
	var err, innerErr error
	for _, db := range b.GetAllDbs() {
		innerErr = db.Db.Ping()
		if innerErr != nil {
			err = innerErr
		}
	}
	return err
}

// SetMaxIdleConns sets the maximum number of connections
// If MaxOpenConns is greater than 0 but less than the new MaxIdleConns then the
// new MaxIdleConns will be reduced to match the MaxOpenConns limit
// If n <= 0, no idle connections are retained.
func (b *Balancer) SetMaxIdleConns(n int) {
	for _, db := range b.GetAllDbs() {
		db.Db.SetMaxIdleConns(n)
	}
}

// SetMaxOpenConns sets the maximum number of open connections
// If MaxIdleConns is greater than 0 and the new MaxOpenConns
// is less than MaxIdleConns, then MaxIdleConns will be reduced to match
// the new MaxOpenConns limit. If n <= 0, then there is no limit on the number
// of open connections. The default is 0 (unlimited).
func (b *Balancer) SetMaxOpenConns(n int) {
	for _, db := range b.GetAllDbs() {
		db.Db.SetMaxOpenConns(n)
	}
}

// SetConnMaxLifetime sets the maximum amount of time a connection may be reused.
// Expired connections may be closed lazily before reuse.
// If d <= 0, connections are reused forever.
func (b *Balancer) SetConnMaxLifetime(d time.Duration) {
	for _, db := range b.GetAllDbs() {
		db.Db.SetConnMaxLifetime(d)
	}
}

// Master returns the master database
func (b *Balancer) Master() *gorp.DbMap {
	return b.DbMap
}

// Slave returns one of the slaves databases
func (b *Balancer) Slave() *gorp.DbMap {
	b.mu.RLock()
	b.mu.RUnlock()
	return b.slaves[b.slave()]
}

// GetAllDbs returns each underlying physical database,
// the first one is the master
func (b *Balancer) GetAllDbs() []*gorp.DbMap {
	dbs := []*gorp.DbMap{}
	dbs = append(dbs, b.DbMap)
	dbs = append(dbs, b.slaves...)
	return dbs
}

// Close closes all physical databases
func (b *Balancer) Close() error {
	var err, innerErr error
	for _, db := range b.GetAllDbs() {
		innerErr = db.Db.Close()
		if innerErr != nil {
			err = innerErr
		}

	}
	return err
}

func (b *Balancer) slave() int {
	if len(b.slaves) == 1 {
		return 0
	}
	return int((atomic.AddUint64(&b.count, 1) % uint64(len(b.slaves))))
}
<|endoftext|>


<commit_before>package base

import (
	"errors"
	"fmt"
)

var (
	// ErrRecordNotFound record not found error, happens when haven't find any matched data when looking up with a struct
	ErrRecordNotFound = errors.New("record not found")
	// ErrInvalidSQL invalid SQL error, happens when you passed invalid SQL
	ErrInvalidSQL = errors.New("invalid SQL")
	// ErrInvalidTransaction invalid transaction when you are trying to `Commit` or `Rollback`
	ErrInvalidTransaction = errors.New("no valid transaction")
	// ErrCantStartTransaction can't start transaction when you are trying to start one with `Begin`
	ErrCantStartTransaction = errors.New("can't start transaction")
	// ErrUnaddressable unaddressable value
	ErrUnaddressable = errors.New("using unaddressable value")
)

func GetInterfaceAsSQL(value interface{}) (string, error) {
	switch value.(type) {
	case string, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
		return fmt.Sprintf("%v", value), nil
	default:
	}

	return "", ErrInvalidSQL
}
<commit_msg>[base]: Add SQLCommon interface<commit_after>package base

import (
	"database/sql"
	"errors"
	"fmt"
)

var (
	// ErrRecordNotFound record not found error, happens when haven't find any matched data when looking up with a struct
	ErrRecordNotFound = errors.New("record not found")
	// ErrInvalidSQL invalid SQL error, happens when you passed invalid SQL
	ErrInvalidSQL = errors.New("invalid SQL")
	// ErrInvalidTransaction invalid transaction when you are trying to `Commit` or `Rollback`
	ErrInvalidTransaction = errors.New("no valid transaction")
	// ErrCantStartTransaction can't start transaction when you are trying to start one with `Begin`
	ErrCantStartTransaction = errors.New("can't start transaction")
	// ErrUnaddressable unaddressable value
	ErrUnaddressable = errors.New("using unaddressable value")
)

type SQLCommon interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Prepare(query string) (*sql.Stmt, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
}

func GetInterfaceAsSQL(value interface{}) (string, error) {
	switch value.(type) {
	case string, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
		return fmt.Sprintf("%v", value), nil
	default:
	}

	return "", ErrInvalidSQL
}
<|endoftext|>


<commit_before>package main_test

import (
	"testing"

	. "github.com/cloudfoundry-incubator/buildpack_app_lifecycle/Godeps/_workspace/src/github.com/onsi/ginkgo"
	. "github.com/cloudfoundry-incubator/buildpack_app_lifecycle/Godeps/_workspace/src/github.com/onsi/gomega"
	"github.com/cloudfoundry-incubator/buildpack_app_lifecycle/Godeps/_workspace/src/github.com/onsi/gomega/gexec"
)

var launcher string

func TestBuildpackLifecycleLauncher(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Buildpack-Lifecycle-Launcher Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	launcherPath, err := gexec.Build("github.com/cloudfoundry-incubator/buildpack_app_lifecycle/launcher")
	Ω(err).ShouldNot(HaveOccurred())
	return []byte(launcherPath)
}, func(launcherPath []byte) {
	launcher = string(launcherPath)
})

var _ = SynchronizedAfterSuite(func() {
	//noop
}, func() {
	gexec.CleanupBuildArtifacts()
})
<commit_msg>Adding race flag to gexec.Build<commit_after>package main_test

import (
	"testing"

	. "github.com/cloudfoundry-incubator/buildpack_app_lifecycle/Godeps/_workspace/src/github.com/onsi/ginkgo"
	. "github.com/cloudfoundry-incubator/buildpack_app_lifecycle/Godeps/_workspace/src/github.com/onsi/gomega"
	"github.com/cloudfoundry-incubator/buildpack_app_lifecycle/Godeps/_workspace/src/github.com/onsi/gomega/gexec"
)

var launcher string

func TestBuildpackLifecycleLauncher(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Buildpack-Lifecycle-Launcher Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	launcherPath, err := gexec.Build("github.com/cloudfoundry-incubator/buildpack_app_lifecycle/launcher", "-race")
	Ω(err).ShouldNot(HaveOccurred())
	return []byte(launcherPath)
}, func(launcherPath []byte) {
	launcher = string(launcherPath)
})

var _ = SynchronizedAfterSuite(func() {
	//noop
}, func() {
	gexec.CleanupBuildArtifacts()
})
<|endoftext|>


<commit_before>// Copyright 2014 Markus Dittrich. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// pagoda is a go library for command line parsing
package pagoda

import (
  "encoding/json"
  "os"
  "fmt"
  "strconv"
  "unicode/utf8"
)

// parse_spec describes all options and usage actually present on the
// command line and also keeps track of any remaining command line entries
type parsedSpec struct {
  templateSpec
  cmdlOptions jsonOptions

  subcommand_mode bool
  subcommand string
  subcommand_options map[string]jsonOptions

  remainder []string      // unparsed remainder of command line
}

// target_spec describes all possible options and usage per json spec file
type templateSpec struct {
  Usage_info string
  Options jsonOptions   // options according to the spec
}

// json_option describes a single option specification as parsed from the
// JSON description
type jsonOption struct {
  Short_option string
  Long_option string
  Description string
  Type string
  Default *string      // use a pointer to be able to distinguish the empty
                       // string from non-present option
  Subcommand *string
  value interface{}    // option value of type Type
}

type jsonOptions []jsonOption

// Value returns the value known for the given option. Either the
// long or short option can be provided.
// NOTE: The look up could be made more efficient via a map
func (p *parsedSpec) Value(key string) (interface{}, error) {

  for _, opt := range p.cmdlOptions {
    if key == opt.Short_option || key == opt.Long_option {
      return opt.value, nil
    }
  }

  return nil, fmt.Errorf("Pagoda: command line option %s not found", key)
}

// Remainder returns a slice with all command line options that
// were not parsed into options
func (p *parsedSpec) Remainder() []string {
  return p.remainder
}

// Init parses the specification of option in JSON format
func Init(content []byte) (*parsedSpec, error) {

  var parse_info templateSpec
  err := json.Unmarshal(content, &parse_info)
  if err != nil {
    return nil, err
  }

  haveGroupMode := check_for_group_mode(&parse_info)

  err = validate_specs(&parse_info, haveGroupMode)
  if err != nil {
    return nil, err
  }

  err = extract_defaults(&parse_info)
  if err != nil {
    return nil, err
  }

  matched_info, err := match_spec_to_args(&parse_info, os.Args, haveGroupMode)
  if err != nil {
    return nil, err
  }

  // check if help was requested. In that case show Usage() and then exit
  if _, err := matched_info.Value("h"); err == nil {
    if matched_info.subcommand_mode {
      command_usage(matched_info.Usage_info, matched_info.subcommand,
        matched_info.Options, os.Args)
    } else {
      matched_info.Usage()
    }
    os.Exit(0)
  }

  return matched_info, nil
}

// Usage prints the usage information for the package
func (p *parsedSpec) Usage() {

  fmt.Println(p.Usage_info)
  fmt.Println()

  if p.subcommand_mode {
    fmt.Printf("Usage: %s SUBCOMMAND [arguments]\n", os.Args[0])
    fmt.Println("\nAvailable SUBCOMMANDS are:\n")
    for k, _ := range p.subcommand_options {
      fmt.Printf("\t%-10s\n", k)
    }
  } else {
    command_usage(p.Usage_info, "", p.Options, os.Args)
  }

  fmt.Println()
}

// command_usage prints the usage for a specific subcommand
func command_usage(info string, subcommand string, options jsonOptions,
  args []string) {

  fmt.Printf("Usage: %s %s [arguments]\n\n", args[0], subcommand)
  for _, opt := range options {

    if opt.Long_option == "" {
      fmt.Printf("\t-%-15s  %s", opt.Short_option, opt.Description)
    } else if opt.Short_option == "" {
      fmt.Printf("\t    --%-10s  %s", opt.Long_option, opt.Description)
    } else {
      fmt.Printf("\t-%s  --%-10s  %s", opt.Short_option, opt.Long_option,
        opt.Description)
    }

    if opt.Default != nil && opt.Type != "bool" {
      fmt.Printf("  [default: %s]", *opt.Default)
    }
    fmt.Printf("\n")
  }
}

// check_for_group_mode tests if at least one option has a
// group mode set
func check_for_group_mode(parse_info *templateSpec) bool {

  haveGroupMode := false
  for _, item := range parse_info.Options {
    if item.Subcommand != nil {
      haveGroupMode = true
      break
    }
  }

  return haveGroupMode
}

// validate_defaults checks that a usage string was given and
// that each spec has at least a short or a long option.
func validate_specs(parse_info *templateSpec, haveGroupMode bool) error {
  if parse_info.Usage_info == "" {
    return fmt.Errorf("Pagoda: Usage string missing")
  }

  for _, opt := range parse_info.Options {
    if opt.Short_option == "" && opt.Long_option == "" {
      return fmt.Errorf("Pagoda: Need at least a short or long description.")
    }

    if opt.Type == "" {
      return fmt.Errorf("Pagoda: Need a type descriptor for option %s.",
        opt.Short_option)
    }

    if haveGroupMode && opt.Subcommand == nil {
      return fmt.Errorf("Pagoda: Only some options have command group " +
        "mode selected.")
    }
  }

  return nil
}

// extract_defaults extracts the default field in the proper type
func extract_defaults(parse_info *templateSpec) error {

  opts := parse_info.Options
  for i := 0; i < len(opts); i++ {
    if opts[i].Default != nil {
      val, err := string_to_type(*opts[i].Default, opts[i].Type)
      if err != nil {
        return err
      }

      opts[i].value = val
    }
  }

  return nil
}

// string_to_type converts value to the requested type and strows
// an error if that fails
func string_to_type(value string, theType string) (interface{}, error) {

  switch theType {
  case "bool":
    if value == "true" {
      return true, nil
    } else if value == "false" {
      return false, nil
    } else {
      return nil, fmt.Errorf("cannot convert %s to requested type %s",
        value, theType)
    }

  case "int":
    if i, err := strconv.Atoi(value); err == nil {
      return i, nil
    } else {
      return nil, fmt.Errorf("Pagoda: cannot convert %s to requested type %s",
        value, theType)
    }

  case "float":
    if v, err := strconv.ParseFloat(value, 64); err == nil {
      return v, nil
    } else {
      return nil, fmt.Errorf("Pagoda: cannot convert %s to requested type %s",
        value, theType)
    }

  case "string":
    return value, nil

  default:
    return nil, fmt.Errorf("Pagoda: unknow type %s", theType)
  }
}

// inject_default_help_option adds a default help option to the list of
// command line switches
func inject_default_help_option(opts *jsonOptions) {

  helpDefault := "true"
  *opts = append(*opts,
    jsonOption{"h", "help", "print this help text and exit", "bool",
      &helpDefault, nil, nil})
}

// test_for_option determines if a string is an option (i.e. starts
// either with a dash ('-') or a double dash ('--')). In that
// case it returns the name of the option and the value if the
// option was given via --opt=val.
func decode_option(item string) (string, string, bool) {

  // check for dash
  c, s := utf8.DecodeRuneInString(item)
  if s == 0 || string(c) != "-" {
    return "", "", false
  }
  i := s

  // skip next dash if present
  c, s = utf8.DecodeRuneInString(item[i:])
  if s != 0 && string(c) == "-" {
    i += s;
  }

  // scan until end of string or until we hit a "="
  opt := ""
  for i < len(item) {
    c, s = utf8.DecodeRuneInString(item[i:])
    i += s

    if s == 0 {
      return "", "", false
    } else if string(c) == "=" {
      break
    }

    opt += string(c)
  }

  // scan for optional value specified via opt=val
  val := ""
  for i < len(item) {
    c, s = utf8.DecodeRuneInString(item[i:])
    i += s

    if s == 0 {
      return "", "", false
    }

    val += string(c)
  }

  return opt, val, true
}

// find_option retrieves the parse_spec option entry corresponding
// to the given name f present. Otherwise returns false.
func find_parse_spec(opts jsonOptions, name string) (*jsonOption, bool) {

  for i := 0; i < len(opts); i++ {
    if opts[i].Short_option == name || opts[i].Long_option == name {
      return &opts[i], true
    }
  }

  return &jsonOption{}, false
}

// group_options distributes all options across a map with group
// name as key
func group_options(parse_data *parsedSpec, template *templateSpec) map[string]jsonOptions {

  subcommand_options := make(map[string]jsonOptions)
  for _, opts := range template.Options {
    key := *opts.Subcommand
    if _, ok := subcommand_options[key]; !ok {
      option_list := make(jsonOptions, 0)
      option_list = append(option_list, opts)
      subcommand_options[key] = option_list
    } else {
      subcommand_options[key] = append(subcommand_options[key], opts)
    }
  }
  return subcommand_options
}

// inialize_parsed_spec initialize the parsed spec
func initialize_parsed_spec(haveSubcommands bool, template *templateSpec) *parsedSpec {

  // initialize final parsed specs
  parsed := parsedSpec{}
  parsed.Options = template.Options
  parsed.cmdlOptions = make([]jsonOption, 0)
  parsed.Usage_info = template.Usage_info

  if haveSubcommands {
    parsed.subcommand_options = group_options(&parsed, template)
    parsed.subcommand_mode= haveSubcommands
  }

  return &parsed
}

func extract_subcommand(parsed *parsedSpec, args []string) ([]string, error) {

  subcommand := args[1]
  opts, ok := parsed.subcommand_options[subcommand];
  if !ok {
    return nil, fmt.Errorf("Pagoda: Unknown command group %s", subcommand)
  }

  parsed.Options = opts
  parsed.subcommand = subcommand
  return args[2:], nil
}

// match_spec_to_args matches a parse_info spec to the provided command
// line options. Entries in parse_info which are lacking are ignored.
// If the command line contains entries which are not in the spec the
// function throws an error.
func match_spec_to_args(template *templateSpec, args []string,
  haveSubcommands bool) (*parsedSpec, error) {

  parsed := initialize_parsed_spec(haveSubcommands, template)

  if len(args) <= 1 {
    return parsed, nil
  }

  if haveSubcommands {
    var err error
    args, err = extract_subcommand(parsed, args)
    if err != nil {
      parsed.Usage()
      return nil, err
    }
  } else {
    args = args[1:]
  }

  inject_default_help_option(&parsed.Options)

  var opt_name, opt_val string
  var ok bool
  for i := 0; i < len(args); i++ {
    opt_name, opt_val, ok = decode_option(args[i])
    if !ok {
      parsed.remainder = args[i:]
      break
    }

    opt_spec, ok := find_parse_spec(parsed.Options, opt_name)
    if !ok {
      parsed.Usage()
      return nil, fmt.Errorf("Pagoda: Unknown command line option %s",
        args[i])
    }

    // if the option is not of type bool and we don't have
    // a value yet we peak at the next arg if present
    if opt_val == "" && opt_spec.Type != "bool" {
      if i+1 < len(args) {
        if _, _, ok := decode_option(args[i+1]); !ok {
          i++
          opt_val = args[i]
        }
      }
    }

    // check that we got a value if the option doesn't have default
    if opt_spec.Default == nil && opt_val == "" {
      parsed.Usage()
      return nil, fmt.Errorf("Pagoda: Missing value for option %s",
        opt_spec.Short_option)
    }

    // check that the provided option has the correct type
    if opt_val != "" {
      val, err := string_to_type(opt_val, opt_spec.Type)
      if err != nil {
        return nil, err
      }
      opt_spec.value = val
    }

    parsed.cmdlOptions = append(parsed.cmdlOptions, *opt_spec)
  }
  return parsed, nil
}
<commit_msg>Refactored code.<commit_after>// Copyright 2014 Markus Dittrich. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// pagoda is a go library for command line parsing
package pagoda

import (
  "encoding/json"
  "os"
  "fmt"
  "strconv"
  "unicode/utf8"
)

// parse_spec describes all options and usage actually present on the
// command line and also keeps track of any remaining command line entries
type parsedSpec struct {
  templateSpec
  cmdlOptions jsonOptions  // list of commandline options provided at runtime

  subcommand_mode bool
  subcommand string
  subcommand_options map[string]jsonOptions

  remainder []string      // unparsed remainder of command line
}

// target_spec is used for the initial parsing of the user provided
// JSON spec of commandline options
type templateSpec struct {
  Usage_info string
  Options jsonOptions
}

// json_option describes a single option specification as parsed from the
// JSON description
type jsonOption struct {
  Short_option string
  Long_option string
  Description string
  Type string
  Default *string      // use a pointer to be able to distinguish the empty
                       // string from non-present option
  Subcommand *string   // subcommand this option belongs to
  value interface{}    // option value of type Type
}

type jsonOptions []jsonOption

// Value returns the value known for the given option. Either the
// long or short option can be provided.
// NOTE: The look up could be made more efficient via a map
func (p *parsedSpec) Value(key string) (interface{}, error) {

  for _, opt := range p.cmdlOptions {
    if key == opt.Short_option || key == opt.Long_option {
      return opt.value, nil
    }
  }

  return nil, fmt.Errorf("Pagoda: command line option %s not found", key)
}

// Remainder returns a slice with all command line options that
// were not parsed into options
func (p *parsedSpec) Remainder() []string {
  return p.remainder
}

// Init parses the specification of option in JSON format
func Init(content []byte) (*parsedSpec, error) {

  var parse_info templateSpec
  err := json.Unmarshal(content, &parse_info)
  if err != nil {
    return nil, err
  }

  haveSubcommands := check_for_subcommand_mode(&parse_info)

  err = validate_specs(&parse_info, haveSubcommands)
  if err != nil {
    return nil, err
  }

  err = extract_defaults(&parse_info)
  if err != nil {
    return nil, err
  }

  parsed, args, err := initialize_parsed_spec(haveSubcommands, &parse_info,
    os.Args)
  if err != nil {
    return nil, err
  }

  inject_default_help_option(&parsed.Options)

  matched_info, err := match_spec_to_args(parsed, args)
  if err != nil {
    return nil, err
  }

  // check if help was requested. In that case show Usage() and then exit
  if _, err := matched_info.Value("h"); err == nil {
    if matched_info.subcommand_mode {
      command_usage(matched_info.Usage_info, matched_info.subcommand,
        matched_info.Options, os.Args)
    } else {
      matched_info.Usage()
    }
    os.Exit(0)
  }

  return matched_info, nil
}

// Usage prints the usage information for the package
func (p *parsedSpec) Usage() {

  fmt.Println(p.Usage_info)
  fmt.Println()

  if p.subcommand_mode {
    fmt.Printf("Usage: %s SUBCOMMAND [arguments]\n", os.Args[0])
    fmt.Println("\nAvailable SUBCOMMANDS are:\n")
    for k, _ := range p.subcommand_options {
      fmt.Printf("\t%-10s\n", k)
    }
  } else {
    command_usage(p.Usage_info, "", p.Options, os.Args)
  }

  fmt.Println()
}

// command_usage prints the usage for a specific subcommand
func command_usage(info string, subcommand string, options jsonOptions,
  args []string) {

  fmt.Printf("Usage: %s %s [arguments]\n\n", args[0], subcommand)
  for _, opt := range options {

    if opt.Long_option == "" {
      fmt.Printf("\t-%-15s  %s", opt.Short_option, opt.Description)
    } else if opt.Short_option == "" {
      fmt.Printf("\t    --%-10s  %s", opt.Long_option, opt.Description)
    } else {
      fmt.Printf("\t-%s  --%-10s  %s", opt.Short_option, opt.Long_option,
        opt.Description)
    }

    if opt.Default != nil && opt.Type != "bool" {
      fmt.Printf("  [default: %s]", *opt.Default)
    }
    fmt.Printf("\n")
  }
}

// check_for_group_mode tests if at least one option has a
// group mode set
func check_for_subcommand_mode(parse_info *templateSpec) bool {

  haveGroupMode := false
  for _, item := range parse_info.Options {
    if item.Subcommand != nil {
      haveGroupMode = true
      break
    }
  }

  return haveGroupMode
}

// validate_defaults checks that a usage string was given and
// that each spec has at least a short or a long option.
func validate_specs(parse_info *templateSpec, haveGroupMode bool) error {
  if parse_info.Usage_info == "" {
    return fmt.Errorf("Pagoda: Usage string missing")
  }

  for _, opt := range parse_info.Options {
    if opt.Short_option == "" && opt.Long_option == "" {
      return fmt.Errorf("Pagoda: Need at least a short or long description.")
    }

    if opt.Type == "" {
      return fmt.Errorf("Pagoda: Need a type descriptor for option %s.",
        opt.Short_option)
    }

    if haveGroupMode && opt.Subcommand == nil {
      return fmt.Errorf("Pagoda: Only some options have command group " +
        "mode selected.")
    }
  }

  return nil
}

// extract_defaults extracts the default field in the proper type
func extract_defaults(parse_info *templateSpec) error {

  opts := parse_info.Options
  for i := 0; i < len(opts); i++ {
    if opts[i].Default != nil {
      val, err := string_to_type(*opts[i].Default, opts[i].Type)
      if err != nil {
        return err
      }

      opts[i].value = val
    }
  }

  return nil
}

// string_to_type converts value to the requested type and strows
// an error if that fails
func string_to_type(value string, theType string) (interface{}, error) {

  switch theType {
  case "bool":
    if value == "true" {
      return true, nil
    } else if value == "false" {
      return false, nil
    } else {
      return nil, fmt.Errorf("cannot convert %s to requested type %s",
        value, theType)
    }

  case "int":
    if i, err := strconv.Atoi(value); err == nil {
      return i, nil
    } else {
      return nil, fmt.Errorf("Pagoda: cannot convert %s to requested type %s",
        value, theType)
    }

  case "float":
    if v, err := strconv.ParseFloat(value, 64); err == nil {
      return v, nil
    } else {
      return nil, fmt.Errorf("Pagoda: cannot convert %s to requested type %s",
        value, theType)
    }

  case "string":
    return value, nil

  default:
    return nil, fmt.Errorf("Pagoda: unknow type %s", theType)
  }
}

// inject_default_help_option adds a default help option to the list of
// command line switches
func inject_default_help_option(opts *jsonOptions) {

  helpDefault := "true"
  *opts = append(*opts,
    jsonOption{"h", "help", "print this help text and exit", "bool",
      &helpDefault, nil, nil})
}

// test_for_option determines if a string is an option (i.e. starts
// either with a dash ('-') or a double dash ('--')). In that
// case it returns the name of the option and the value if the
// option was given via --opt=val.
func decode_option(item string) (string, string, bool) {

  // check for dash
  c, s := utf8.DecodeRuneInString(item)
  if s == 0 || string(c) != "-" {
    return "", "", false
  }
  i := s

  // skip next dash if present
  c, s = utf8.DecodeRuneInString(item[i:])
  if s != 0 && string(c) == "-" {
    i += s;
  }

  // scan until end of string or until we hit a "="
  opt := ""
  for i < len(item) {
    c, s = utf8.DecodeRuneInString(item[i:])
    i += s

    if s == 0 {
      return "", "", false
    } else if string(c) == "=" {
      break
    }

    opt += string(c)
  }

  // scan for optional value specified via opt=val
  val := ""
  for i < len(item) {
    c, s = utf8.DecodeRuneInString(item[i:])
    i += s

    if s == 0 {
      return "", "", false
    }

    val += string(c)
  }

  return opt, val, true
}

// find_option retrieves the parse_spec option entry corresponding
// to the given name f present. Otherwise returns false.
func find_parse_spec(opts jsonOptions, name string) (*jsonOption, bool) {

  for i := 0; i < len(opts); i++ {
    if opts[i].Short_option == name || opts[i].Long_option == name {
      return &opts[i], true
    }
  }

  return &jsonOption{}, false
}

// group_options distributes all options across a map with group
// name as key
func group_options(parse_data *parsedSpec, template *templateSpec) map[string]jsonOptions {

  subcommand_options := make(map[string]jsonOptions)
  for _, opts := range template.Options {
    key := *opts.Subcommand
    if _, ok := subcommand_options[key]; !ok {
      option_list := make(jsonOptions, 0)
      option_list = append(option_list, opts)
      subcommand_options[key] = option_list
    } else {
      subcommand_options[key] = append(subcommand_options[key], opts)
    }
  }
  return subcommand_options
}

// inialize_parsed_spec initialize the parsed spec
func initialize_parsed_spec(haveSubcommands bool, template *templateSpec,
  args []string) (*parsedSpec, []string, error) {

  // initialize final parsed specs
  parsed := parsedSpec{}
  parsed.Options = template.Options
  parsed.cmdlOptions = make([]jsonOption, 0)
  parsed.Usage_info = template.Usage_info

  if haveSubcommands {
    parsed.subcommand_options = group_options(&parsed, template)
    parsed.subcommand_mode= haveSubcommands
  }

  if haveSubcommands && len(args) > 1 {
    subcommand := args[1]
    opts, ok := parsed.subcommand_options[subcommand];
    if !ok {
      parsed.Usage()
      return nil, nil,
        fmt.Errorf("Pagoda: Unknown command group %s", subcommand)
    }

    parsed.Options = opts
    parsed.subcommand = subcommand
    args = args[2:]
  } else {
    args = args[1:]
  }

  return &parsed, args, nil
}

// match_spec_to_args matches a parse_info spec to the provided command
// line options. Entries in parse_info which are lacking are ignored.
// If the command line contains entries which are not in the spec the
// function throws an error.
func match_spec_to_args(parsed *parsedSpec, args []string) (*parsedSpec, error) {
  var opt_name, opt_val string
  var ok bool
  for i := 0; i < len(args); i++ {
    opt_name, opt_val, ok = decode_option(args[i])
    if !ok {
      parsed.remainder = args[i:]
      break
    }

    opt_spec, ok := find_parse_spec(parsed.Options, opt_name)
    if !ok {
      parsed.Usage()
      return nil, fmt.Errorf("Pagoda: Unknown command line option %s",
        args[i])
    }

    // if the option is not of type bool and we don't have
    // a value yet we peak at the next arg if present
    if opt_val == "" && opt_spec.Type != "bool" {
      if i+1 < len(args) {
        if _, _, ok := decode_option(args[i+1]); !ok {
          i++
          opt_val = args[i]
        }
      }
    }

    // check that we got a value if the option doesn't have default
    if opt_spec.Default == nil && opt_val == "" {
      parsed.Usage()
      return nil, fmt.Errorf("Pagoda: Missing value for option %s",
        opt_spec.Short_option)
    }

    // check that the provided option has the correct type
    if opt_val != "" {
      val, err := string_to_type(opt_val, opt_spec.Type)
      if err != nil {
        return nil, err
      }
      opt_spec.value = val
    }

    parsed.cmdlOptions = append(parsed.cmdlOptions, *opt_spec)
  }
  return parsed, nil
}
<|endoftext|>


<commit_before>package main

// pandoc.go adds pandoc support for converting listings to PDF format.

import (
	"io/ioutil"
	"os"
	"os/exec"
)

// PdfWriter transforms the listing to a PDF.
func PdfWriter(markdown, filename string) (err error) {
	tmp, err := ioutil.TempFile("", "golst_pandoc")
	if err != nil {
		return
	}
	defer tmp.Close()
	defer os.Remove(tmp.Name())
	_, err = tmp.Write([]byte(markdown))
	if err != nil {
		return
	}

        outName := "-o " + filename + ".pdf"
	pandoc := exec.Command("pandoc", outName, tmp.Name())
	err = pandoc.Run()
	return
}
<commit_msg>go fmt<commit_after>package main

// pandoc.go adds pandoc support for converting listings to PDF format.

import (
	"io/ioutil"
	"os"
	"os/exec"
)

// PdfWriter transforms the listing to a PDF. First, the markdown is
// written to a temporary file (which is removed when the function
// returns); this temporary file is then passed to pandoc for conversion.
func PdfWriter(markdown, filename string) (err error) {
	tmp, err := ioutil.TempFile("", "golst_pandoc")
	if err != nil {
		return
	}
	defer tmp.Close()
	defer os.Remove(tmp.Name())

	_, err = tmp.Write([]byte(markdown))
	if err != nil {
		return
	}

	outName := "-o " + filename + ".pdf"
	pandoc := exec.Command("pandoc", outName, tmp.Name())
	err = pandoc.Run()
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	. "goprotobuf.googlecode.com/hg/compiler/descriptor"
	"goprotobuf.googlecode.com/hg/proto"
)

func ParseFiles(filenames []string) (*FileDescriptorSet, os.Error) {
	fds := &FileDescriptorSet{
		File: make([]*FileDescriptorProto, len(filenames)),
	}

	for i, filename := range filenames {
		fds.File[i] = &FileDescriptorProto{
			Name: proto.String(filename),
		}
		buf, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, err
		}

		p := newParser(string(buf))
		if pe := p.readFile(fds.File[i]); pe != nil {
			return nil, pe
		}
		log.Printf("Leftovers: %q", p.s)
	}

	return fds, nil
}

type parseError struct {
	message string
	line    int // 1-based line number
	offset  int // 0-based byte offset from start of input
}

func (pe *parseError) String() string {
	if pe.line == 1 {
		return fmt.Sprintf("line 1.%d: %v", pe.offset, pe.message)
	}
	return fmt.Sprintf("line %d: %v", pe.line, pe.message)
}

type token struct {
	value        string
	err          *parseError
	line, offset int
}

type parser struct {
	s            string // remaining input
	done         bool   // whether the parsing is finished
	backed       bool   // whether back() was called
	offset, line int
	cur          token
}

func newParser(s string) *parser {
	return &parser{
		s:    s,
		line: 1,
		cur: token{
			line: 1,
		},
	}
}

func (p *parser) readFile(fd *FileDescriptorProto) *parseError {
	// Read package
	if err := p.readToken("package"); err != nil {
		return err
	}
	tok := p.next()
	if tok.err != nil {
		return tok.err
	}
	// TODO: check for a good package name
	fd.Package = proto.String(tok.value)
	if err := p.readToken(";"); err != nil {
		return err
	}

	// Parse the rest of the file.
	for !p.done {
		tok := p.next()
		if tok.err != nil {
			return tok.err
		}
		switch tok.value {
		case "message":
			msg := new(DescriptorProto)
			fd.MessageType = append(fd.MessageType, msg)
			if err := p.readMessage(msg); err != nil {
				return err
			}
		default:
			return p.error("unknown top-level thing %q", tok.value)
		}
	}

	// TODO: more

	return nil
}

func (p *parser) readMessage(d *DescriptorProto) *parseError {
	// TODO
	return nil
}

func (p *parser) readToken(expected string) *parseError {
	tok := p.next()
	if tok.err != nil {
		return tok.err
	}
	if tok.value != expected {
		return p.error("expected %q, found %q", expected, tok.value)
	}
	return nil
}

// Back off the parser by one token; may only be done between calls to next().
func (p *parser) back() {
	p.backed = true
}

// Advances the parser and returns the new current token.
func (p *parser) next() *token {
	if p.backed || p.done {
		p.backed = false
		return &p.cur
	}
	p.advance()
	if p.done {
		p.cur.value = ""
	}
	return &p.cur
}

func (p *parser) advance() {
	// Skip whitespace
	p.skipWhitespaceAndComments()
	if p.done {
		return
	}

	// Start of non-whitespace
	p.cur.err = nil
	p.cur.offset, p.cur.line = p.offset, p.line
	switch p.s[0] {
	// TODO: more cases, like punctuation.
	case ';':
		// Single symbol
		p.cur.value, p.s = p.s[:1], p.s[1:]
	default:
		i := 0
		for i < len(p.s) && isIdentOrNumberChar(p.s[i]) {
			i++
		}
		if i == 0 {
			p.error("unexpected byte %#x", p.s[0])
			return
		}
		p.cur.value, p.s = p.s[:i], p.s[i:]
	}
	p.offset += len(p.cur.value)
}

func (p *parser) skipWhitespaceAndComments() {
	i := 0
	for i < len(p.s) {
		if isWhitespace(p.s[i]) {
			if p.s[i] == '\n' {
				p.line++
			}
			i++
			continue
		}
		if i+1 < len(p.s) && p.s[i] == '/' && p.s[i+1] == '/' {
			// comment; skip to end of line or input
			for i < len(p.s) && p.s[i] != '\n' {
				i++
			}
			if i < len(p.s) {
				// end of line; keep going
				p.line++
				i++
				continue
			}
			// end of input; fall out of loop
		}
		break
	}
	p.offset += i
	p.s = p.s[i:]
	if len(p.s) == 0 {
		p.done = true
	}
}

func (p *parser) error(format string, a ...interface{}) *parseError {
	pe := &parseError{
		message: fmt.Sprintf(format, a...),
		line:    p.cur.line,
		offset:  p.cur.offset,
	}
	p.cur.err = pe
	p.done = true
	return pe
}

func isWhitespace(c byte) bool {
	// TODO: do better
	return c == ' '
}

// Numbers and identifiers are matched by [-+._A-Za-z0-9]
func isIdentOrNumberChar(c byte) bool {
	switch {
	case 'A' <= c && c <= 'Z', 'a' <= c && c <= 'z':
		return true
	case '0' <= c && c <= '9':
		return true
	}
	switch c {
	case '-', '+', '.', '_':
		return true
	}
	return false
}
<commit_msg>Better whitespace and debug error reporting.<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	. "goprotobuf.googlecode.com/hg/compiler/descriptor"
	"goprotobuf.googlecode.com/hg/proto"
)

func ParseFiles(filenames []string) (*FileDescriptorSet, os.Error) {
	fds := &FileDescriptorSet{
		File: make([]*FileDescriptorProto, len(filenames)),
	}

	for i, filename := range filenames {
		fds.File[i] = &FileDescriptorProto{
			Name: proto.String(filename),
		}
		buf, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, err
		}

		p := newParser(string(buf))
		if pe := p.readFile(fds.File[i]); pe != nil {
			return nil, pe
		}
		log.Printf("Leftovers: %q", p.s)
	}

	return fds, nil
}

type parseError struct {
	message string
	line    int // 1-based line number
	offset  int // 0-based byte offset from start of input
}

func (pe *parseError) String() string {
	if pe == nil {
		return "<nil>"
	}
	if pe.line == 1 {
		return fmt.Sprintf("line 1.%d: %v", pe.offset, pe.message)
	}
	return fmt.Sprintf("line %d: %v", pe.line, pe.message)
}

type token struct {
	value        string
	err          *parseError
	line, offset int
}

type parser struct {
	s            string // remaining input
	done         bool   // whether the parsing is finished
	backed       bool   // whether back() was called
	offset, line int
	cur          token
}

func newParser(s string) *parser {
	return &parser{
		s:    s,
		line: 1,
		cur: token{
			line: 1,
		},
	}
}

func (p *parser) readFile(fd *FileDescriptorProto) *parseError {
	// Read package
	if err := p.readToken("package"); err != nil {
		return err
	}
	tok := p.next()
	if tok.err != nil {
		return tok.err
	}
	// TODO: check for a good package name
	fd.Package = proto.String(tok.value)
	if err := p.readToken(";"); err != nil {
		return err
	}

	// Parse the rest of the file.
	for !p.done {
		tok := p.next()
		if tok.err != nil {
			return tok.err
		}
		switch tok.value {
		case "message":
			msg := new(DescriptorProto)
			fd.MessageType = append(fd.MessageType, msg)
			if err := p.readMessage(msg); err != nil {
				return err
			}
		default:
			return p.error("unknown top-level thing %q", tok.value)
		}
	}

	// TODO: more

	return nil
}

func (p *parser) readMessage(d *DescriptorProto) *parseError {
	// TODO
	return nil
}

func (p *parser) readToken(expected string) *parseError {
	tok := p.next()
	if tok.err != nil {
		return tok.err
	}
	if tok.value != expected {
		return p.error("expected %q, found %q", expected, tok.value)
	}
	return nil
}

// Back off the parser by one token; may only be done between calls to next().
func (p *parser) back() {
	p.backed = true
}

// Advances the parser and returns the new current token.
func (p *parser) next() *token {
	if p.backed || p.done {
		p.backed = false
	} else {
		p.advance()
		if p.done {
			p.cur.value = ""
		}
	}
	log.Printf("parser·next(): returning %q [err: %v]", p.cur.value, p.cur.err)
	return &p.cur
}

func (p *parser) advance() {
	// Skip whitespace
	p.skipWhitespaceAndComments()
	if p.done {
		return
	}

	// Start of non-whitespace
	p.cur.err = nil
	p.cur.offset, p.cur.line = p.offset, p.line
	switch p.s[0] {
	// TODO: more cases, like punctuation.
	case ';':
		// Single symbol
		p.cur.value, p.s = p.s[:1], p.s[1:]
	default:
		i := 0
		for i < len(p.s) && isIdentOrNumberChar(p.s[i]) {
			i++
		}
		if i == 0 {
			p.error("unexpected byte 0x%02x", p.s[0])
			return
		}
		p.cur.value, p.s = p.s[:i], p.s[i:]
	}
	p.offset += len(p.cur.value)
}

func (p *parser) skipWhitespaceAndComments() {
	i := 0
	for i < len(p.s) {
		if isWhitespace(p.s[i]) {
			if p.s[i] == '\n' {
				p.line++
			}
			i++
			continue
		}
		if i+1 < len(p.s) && p.s[i] == '/' && p.s[i+1] == '/' {
			// comment; skip to end of line or input
			for i < len(p.s) && p.s[i] != '\n' {
				i++
			}
			if i < len(p.s) {
				// end of line; keep going
				p.line++
				i++
				continue
			}
			// end of input; fall out of loop
		}
		break
	}
	p.offset += i
	p.s = p.s[i:]
	if len(p.s) == 0 {
		p.done = true
	}
}

func (p *parser) error(format string, a ...interface{}) *parseError {
	pe := &parseError{
		message: fmt.Sprintf(format, a...),
		line:    p.cur.line,
		offset:  p.cur.offset,
	}
	p.cur.err = pe
	p.done = true
	return pe
}

func isWhitespace(c byte) bool {
	// TODO: do better
	return c == ' ' || c == '\n'
}

// Numbers and identifiers are matched by [-+._A-Za-z0-9]
func isIdentOrNumberChar(c byte) bool {
	switch {
	case 'A' <= c && c <= 'Z', 'a' <= c && c <= 'z':
		return true
	case '0' <= c && c <= '9':
		return true
	}
	switch c {
	case '-', '+', '.', '_':
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package nmea

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"
)

var notHandled = errors.New("not handled")

var parsers = map[string]func([]string, interface{}) error{
	"$GPRMC": rmcParser,
	"$GPVTG": vtgParser,
	"$GPGGA": ggaParser,
	"$GPGSA": gsaParser,
	"$GPGLL": gllParser,
	"$GPZDA": zdaParser,
}

type cumulativeErrorParser struct {
	err error
}

func (c *cumulativeErrorParser) parseFloat(s string) float64 {
	if s == "" {
		return 0
	}
	rv, err := strconv.ParseFloat(s, 64)
	if err != nil {
		c.err = err
	}
	return rv
}

func (c *cumulativeErrorParser) parseInt(s string) int {
	if s == "" {
		return 0
	}
	rv, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		c.err = err
	}
	return int(rv)
}

func (c *cumulativeErrorParser) parseDMS(s, ref string) float64 {
	n := 2
	m := 1.0
	if ref == "E" || ref == "W" {
		n = 3
	}
	if ref == "S" || ref == "W" {
		m = -1
	}

	deg := c.parseFloat(s[:n])
	min := c.parseFloat(s[n:])
	deg += (min / 60.0)
	deg *= m

	return deg
}

/*
   0:   RMC          Recommended Minimum sentence C
   1:   123519       Fix taken at 12:35:19 UTC
   2:   A            Status A=active or V=Void.
   3,4: 4807.038,N   Latitude 48 deg 07.038' N
   5,6: 01131.000,E  Longitude 11 deg 31.000' E
   7:   022.4        Speed over the ground in knots
   8:   084.4        Track angle in degrees True
   9:   230394       Date - 23rd of March 1994
   10,11:  003.1,W      Magnetic Variation
*/
func rmcParser(parts []string, handler interface{}) error {
	h, ok := handler.(RMCHandler)
	if !ok {
		return notHandled
	}

	t, err := time.Parse("150405.99 020106 UTC", parts[1]+" "+parts[9]+" UTC")
	if err != nil {
		return err
	}

	cp := &cumulativeErrorParser{}

	lat := cp.parseDMS(parts[3], parts[4])
	lon := cp.parseDMS(parts[5], parts[6])
	speed := cp.parseFloat(parts[7])
	angle := cp.parseFloat(parts[8])
	magvar := 0.0
	if parts[10] != "" {
		magvar = cp.parseFloat(parts[10])
		if parts[11] == "W" {
			magvar *= -1
		}
	}

	if cp.err != nil {
		return cp.err
	}

	h.HandleRMC(RMC{
		Timestamp: t,
		Status:    rune(parts[2][0]),
		Latitude:  lat,
		Longitude: lon,
		Speed:     speed,
		Angle:     angle,
		Magvar:    magvar,
	})

	return nil
}

/*
VTG - Velocity made good. The gps receiver may use the LC prefix
instead of GP if it is emulating Loran output.

  $GPVTG,054.7,T,034.4,M,005.5,N,010.2,K*48

where:
        // 0:    VTG          Track made good and ground speed
        // 1,2:  054.7,T      True track made good (degrees)
        // 3,4:  034.4,M      Magnetic track made good
        // 5,6:  005.5,N      Ground speed, knots
        // 7,8:  010.2,K      Ground speed, Kilometers per hour
*/
func vtgParser(parts []string, handler interface{}) error {
	h, ok := handler.(VTGHandler)
	if !ok {
		return notHandled
	}

	if parts[2] != "T" || parts[4] != "M" || parts[6] != "N" || parts[8] != "K" {
		return fmt.Errorf("Unexpected VTG packet: %#v", parts)
	}

	cp := &cumulativeErrorParser{}
	vtg := VTG{
		True:     cp.parseFloat(parts[1]),
		Magnetic: cp.parseFloat(parts[3]),
		Knots:    cp.parseFloat(parts[5]),
		KMH:      cp.parseFloat(parts[7]),
	}

	if cp.err != nil {
		return cp.err
	}

	h.HandleVTG(vtg)

	return nil
}

/*
 $GPGGA,123519,4807.038,N,01131.000,E,1,08,0.9,545.4,M,46.9,M,,*47

Where:
     1:    123519       Fix taken at 12:35:19 UTC
     2,3:  4807.038,N   Latitude 48 deg 07.038' N
     4,5:  01131.000,E  Longitude 11 deg 31.000' E
     6:    1            Fix quality: 0 = invalid
                               1 = GPS fix (SPS)
                               2 = DGPS fix
                               3 = PPS fix
			       4 = Real Time Kinematic
			       5 = Float RTK
                               6 = estimated (dead reckoning) (2.3 feature)
			       7 = Manual input mode
			       8 = Simulation mode
     7:     08           Number of satellites being tracked
     8:     0.9          Horizontal dilution of position
     9,10:  545.4,M      Altitude, Meters, above mean sea level
     11,12: 46.9,M       Height of geoid (mean sea level) above WGS84
                      ellipsoid
     (empty field) time in seconds since last DGPS update
     (empty field) DGPS station ID number
     *47          the checksum data, always begins with *

*/
func ggaParser(parts []string, handler interface{}) error {
	h, ok := handler.(GGAHandler)
	if !ok {
		return notHandled
	}

	if len(parts) < 13 || parts[10] != "M" || parts[12] != "M" {
		return fmt.Errorf("Unexpected GGA packet: %#v", parts)
	}

	t, err := time.Parse("150405 UTC", parts[1]+" UTC")
	if err != nil {
		return err
	}

	cp := &cumulativeErrorParser{}
	h.HandleGGA(GGA{
		Taken:              t,
		Latitude:           cp.parseDMS(parts[2], parts[3]),
		Longitude:          cp.parseDMS(parts[4], parts[5]),
		Quality:            FixQuality(cp.parseInt(parts[6])),
		HorizontalDilution: cp.parseFloat(parts[8]),
		NumSats:            cp.parseInt(parts[7]),
		Altitude:           cp.parseFloat(parts[9]),
		GeoidHeight:        cp.parseFloat(parts[11]),
	})

	return cp.err
}

/*
  $GPGSA,A,3,04,05,,09,12,,,24,,,,,2.5,1.3,2.1*39

Where:
     1. A        Auto selection of 2D or 3D fix (M = manual)
     2. 3        3D fix - values include: 1 = no fix
                                       2 = 2D fix
                                       3 = 3D fix
     3-15.  04,05... PRNs of satellites used for fix (space for 12)
     15.    2.5      PDOP (dilution of precision)
     16. 1.3      Horizontal dilution of precision (HDOP)
     17. 2.1      Vertical dilution of precision (VDOP)
*/
func gsaParser(parts []string, handler interface{}) error {
	h, ok := handler.(GSAHandler)
	if !ok {
		return notHandled
	}

	if len(parts) != 18 {
		return fmt.Errorf("Unexpected GSA packet: %#v (len=%v)", parts, len(parts))
	}

	cp := &cumulativeErrorParser{}
	sats := []int{}
	for _, s := range parts[3:15] {
		if s != "" {
			sats = append(sats, cp.parseInt(s))
		}
	}

	h.HandleGSA(GSA{
		Auto:     parts[1] == "A",
		Fix:      GSAFix(cp.parseInt(parts[2])),
		SatsUsed: sats,
		PDOP:     cp.parseFloat(parts[15]),
		HDOP:     cp.parseFloat(parts[16]),
		VDOP:     cp.parseFloat(parts[17]),
	})

	return cp.err
}

/*
  $GPGLL,4916.45,N,12311.12,W,225444,A,*1D

Where:
     0,   GLL          Geographic position, Latitude and Longitude
     1,2: 4916.46,N    Latitude 49 deg. 16.45 min. North
     3,4: 12311.12,W   Longitude 123 deg. 11.12 min. West
     5:   225444       Fix taken at 22:54:44 UTC
     6:   A            Data Active or V (void)
*/
func gllParser(parts []string, handler interface{}) error {
	h, ok := handler.(GLLHandler)
	if !ok {
		return notHandled
	}

	t, err := time.Parse("150405 UTC", parts[5]+" UTC")
	if err != nil {
		return err
	}

	cp := &cumulativeErrorParser{}
	h.HandleGLL(GLL{
		Taken:     t,
		Latitude:  cp.parseDMS(parts[1], parts[2]),
		Longitude: cp.parseDMS(parts[3], parts[4]),
		Active:    parts[6] == "A",
	})
	return nil
}

/*
ZDA - Data and Time

  $GPZDA,hhmmss.ss,dd,mm,yyyy,xx,yy*CC
  $GPZDA,201530.00,04,07,2002,00,00*60

where:
	1.     hhmmss    HrMinSec(UTC)
        2,3,4. dd,mm,yyy Day,Month,Year
        5.     xx        local zone hours -13..13
        6.     yy        local zone minutes 0..59
*/
func zdaParser(parts []string, handler interface{}) error {
	h, ok := handler.(ZDAHandler)
	if !ok {
		return notHandled
	}

	if len(parts) != 7 || len(parts[1]) < 6 {
		return fmt.Errorf("Unexpected ZDA packet: %#v (len=%v)", parts, len(parts))
	}

	cp := &cumulativeErrorParser{}
	ts := time.Date(
		cp.parseInt(parts[4]),
		time.Month(cp.parseInt(parts[3])),
		cp.parseInt(parts[2]),
		cp.parseInt(parts[1][:2]),
		cp.parseInt(parts[1][2:4]),
		cp.parseInt(parts[1][4:6]),
		int(float64(time.Second)*cp.parseFloat(parts[1][6:])),
		time.UTC)

	h.HandleZDA(ZDA{ts})

	return cp.err
}

func checkChecksum(line string) bool {
	cs := 0
	if len(line) < 4 {
		return false
	}

	if line[0] != '$' {
		return false
	}
	if line[len(line)-3] != '*' {
		return false
	}
	exp, err := strconv.ParseInt(line[len(line)-2:], 16, 64)
	if err != nil {
		log.Printf("Failed to parse checksum: %v", err)
		return false
	}

	for _, c := range line[1:] {
		if c == '*' {
			break
		}
		cs = cs ^ int(c)
	}

	return cs == int(exp)
}

func parseMessage(line string, handler interface{}) {
	if !checkChecksum(line) {
		// skip bad checksums
		return
	}

	parts := strings.Split(line[:len(line)-3], ",")

	if p, ok := parsers[parts[0]]; ok {
		if err := p(parts, handler); err != nil {
			log.Printf("Error parsing %#v: %v", parts, err)
		}
	}
}
<commit_msg>Use a var block for parser vars<commit_after>package nmea

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"
)

var (
	notHandled = errors.New("not handled")

	parsers = map[string]func([]string, interface{}) error{
		"$GPRMC": rmcParser,
		"$GPVTG": vtgParser,
		"$GPGGA": ggaParser,
		"$GPGSA": gsaParser,
		"$GPGLL": gllParser,
		"$GPZDA": zdaParser,
	}
)

type cumulativeErrorParser struct {
	err error
}

func (c *cumulativeErrorParser) parseFloat(s string) float64 {
	if s == "" {
		return 0
	}
	rv, err := strconv.ParseFloat(s, 64)
	if err != nil {
		c.err = err
	}
	return rv
}

func (c *cumulativeErrorParser) parseInt(s string) int {
	if s == "" {
		return 0
	}
	rv, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		c.err = err
	}
	return int(rv)
}

func (c *cumulativeErrorParser) parseDMS(s, ref string) float64 {
	n := 2
	m := 1.0
	if ref == "E" || ref == "W" {
		n = 3
	}
	if ref == "S" || ref == "W" {
		m = -1
	}

	deg := c.parseFloat(s[:n])
	min := c.parseFloat(s[n:])
	deg += (min / 60.0)
	deg *= m

	return deg
}

/*
   0:   RMC          Recommended Minimum sentence C
   1:   123519       Fix taken at 12:35:19 UTC
   2:   A            Status A=active or V=Void.
   3,4: 4807.038,N   Latitude 48 deg 07.038' N
   5,6: 01131.000,E  Longitude 11 deg 31.000' E
   7:   022.4        Speed over the ground in knots
   8:   084.4        Track angle in degrees True
   9:   230394       Date - 23rd of March 1994
   10,11:  003.1,W      Magnetic Variation
*/
func rmcParser(parts []string, handler interface{}) error {
	h, ok := handler.(RMCHandler)
	if !ok {
		return notHandled
	}

	t, err := time.Parse("150405.99 020106 UTC", parts[1]+" "+parts[9]+" UTC")
	if err != nil {
		return err
	}

	cp := &cumulativeErrorParser{}

	lat := cp.parseDMS(parts[3], parts[4])
	lon := cp.parseDMS(parts[5], parts[6])
	speed := cp.parseFloat(parts[7])
	angle := cp.parseFloat(parts[8])
	magvar := 0.0
	if parts[10] != "" {
		magvar = cp.parseFloat(parts[10])
		if parts[11] == "W" {
			magvar *= -1
		}
	}

	if cp.err != nil {
		return cp.err
	}

	h.HandleRMC(RMC{
		Timestamp: t,
		Status:    rune(parts[2][0]),
		Latitude:  lat,
		Longitude: lon,
		Speed:     speed,
		Angle:     angle,
		Magvar:    magvar,
	})

	return nil
}

/*
VTG - Velocity made good. The gps receiver may use the LC prefix
instead of GP if it is emulating Loran output.

  $GPVTG,054.7,T,034.4,M,005.5,N,010.2,K*48

where:
        // 0:    VTG          Track made good and ground speed
        // 1,2:  054.7,T      True track made good (degrees)
        // 3,4:  034.4,M      Magnetic track made good
        // 5,6:  005.5,N      Ground speed, knots
        // 7,8:  010.2,K      Ground speed, Kilometers per hour
*/
func vtgParser(parts []string, handler interface{}) error {
	h, ok := handler.(VTGHandler)
	if !ok {
		return notHandled
	}

	if parts[2] != "T" || parts[4] != "M" || parts[6] != "N" || parts[8] != "K" {
		return fmt.Errorf("Unexpected VTG packet: %#v", parts)
	}

	cp := &cumulativeErrorParser{}
	vtg := VTG{
		True:     cp.parseFloat(parts[1]),
		Magnetic: cp.parseFloat(parts[3]),
		Knots:    cp.parseFloat(parts[5]),
		KMH:      cp.parseFloat(parts[7]),
	}

	if cp.err != nil {
		return cp.err
	}

	h.HandleVTG(vtg)

	return nil
}

/*
 $GPGGA,123519,4807.038,N,01131.000,E,1,08,0.9,545.4,M,46.9,M,,*47

Where:
     1:    123519       Fix taken at 12:35:19 UTC
     2,3:  4807.038,N   Latitude 48 deg 07.038' N
     4,5:  01131.000,E  Longitude 11 deg 31.000' E
     6:    1            Fix quality: 0 = invalid
                               1 = GPS fix (SPS)
                               2 = DGPS fix
                               3 = PPS fix
			       4 = Real Time Kinematic
			       5 = Float RTK
                               6 = estimated (dead reckoning) (2.3 feature)
			       7 = Manual input mode
			       8 = Simulation mode
     7:     08           Number of satellites being tracked
     8:     0.9          Horizontal dilution of position
     9,10:  545.4,M      Altitude, Meters, above mean sea level
     11,12: 46.9,M       Height of geoid (mean sea level) above WGS84
                      ellipsoid
     (empty field) time in seconds since last DGPS update
     (empty field) DGPS station ID number
     *47          the checksum data, always begins with *

*/
func ggaParser(parts []string, handler interface{}) error {
	h, ok := handler.(GGAHandler)
	if !ok {
		return notHandled
	}

	if len(parts) < 13 || parts[10] != "M" || parts[12] != "M" {
		return fmt.Errorf("Unexpected GGA packet: %#v", parts)
	}

	t, err := time.Parse("150405 UTC", parts[1]+" UTC")
	if err != nil {
		return err
	}

	cp := &cumulativeErrorParser{}
	h.HandleGGA(GGA{
		Taken:              t,
		Latitude:           cp.parseDMS(parts[2], parts[3]),
		Longitude:          cp.parseDMS(parts[4], parts[5]),
		Quality:            FixQuality(cp.parseInt(parts[6])),
		HorizontalDilution: cp.parseFloat(parts[8]),
		NumSats:            cp.parseInt(parts[7]),
		Altitude:           cp.parseFloat(parts[9]),
		GeoidHeight:        cp.parseFloat(parts[11]),
	})

	return cp.err
}

/*
  $GPGSA,A,3,04,05,,09,12,,,24,,,,,2.5,1.3,2.1*39

Where:
     1. A        Auto selection of 2D or 3D fix (M = manual)
     2. 3        3D fix - values include: 1 = no fix
                                       2 = 2D fix
                                       3 = 3D fix
     3-15.  04,05... PRNs of satellites used for fix (space for 12)
     15.    2.5      PDOP (dilution of precision)
     16. 1.3      Horizontal dilution of precision (HDOP)
     17. 2.1      Vertical dilution of precision (VDOP)
*/
func gsaParser(parts []string, handler interface{}) error {
	h, ok := handler.(GSAHandler)
	if !ok {
		return notHandled
	}

	if len(parts) != 18 {
		return fmt.Errorf("Unexpected GSA packet: %#v (len=%v)", parts, len(parts))
	}

	cp := &cumulativeErrorParser{}
	sats := []int{}
	for _, s := range parts[3:15] {
		if s != "" {
			sats = append(sats, cp.parseInt(s))
		}
	}

	h.HandleGSA(GSA{
		Auto:     parts[1] == "A",
		Fix:      GSAFix(cp.parseInt(parts[2])),
		SatsUsed: sats,
		PDOP:     cp.parseFloat(parts[15]),
		HDOP:     cp.parseFloat(parts[16]),
		VDOP:     cp.parseFloat(parts[17]),
	})

	return cp.err
}

/*
  $GPGLL,4916.45,N,12311.12,W,225444,A,*1D

Where:
     0,   GLL          Geographic position, Latitude and Longitude
     1,2: 4916.46,N    Latitude 49 deg. 16.45 min. North
     3,4: 12311.12,W   Longitude 123 deg. 11.12 min. West
     5:   225444       Fix taken at 22:54:44 UTC
     6:   A            Data Active or V (void)
*/
func gllParser(parts []string, handler interface{}) error {
	h, ok := handler.(GLLHandler)
	if !ok {
		return notHandled
	}

	t, err := time.Parse("150405 UTC", parts[5]+" UTC")
	if err != nil {
		return err
	}

	cp := &cumulativeErrorParser{}
	h.HandleGLL(GLL{
		Taken:     t,
		Latitude:  cp.parseDMS(parts[1], parts[2]),
		Longitude: cp.parseDMS(parts[3], parts[4]),
		Active:    parts[6] == "A",
	})
	return nil
}

/*
ZDA - Data and Time

  $GPZDA,hhmmss.ss,dd,mm,yyyy,xx,yy*CC
  $GPZDA,201530.00,04,07,2002,00,00*60

where:
	1.     hhmmss    HrMinSec(UTC)
        2,3,4. dd,mm,yyy Day,Month,Year
        5.     xx        local zone hours -13..13
        6.     yy        local zone minutes 0..59
*/
func zdaParser(parts []string, handler interface{}) error {
	h, ok := handler.(ZDAHandler)
	if !ok {
		return notHandled
	}

	if len(parts) != 7 || len(parts[1]) < 6 {
		return fmt.Errorf("Unexpected ZDA packet: %#v (len=%v)", parts, len(parts))
	}

	cp := &cumulativeErrorParser{}
	ts := time.Date(
		cp.parseInt(parts[4]),
		time.Month(cp.parseInt(parts[3])),
		cp.parseInt(parts[2]),
		cp.parseInt(parts[1][:2]),
		cp.parseInt(parts[1][2:4]),
		cp.parseInt(parts[1][4:6]),
		int(float64(time.Second)*cp.parseFloat(parts[1][6:])),
		time.UTC)

	h.HandleZDA(ZDA{ts})

	return cp.err
}

func checkChecksum(line string) bool {
	cs := 0
	if len(line) < 4 {
		return false
	}

	if line[0] != '$' {
		return false
	}
	if line[len(line)-3] != '*' {
		return false
	}
	exp, err := strconv.ParseInt(line[len(line)-2:], 16, 64)
	if err != nil {
		log.Printf("Failed to parse checksum: %v", err)
		return false
	}

	for _, c := range line[1:] {
		if c == '*' {
			break
		}
		cs = cs ^ int(c)
	}

	return cs == int(exp)
}

func parseMessage(line string, handler interface{}) {
	if !checkChecksum(line) {
		// skip bad checksums
		return
	}

	parts := strings.Split(line[:len(line)-3], ",")

	if p, ok := parsers[parts[0]]; ok {
		if err := p(parts, handler); err != nil {
			log.Printf("Error parsing %#v: %v", parts, err)
		}
	}
}
<|endoftext|>


<commit_before>package main
import (
    "fmt"
)
func getGrammar() map[string] (map[string] float64) {
    grammar := map[string] map[string] float64 {}
    temp := make(map[string] float64)
    temp["NP VP"] = 1.0
    grammar["S"] = make(map[string] float64)
    grammar["S"] = temp

    fmt.Printf("WHLLOOO")
    temp = make(map[string] float64,0)
    temp["P NP"] = 1.0
    grammar["PP"] = temp

    temp = make(map[string] float64,0)
    temp["V NP"] = 0.7
    grammar["VP"] = temp

    temp = make(map[string] float64,0)
    temp["VP PP"] = 0.3
    grammar["VP"] = temp

    return grammar
}

func parser(tokens []string) {
    cyk_grid := make([][]string,len(tokens))
    for i,_ := range tokens {
        cyk_grid[0] = append(cyk_grid[0],tokens[i])
    }
    grammar := getGrammar()
    fmt.Printf("%#v\n%#v\n",cyk_grid,grammar)
}
<commit_msg>Removing printf<commit_after>package main
import (
    "fmt"
)
func getGrammar() map[string] (map[string] float64) {
    grammar := map[string] map[string] float64 {}
    temp := make(map[string] float64)
    temp["NP VP"] = 1.0
    grammar["S"] = make(map[string] float64)
    grammar["S"] = temp

    temp = make(map[string] float64,0)
    temp["P NP"] = 1.0
    grammar["PP"] = temp

    temp = make(map[string] float64,0)
    temp["V NP"] = 0.7
    grammar["VP"] = temp

    temp = make(map[string] float64,0)
    temp["VP PP"] = 0.3
    grammar["VP"] = temp

    return grammar
}

func parser(tokens []string) {
    cyk_grid := make([][]string,len(tokens))
    for i,_ := range tokens {
        cyk_grid[0] = append(cyk_grid[0],tokens[i])
    }
    grammar := getGrammar()
    fmt.Printf("%#v\n%#v\n",cyk_grid,grammar)
}
<|endoftext|>


<commit_before>package recon

import (
	"fmt"
	"golang.org/x/net/html"
	"io"
	"math"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"sort"
	"time"

	"image/gif"
	"image/jpeg"
	"image/png"
)

type Parser struct {
	client   *http.Client
	req      *http.Request
	metaTags []metaTag
	imgTags  []imgTag
}

type ParseResult struct {
	URL         string             `json:"url"`
	Host        string             `json:"host"`
	Site        string             `json:"site_name"`
	Title       string             `json:"title"`
	Type        string             `json:"type"`
	Description string             `json:"description"`
	Author      string             `json:"author"`
	Publisher   string             `json:"publisher"`
	Images      []parseResultImage `json:"images"`
	Scraped     time.Time          `json:"scraped"`
}

type metaTag struct {
	name     string
	value    string
	priority float64
}

type imgTag struct {
	url       string
	alt       string
	preferred bool
}

type parseResultImage struct {
	URL         string  `json:"url"`
	Type        string  `json:"type"`
	Width       int     `json:"width"`
	Height      int     `json:"height"`
	Alt         string  `json:"alt"`
	AspectRatio float64 `json:"aspectRatio"`
	Preferred   bool    `json:"preferred,omitempty"`
}

type byPreferred []parseResultImage

var targetedProperties = map[string]float64{
	"og:site_name":   1,
	"og:title":       1,
	"og:type":        1,
	"og:description": 1,
	"og:author":      1,
	"og:publisher":   1,
	"og:url":         1,
	"og:image":       1,

	"site_name":   0.5,
	"title":       0.5,
	"type":        0.5,
	"description": 0.5,
	"author":      0.5,
	"publisher":   0.5,
}

var propertyMap = map[string][]string{
	"URL":         []string{"og:url"},
	"Site":        []string{"og:site_name", "site_name"},
	"Title":       []string{"og:title", "title"},
	"Type":        []string{"og:type", "type"},
	"Description": []string{"og:description", "description"},
	"Author":      []string{"og:author", "author"},
	"Publisher":   []string{"og:publisher", "publisher"},
}

var OptimalAspectRatio = 1.91

func NewParser() Parser {
	client := http.DefaultClient
	client.Jar, _ = cookiejar.New(nil)

	return Parser{
		client:   client,
		imgTags:  make([]imgTag, 0),
		metaTags: make([]metaTag, 0),
	}
}

func (p *Parser) Parse(url string) (ParseResult, error) {
	req, err := http.NewRequest("GET", url, nil)

	resp, err := p.client.Do(req)
	if err != nil {
		return ParseResult{}, fmt.Errorf("http error: %s", err)
	}

	p.req = req

	decoder := html.NewTokenizer(resp.Body)
	done := false
	for !done {
		tt := decoder.Next()
		switch tt {
		case html.ErrorToken:
			if decoder.Err() == io.EOF {
				done = true
			} else {
				// fmt.Println(decoder.Err())
			}
		case html.SelfClosingTagToken:
			t := decoder.Token()
			if t.Data == "meta" {
				res := p.parseMeta(t)
				if res.priority > 0 {
					p.metaTags = append(p.metaTags, res)
				}
			} else if t.Data == "img" {
				res := p.parseImg(t)
				if res.url != "" {
					p.imgTags = append(p.imgTags, res)
				}
			}
		}
	}

	res := p.buildResult()

	return res, nil
}

func (p *Parser) parseMeta(t html.Token) metaTag {
	var content string
	var tag string
	var priority float64

	for _, v := range t.Attr {
		if v.Key == "property" || v.Key == "name" {
			if _priority, exists := targetedProperties[v.Val]; exists {
				tag = v.Val
				priority = _priority
			}
		} else if v.Key == "content" {
			content = v.Val
		}
	}

	if priority > 0 {
		return metaTag{name: tag, value: content, priority: priority}
	} else {
		return metaTag{}
	}
}

func (p *Parser) parseImg(t html.Token) (i imgTag) {
	for _, v := range t.Attr {
		if v.Key == "src" {
			i.url = v.Val
		} else if v.Key == "alt" {
			i.alt = v.Val
		}
	}

	if i.url != "" {
		return
	} else {
		return imgTag{}
	}
}

func (p *Parser) buildResult() ParseResult {
	res := ParseResult{}
	if canonical_url := p.getMaxProperty("URL"); canonical_url != "" {
		res.URL = canonical_url
	} else {
		res.URL = p.req.URL.String()
	}
	res.Host = p.req.URL.Host

	res.Site = p.getMaxProperty("Site")
	res.Title = p.getMaxProperty("Title")
	res.Type = p.getMaxProperty("Type")
	res.Description = p.getMaxProperty("Description")
	res.Author = p.getMaxProperty("Author")
	res.Publisher = p.getMaxProperty("Publisher")

	for _, tag := range p.metaTags {
		if tag.name == "og:image" {
			p.imgTags = append(p.imgTags, imgTag{
				url:       tag.value,
				preferred: true,
			})
		}
	}

	res.Images = p.analyzeImages()

	res.Scraped = time.Now()
	return res
}

func (p *Parser) getMaxProperty(key string) (val string) {
	max_priority := 0.0

	for _, search_tag := range propertyMap[key] {
		for _, tag := range p.metaTags {
			if tag.name == search_tag && tag.priority > max_priority {
				val = tag.value
				max_priority = tag.priority
			}
		}
	}

	return
}

func (p *Parser) analyzeImages() []parseResultImage {
	type image struct {
		url          string
		data         io.Reader
		alt          string
		content_type string
		preferred    bool
	}

	ch := make(chan image)
	returned_images := make([]parseResultImage, 0)
	found_images := 0

	for _, tag := range p.imgTags {
		go func(tag imgTag) {
			img_url, _ := url.Parse(tag.url)
			img_url = p.req.URL.ResolveReference(img_url)

			req, _ := http.NewRequest("GET", img_url.String(), nil)
			resp, _ := p.client.Do(req)

			img := image{
				url:          img_url.String(),
				content_type: resp.Header.Get("Content-Type"),

				data: resp.Body,

				alt:       tag.alt,
				preferred: tag.preferred,
			}

			ch <- img
		}(tag)
		found_images++
	}

	if found_images == 0 {
		return returned_images
	}

	for {
		select {
		case incoming_img := <-ch:
			ret_image := parseResultImage{}
			switch incoming_img.content_type {
			case "image/jpeg":
				img, _ := jpeg.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			case "image/gif":
				img, _ := gif.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			case "image/png":
				img, _ := png.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			}

			if ret_image.Height > 0 {
				ret_image.AspectRatio = round(float64(ret_image.Width)/float64(ret_image.Height), 1e-4)
			}

			ret_image.Type = incoming_img.content_type
			ret_image.URL = incoming_img.url
			ret_image.Alt = incoming_img.alt
			ret_image.Preferred = incoming_img.preferred

			returned_images = append(returned_images, ret_image)
		}

		if len(returned_images) >= len(p.imgTags) {
			break
		}
	}

	sort.Sort(byPreferred(returned_images))

	return returned_images
}

func (t byPreferred) Less(a, b int) bool {
	if t[a].Preferred && !t[b].Preferred {
		return true
	}

	if !t[a].Preferred && t[b].Preferred {
		return false
	}

	a_diff := math.Abs(t[a].AspectRatio - OptimalAspectRatio)
	b_diff := math.Abs(t[b].AspectRatio - OptimalAspectRatio)

	return a_diff < b_diff
}

func (t byPreferred) Swap(a, b int) {
	t[a], t[b] = t[b], t[a]
}

func (t byPreferred) Len() int {
	return len(t)
}

func round(a float64, prec float64) float64 {
	return math.Floor(a/prec+0.5) * prec
}
<commit_msg>Renaming image type<commit_after>package recon

import (
	"fmt"
	"golang.org/x/net/html"
	"io"
	"math"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"sort"
	"time"

	"image/gif"
	"image/jpeg"
	"image/png"
)

type Parser struct {
	client   *http.Client
	req      *http.Request
	metaTags []metaTag
	imgTags  []imgTag
}

type ParseResult struct {
	URL         string    `json:"url"`
	Host        string    `json:"host"`
	Site        string    `json:"site_name"`
	Title       string    `json:"title"`
	Type        string    `json:"type"`
	Description string    `json:"description"`
	Author      string    `json:"author"`
	Publisher   string    `json:"publisher"`
	Images      []Image   `json:"images"`
	Scraped     time.Time `json:"scraped"`
}

type metaTag struct {
	name     string
	value    string
	priority float64
}

type imgTag struct {
	url       string
	alt       string
	preferred bool
}

type Image struct {
	URL         string  `json:"url"`
	Type        string  `json:"type"`
	Width       int     `json:"width"`
	Height      int     `json:"height"`
	Alt         string  `json:"alt"`
	AspectRatio float64 `json:"aspectRatio"`
	Preferred   bool    `json:"preferred,omitempty"`
}

type byPreferred []Image

var targetedProperties = map[string]float64{
	"og:site_name":   1,
	"og:title":       1,
	"og:type":        1,
	"og:description": 1,
	"og:author":      1,
	"og:publisher":   1,
	"og:url":         1,
	"og:image":       1,

	"site_name":   0.5,
	"title":       0.5,
	"type":        0.5,
	"description": 0.5,
	"author":      0.5,
	"publisher":   0.5,
}

var propertyMap = map[string][]string{
	"URL":         []string{"og:url"},
	"Site":        []string{"og:site_name", "site_name"},
	"Title":       []string{"og:title", "title"},
	"Type":        []string{"og:type", "type"},
	"Description": []string{"og:description", "description"},
	"Author":      []string{"og:author", "author"},
	"Publisher":   []string{"og:publisher", "publisher"},
}

var OptimalAspectRatio = 1.91

func NewParser() Parser {
	client := http.DefaultClient
	client.Jar, _ = cookiejar.New(nil)

	return Parser{
		client:   client,
		imgTags:  make([]imgTag, 0),
		metaTags: make([]metaTag, 0),
	}
}

func (p *Parser) Parse(url string) (ParseResult, error) {
	req, err := http.NewRequest("GET", url, nil)

	resp, err := p.client.Do(req)
	if err != nil {
		return ParseResult{}, fmt.Errorf("http error: %s", err)
	}

	p.req = req

	decoder := html.NewTokenizer(resp.Body)
	done := false
	for !done {
		tt := decoder.Next()
		switch tt {
		case html.ErrorToken:
			if decoder.Err() == io.EOF {
				done = true
			} else {
				// fmt.Println(decoder.Err())
			}
		case html.SelfClosingTagToken:
			t := decoder.Token()
			if t.Data == "meta" {
				res := p.parseMeta(t)
				if res.priority > 0 {
					p.metaTags = append(p.metaTags, res)
				}
			} else if t.Data == "img" {
				res := p.parseImg(t)
				if res.url != "" {
					p.imgTags = append(p.imgTags, res)
				}
			}
		}
	}

	res := p.buildResult()

	return res, nil
}

func (p *Parser) parseMeta(t html.Token) metaTag {
	var content string
	var tag string
	var priority float64

	for _, v := range t.Attr {
		if v.Key == "property" || v.Key == "name" {
			if _priority, exists := targetedProperties[v.Val]; exists {
				tag = v.Val
				priority = _priority
			}
		} else if v.Key == "content" {
			content = v.Val
		}
	}

	if priority > 0 {
		return metaTag{name: tag, value: content, priority: priority}
	} else {
		return metaTag{}
	}
}

func (p *Parser) parseImg(t html.Token) (i imgTag) {
	for _, v := range t.Attr {
		if v.Key == "src" {
			i.url = v.Val
		} else if v.Key == "alt" {
			i.alt = v.Val
		}
	}

	if i.url != "" {
		return
	} else {
		return imgTag{}
	}
}

func (p *Parser) buildResult() ParseResult {
	res := ParseResult{}
	if canonical_url := p.getMaxProperty("URL"); canonical_url != "" {
		res.URL = canonical_url
	} else {
		res.URL = p.req.URL.String()
	}
	res.Host = p.req.URL.Host

	res.Site = p.getMaxProperty("Site")
	res.Title = p.getMaxProperty("Title")
	res.Type = p.getMaxProperty("Type")
	res.Description = p.getMaxProperty("Description")
	res.Author = p.getMaxProperty("Author")
	res.Publisher = p.getMaxProperty("Publisher")

	for _, tag := range p.metaTags {
		if tag.name == "og:image" {
			p.imgTags = append(p.imgTags, imgTag{
				url:       tag.value,
				preferred: true,
			})
		}
	}

	res.Images = p.analyzeImages()

	res.Scraped = time.Now()
	return res
}

func (p *Parser) getMaxProperty(key string) (val string) {
	max_priority := 0.0

	for _, search_tag := range propertyMap[key] {
		for _, tag := range p.metaTags {
			if tag.name == search_tag && tag.priority > max_priority {
				val = tag.value
				max_priority = tag.priority
			}
		}
	}

	return
}

func (p *Parser) analyzeImages() []Image {
	type image struct {
		url          string
		data         io.Reader
		alt          string
		content_type string
		preferred    bool
	}

	ch := make(chan image)
	returned_images := make([]Image, 0)
	found_images := 0

	for _, tag := range p.imgTags {
		go func(tag imgTag) {
			img_url, _ := url.Parse(tag.url)
			img_url = p.req.URL.ResolveReference(img_url)

			req, _ := http.NewRequest("GET", img_url.String(), nil)
			resp, _ := p.client.Do(req)

			img := image{
				url:          img_url.String(),
				content_type: resp.Header.Get("Content-Type"),

				data: resp.Body,

				alt:       tag.alt,
				preferred: tag.preferred,
			}

			ch <- img
		}(tag)
		found_images++
	}

	if found_images == 0 {
		return returned_images
	}

	for {
		select {
		case incoming_img := <-ch:
			ret_image := Image{}
			switch incoming_img.content_type {
			case "image/jpeg":
				img, _ := jpeg.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			case "image/gif":
				img, _ := gif.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			case "image/png":
				img, _ := png.Decode(incoming_img.data)
				bounds := img.Bounds()
				ret_image.Width = bounds.Max.X
				ret_image.Height = bounds.Max.Y

			}

			if ret_image.Height > 0 {
				ret_image.AspectRatio = round(float64(ret_image.Width)/float64(ret_image.Height), 1e-4)
			}

			ret_image.Type = incoming_img.content_type
			ret_image.URL = incoming_img.url
			ret_image.Alt = incoming_img.alt
			ret_image.Preferred = incoming_img.preferred

			returned_images = append(returned_images, ret_image)
		}

		if len(returned_images) >= len(p.imgTags) {
			break
		}
	}

	sort.Sort(byPreferred(returned_images))

	return returned_images
}

func (t byPreferred) Less(a, b int) bool {
	if t[a].Preferred && !t[b].Preferred {
		return true
	}

	if !t[a].Preferred && t[b].Preferred {
		return false
	}

	a_diff := math.Abs(t[a].AspectRatio - OptimalAspectRatio)
	b_diff := math.Abs(t[b].AspectRatio - OptimalAspectRatio)

	return a_diff < b_diff
}

func (t byPreferred) Swap(a, b int) {
	t[a], t[b] = t[b], t[a]
}

func (t byPreferred) Len() int {
	return len(t)
}

func round(a float64, prec float64) float64 {
	return math.Floor(a/prec+0.5) * prec
}
<|endoftext|>


<commit_before>package kontrol

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/koding/kite"
	"github.com/koding/kite/dnode"
	"github.com/koding/kite/kontrol/onceevery"
	kontrolprotocol "github.com/koding/kite/kontrol/protocol"
	"github.com/koding/kite/protocol"
)

func (k *Kontrol) handleHeartbeat(rw http.ResponseWriter, req *http.Request) {
	id := req.URL.Query().Get("id")
	if id == "" {
		http.Error(rw, "query id is empty", http.StatusBadRequest)
		return
	}

	k.heartbeatsMu.Lock()
	defer k.heartbeatsMu.Unlock()

	k.log.Debug("Heartbeat received '%s'", id)
	if updateTimer, ok := k.heartbeats[id]; ok {
		// try to reset the timer every time the remote kite sends us a
		// heartbeat. Because the timer get reset, the timer is never fired, so
		// the value get always updated with the updater in the background
		// according to the write interval. If the kite doesn't send any
		// heartbeat, the timer func is being called, which stops the updater
		// so the key is being deleted automatically via the TTL mechanism.
		updateTimer.Reset(HeartbeatInterval + HeartbeatDelay)
		k.heartbeats[id] = updateTimer

		k.log.Debug("Sending pong '%s'", id)
		rw.Write([]byte("pong"))
		return
	}

	// if we reach here than it has several meanings:
	// * kite was registered before, but kontrol is restarted
	// * kite was registered before, but kontrol has lost track
	// * kite was no registered and someone else sends an heartbeat
	// we send back "registeragain" so the caller can be added in to the
	// heartbeats map above.
	k.log.Debug("Sending registeragain '%s'", id)
	rw.Write([]byte("registeragain"))
}

func (k *Kontrol) handleRegisterHTTP(r *kite.Request) (interface{}, error) {
	k.log.Info("Register (via HTTP) request from: %s", r.Client.Kite)

	if r.Args.One().MustMap()["url"].MustString() == "" {
		return nil, errors.New("invalid url")
	}

	var args struct {
		URL string `json:"url"`
	}
	r.Args.One().MustUnmarshal(&args)
	if args.URL == "" {
		return nil, errors.New("empty url")
	}

	// Only accept requests with kiteKey because we need this info
	// for generating tokens for this kite.
	if r.Auth.Type != "kiteKey" {
		return nil, fmt.Errorf("Unexpected authentication type: %s", r.Auth.Type)
	}

	kiteURL := args.URL
	remote := r.Client

	if err := validateKiteKey(&remote.Kite); err != nil {
		return nil, err
	}

	value := &kontrolprotocol.RegisterValue{
		URL: kiteURL,
	}

	// Register first by adding the value to the storage. Return if there is
	// any error.
	if err := k.storage.Upsert(&remote.Kite, value); err != nil {
		k.log.Error("storage add '%s' error: %s", remote.Kite, err)
		return nil, errors.New("internal error - register")
	}

	// if there is already just reset it
	k.heartbeatsMu.Lock()
	defer k.heartbeatsMu.Unlock()

	updateTimer, ok := k.heartbeats[remote.Kite.ID]
	if ok {
		// there is already a previous registration, use it
		k.log.Info("Kite was already register (via HTTP), use timer cache %s", remote.Kite)
		updateTimer.Reset(HeartbeatInterval + HeartbeatDelay)
		k.heartbeats[remote.Kite.ID] = updateTimer
	} else {
		// we create a new ticker which is going to update the key periodically in
		// the storage so it's always up to date. Instead of updating the key
		// periodically according to the HeartBeatInterval below, we are buffering
		// the write speed here with the UpdateInterval.
		stopped := make(chan struct{})
		updater := time.NewTicker(UpdateInterval)
		updaterFunc := func() {
			for {
				select {
				case <-updater.C:
					k.log.Info("Kite is active (via HTTP), updating the value %s", remote.Kite)
					err := k.storage.Update(&remote.Kite, value)
					if err != nil {
						k.log.Error("storage update '%s' error: %s", remote.Kite, err)
					}
				case <-stopped:
					k.log.Info("Kite is nonactive (via HTTP). Updater is closed %s", remote.Kite)
					return
				}
			}
		}
		go updaterFunc()

		// we are now creating a timer that is going to call the function which
		// stops the background updater if it's not resetted. The time is being
		// resetted on a separate HTTP endpoint "/heartbeat"
		k.heartbeats[remote.Kite.ID] = time.AfterFunc(HeartbeatInterval+HeartbeatDelay, func() {
			k.log.Info("Kite didn't sent any heartbeat (via HTTP). Stopping the updater %s",
				remote.Kite)
			// stop the updater so it doesn't update it in the background
			updater.Stop()

			k.heartbeatsMu.Lock()
			defer k.heartbeatsMu.Unlock()

			select {
			case <-stopped:
			default:
				close(stopped)
			}

			delete(k.heartbeats, remote.Kite.ID)
		})
	}

	k.log.Info("Kite registered (via HTTP): %s", remote.Kite)

	// send response back to the kite, also identify him with the new name
	return &protocol.RegisterResult{
		URL:               args.URL,
		HeartbeatInterval: int64(HeartbeatInterval / time.Second),
	}, nil
}

func (k *Kontrol) handleRegister(r *kite.Request) (interface{}, error) {
	k.log.Info("Register request from: %s", r.Client.Kite)

	if r.Args.One().MustMap()["url"].MustString() == "" {
		return nil, errors.New("invalid url")
	}

	var args struct {
		URL string `json:"url"`
	}
	r.Args.One().MustUnmarshal(&args)
	if args.URL == "" {
		return nil, errors.New("empty url")
	}

	// Only accept requests with kiteKey because we need this info
	// for generating tokens for this kite.
	if r.Auth.Type != "kiteKey" {
		return nil, fmt.Errorf("Unexpected authentication type: %s", r.Auth.Type)
	}

	kiteURL := args.URL
	remote := r.Client

	if err := validateKiteKey(&remote.Kite); err != nil {
		return nil, err
	}

	value := &kontrolprotocol.RegisterValue{
		URL: kiteURL,
	}

	// Register first by adding the value to the storage. Return if there is
	// any error.
	if err := k.storage.Upsert(&remote.Kite, value); err != nil {
		k.log.Error("storage add '%s' error: %s", remote.Kite, err)
		return nil, errors.New("internal error - register")
	}

	every := onceevery.New(UpdateInterval)

	ping := make(chan struct{}, 1)
	closed := false

	updaterFunc := func() {
		for {
			select {
			case <-ping:
				k.log.Debug("Kite is active, got a ping %s", remote.Kite)
				every.Do(func() {
					k.log.Info("Kite is active, updating the value %s", remote.Kite)
					err := k.storage.Update(&remote.Kite, value)
					if err != nil {
						k.log.Error("storage update '%s' error: %s", remote.Kite, err)
					}
				})
			case <-time.After(HeartbeatInterval + HeartbeatDelay):
				k.log.Info("Kite didn't sent any heartbeat %s.", remote.Kite)
				every.Stop()
				closed = true
				return
			}
		}
	}
	go updaterFunc()

	heartbeatArgs := []interface{}{
		HeartbeatInterval / time.Second,
		dnode.Callback(func(args *dnode.Partial) {
			k.log.Debug("Kite send us an heartbeat. %s", remote.Kite)

			k.clientLocks.Get(remote.Kite.ID).Lock()
			defer k.clientLocks.Get(remote.Kite.ID).Unlock()

			select {
			case ping <- struct{}{}:
			default:
			}

			// seems we miss a heartbeat, so start it again!
			if closed {
				closed = false
				k.log.Warning("Updater was closed, but we are still getting heartbeats. Starting again %s",
					remote.Kite)

				// it might be removed because the ttl cleaner would come
				// before us, so try to add it again, the updater will than
				// continue to update it afterwards.
				k.storage.Upsert(&remote.Kite, value)
				go updaterFunc()
			}
		}),
	}

	// now trigger the remote kite so it sends us periodically an heartbeat
	remote.GoWithTimeout("kite.heartbeat", 4*time.Second, heartbeatArgs...)

	k.log.Info("Kite registered: %s", remote.Kite)

	remote.OnDisconnect(func() {
		k.log.Info("Kite disconnected: %s", remote.Kite)
		every.Stop()
	})

	// send response back to the kite, also identify him with the new name
	return &protocol.RegisterResult{URL: args.URL}, nil
}

func (k *Kontrol) handleGetKites(r *kite.Request) (interface{}, error) {
	// This type is here until inversion branch is merged.
	// Reason: We can't use the same struct for marshaling and unmarshaling.
	// TODO use the struct in protocol
	type GetKitesArgs struct {
		Query *protocol.KontrolQuery `json:"query"`
	}

	var args GetKitesArgs
	r.Args.One().MustUnmarshal(&args)

	query := args.Query

	// audience will go into the token as "aud" claim.
	audience := getAudience(query)

	// Generate token once here because we are using the same token for every
	// kite we return and generating many tokens is really slow.
	token, err := generateToken(audience, r.Username,
		k.Kite.Kite().Username, k.privateKey)
	if err != nil {
		return nil, err
	}

	// Get kites from the storage
	kites, err := k.storage.Get(query)
	if err != nil {
		return nil, err
	}

	// Attach tokens to kites
	kites.Attach(token)

	return &protocol.GetKitesResult{
		Kites: kites,
	}, nil
}

func (k *Kontrol) handleGetToken(r *kite.Request) (interface{}, error) {
	var query *protocol.KontrolQuery
	err := r.Args.One().Unmarshal(&query)
	if err != nil {
		return nil, errors.New("Invalid query")
	}

	// check if it's exist
	kites, err := k.storage.Get(query)
	if err != nil {
		return nil, err
	}

	if len(kites) > 1 {
		return nil, errors.New("query matches more than one kite")
	}

	audience := getAudience(query)

	return generateToken(audience, r.Username, k.Kite.Kite().Username, k.privateKey)
}

func (k *Kontrol) handleMachine(r *kite.Request) (interface{}, error) {
	if k.MachineAuthenticate != nil {
		if err := k.MachineAuthenticate(r); err != nil {
			return nil, errors.New("cannot authenticate user")
		}
	}

	username := r.Args.One().MustString() // username should be send as an argument
	return k.registerUser(username)
}
<commit_msg>kontrol: this has to much output, disable it<commit_after>package kontrol

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/koding/kite"
	"github.com/koding/kite/dnode"
	"github.com/koding/kite/kontrol/onceevery"
	kontrolprotocol "github.com/koding/kite/kontrol/protocol"
	"github.com/koding/kite/protocol"
)

func (k *Kontrol) handleHeartbeat(rw http.ResponseWriter, req *http.Request) {
	id := req.URL.Query().Get("id")
	if id == "" {
		http.Error(rw, "query id is empty", http.StatusBadRequest)
		return
	}

	k.heartbeatsMu.Lock()
	defer k.heartbeatsMu.Unlock()

	k.log.Debug("Heartbeat received '%s'", id)
	if updateTimer, ok := k.heartbeats[id]; ok {
		// try to reset the timer every time the remote kite sends us a
		// heartbeat. Because the timer get reset, the timer is never fired, so
		// the value get always updated with the updater in the background
		// according to the write interval. If the kite doesn't send any
		// heartbeat, the timer func is being called, which stops the updater
		// so the key is being deleted automatically via the TTL mechanism.
		updateTimer.Reset(HeartbeatInterval + HeartbeatDelay)
		k.heartbeats[id] = updateTimer

		k.log.Debug("Sending pong '%s'", id)
		rw.Write([]byte("pong"))
		return
	}

	// if we reach here than it has several meanings:
	// * kite was registered before, but kontrol is restarted
	// * kite was registered before, but kontrol has lost track
	// * kite was no registered and someone else sends an heartbeat
	// we send back "registeragain" so the caller can be added in to the
	// heartbeats map above.
	k.log.Debug("Sending registeragain '%s'", id)
	rw.Write([]byte("registeragain"))
}

func (k *Kontrol) handleRegisterHTTP(r *kite.Request) (interface{}, error) {
	k.log.Info("Register (via HTTP) request from: %s", r.Client.Kite)

	if r.Args.One().MustMap()["url"].MustString() == "" {
		return nil, errors.New("invalid url")
	}

	var args struct {
		URL string `json:"url"`
	}
	r.Args.One().MustUnmarshal(&args)
	if args.URL == "" {
		return nil, errors.New("empty url")
	}

	// Only accept requests with kiteKey because we need this info
	// for generating tokens for this kite.
	if r.Auth.Type != "kiteKey" {
		return nil, fmt.Errorf("Unexpected authentication type: %s", r.Auth.Type)
	}

	kiteURL := args.URL
	remote := r.Client

	if err := validateKiteKey(&remote.Kite); err != nil {
		return nil, err
	}

	value := &kontrolprotocol.RegisterValue{
		URL: kiteURL,
	}

	// Register first by adding the value to the storage. Return if there is
	// any error.
	if err := k.storage.Upsert(&remote.Kite, value); err != nil {
		k.log.Error("storage add '%s' error: %s", remote.Kite, err)
		return nil, errors.New("internal error - register")
	}

	// if there is already just reset it
	k.heartbeatsMu.Lock()
	defer k.heartbeatsMu.Unlock()

	updateTimer, ok := k.heartbeats[remote.Kite.ID]
	if ok {
		// there is already a previous registration, use it
		k.log.Info("Kite was already register (via HTTP), use timer cache %s", remote.Kite)
		updateTimer.Reset(HeartbeatInterval + HeartbeatDelay)
		k.heartbeats[remote.Kite.ID] = updateTimer
	} else {
		// we create a new ticker which is going to update the key periodically in
		// the storage so it's always up to date. Instead of updating the key
		// periodically according to the HeartBeatInterval below, we are buffering
		// the write speed here with the UpdateInterval.
		stopped := make(chan struct{})
		updater := time.NewTicker(UpdateInterval)
		updaterFunc := func() {
			for {
				select {
				case <-updater.C:
					k.log.Debug("Kite is active (via HTTP), updating the value %s", remote.Kite)
					err := k.storage.Update(&remote.Kite, value)
					if err != nil {
						k.log.Error("storage update '%s' error: %s", remote.Kite, err)
					}
				case <-stopped:
					k.log.Info("Kite is nonactive (via HTTP). Updater is closed %s", remote.Kite)
					return
				}
			}
		}
		go updaterFunc()

		// we are now creating a timer that is going to call the function which
		// stops the background updater if it's not resetted. The time is being
		// resetted on a separate HTTP endpoint "/heartbeat"
		k.heartbeats[remote.Kite.ID] = time.AfterFunc(HeartbeatInterval+HeartbeatDelay, func() {
			k.log.Info("Kite didn't sent any heartbeat (via HTTP). Stopping the updater %s",
				remote.Kite)
			// stop the updater so it doesn't update it in the background
			updater.Stop()

			k.heartbeatsMu.Lock()
			defer k.heartbeatsMu.Unlock()

			select {
			case <-stopped:
			default:
				close(stopped)
			}

			delete(k.heartbeats, remote.Kite.ID)
		})
	}

	k.log.Info("Kite registered (via HTTP): %s", remote.Kite)

	// send response back to the kite, also identify him with the new name
	return &protocol.RegisterResult{
		URL:               args.URL,
		HeartbeatInterval: int64(HeartbeatInterval / time.Second),
	}, nil
}

func (k *Kontrol) handleRegister(r *kite.Request) (interface{}, error) {
	k.log.Info("Register request from: %s", r.Client.Kite)

	if r.Args.One().MustMap()["url"].MustString() == "" {
		return nil, errors.New("invalid url")
	}

	var args struct {
		URL string `json:"url"`
	}
	r.Args.One().MustUnmarshal(&args)
	if args.URL == "" {
		return nil, errors.New("empty url")
	}

	// Only accept requests with kiteKey because we need this info
	// for generating tokens for this kite.
	if r.Auth.Type != "kiteKey" {
		return nil, fmt.Errorf("Unexpected authentication type: %s", r.Auth.Type)
	}

	kiteURL := args.URL
	remote := r.Client

	if err := validateKiteKey(&remote.Kite); err != nil {
		return nil, err
	}

	value := &kontrolprotocol.RegisterValue{
		URL: kiteURL,
	}

	// Register first by adding the value to the storage. Return if there is
	// any error.
	if err := k.storage.Upsert(&remote.Kite, value); err != nil {
		k.log.Error("storage add '%s' error: %s", remote.Kite, err)
		return nil, errors.New("internal error - register")
	}

	every := onceevery.New(UpdateInterval)

	ping := make(chan struct{}, 1)
	closed := false

	updaterFunc := func() {
		for {
			select {
			case <-ping:
				k.log.Debug("Kite is active, got a ping %s", remote.Kite)
				every.Do(func() {
					k.log.Info("Kite is active, updating the value %s", remote.Kite)
					err := k.storage.Update(&remote.Kite, value)
					if err != nil {
						k.log.Error("storage update '%s' error: %s", remote.Kite, err)
					}
				})
			case <-time.After(HeartbeatInterval + HeartbeatDelay):
				k.log.Info("Kite didn't sent any heartbeat %s.", remote.Kite)
				every.Stop()
				closed = true
				return
			}
		}
	}
	go updaterFunc()

	heartbeatArgs := []interface{}{
		HeartbeatInterval / time.Second,
		dnode.Callback(func(args *dnode.Partial) {
			k.log.Debug("Kite send us an heartbeat. %s", remote.Kite)

			k.clientLocks.Get(remote.Kite.ID).Lock()
			defer k.clientLocks.Get(remote.Kite.ID).Unlock()

			select {
			case ping <- struct{}{}:
			default:
			}

			// seems we miss a heartbeat, so start it again!
			if closed {
				closed = false
				k.log.Warning("Updater was closed, but we are still getting heartbeats. Starting again %s",
					remote.Kite)

				// it might be removed because the ttl cleaner would come
				// before us, so try to add it again, the updater will than
				// continue to update it afterwards.
				k.storage.Upsert(&remote.Kite, value)
				go updaterFunc()
			}
		}),
	}

	// now trigger the remote kite so it sends us periodically an heartbeat
	remote.GoWithTimeout("kite.heartbeat", 4*time.Second, heartbeatArgs...)

	k.log.Info("Kite registered: %s", remote.Kite)

	remote.OnDisconnect(func() {
		k.log.Info("Kite disconnected: %s", remote.Kite)
		every.Stop()
	})

	// send response back to the kite, also identify him with the new name
	return &protocol.RegisterResult{URL: args.URL}, nil
}

func (k *Kontrol) handleGetKites(r *kite.Request) (interface{}, error) {
	// This type is here until inversion branch is merged.
	// Reason: We can't use the same struct for marshaling and unmarshaling.
	// TODO use the struct in protocol
	type GetKitesArgs struct {
		Query *protocol.KontrolQuery `json:"query"`
	}

	var args GetKitesArgs
	r.Args.One().MustUnmarshal(&args)

	query := args.Query

	// audience will go into the token as "aud" claim.
	audience := getAudience(query)

	// Generate token once here because we are using the same token for every
	// kite we return and generating many tokens is really slow.
	token, err := generateToken(audience, r.Username,
		k.Kite.Kite().Username, k.privateKey)
	if err != nil {
		return nil, err
	}

	// Get kites from the storage
	kites, err := k.storage.Get(query)
	if err != nil {
		return nil, err
	}

	// Attach tokens to kites
	kites.Attach(token)

	return &protocol.GetKitesResult{
		Kites: kites,
	}, nil
}

func (k *Kontrol) handleGetToken(r *kite.Request) (interface{}, error) {
	var query *protocol.KontrolQuery
	err := r.Args.One().Unmarshal(&query)
	if err != nil {
		return nil, errors.New("Invalid query")
	}

	// check if it's exist
	kites, err := k.storage.Get(query)
	if err != nil {
		return nil, err
	}

	if len(kites) > 1 {
		return nil, errors.New("query matches more than one kite")
	}

	audience := getAudience(query)

	return generateToken(audience, r.Username, k.Kite.Kite().Username, k.privateKey)
}

func (k *Kontrol) handleMachine(r *kite.Request) (interface{}, error) {
	if k.MachineAuthenticate != nil {
		if err := k.MachineAuthenticate(r); err != nil {
			return nil, errors.New("cannot authenticate user")
		}
	}

	username := r.Args.One().MustString() // username should be send as an argument
	return k.registerUser(username)
}
<|endoftext|>